会python真的可以为所欲为0(回)
365天前
这里还有人吗1(回)
550天前这里还有人吗0(回)
550天前
每天面对着电脑屏幕，敲打键盘。我所面对的并不只是代码，而是一种生活方式。0(回)
775天前到处都是羊，不想上班0(回)
855天前
鸽子
0(回)
859天前import java.io.BufferedReader; |
import java.io.IOException; |
import java.io.InputStream; |
import java.io.InputStreamReader; |
import java.net.HttpURLConnection; |
import java.net.URL; |
/** |
* java采集网页 |
* |
*/ |
public class HttpWebCollecter { |
/** |
* 网页抓取方法 |
* |
* @param urlString |
* 要抓取的url地址 |
* @param charset |
* 网页编码方式 |
* @param timeout |
* 超时时间 |
* @return 抓取的网页内容 |
* @throws IOException |
* 抓取异常 |
*/ |
public static String GetWebContent(String urlString, final String charset, |
int timeout) throws IOException { |
if (urlString == null || urlString.length() == 0) { |
return ""; |
} |
urlString = (urlString.startsWith("http://") || urlString |
.startsWith("https://")) ? urlString : ("http://" + urlString) |
.intern(); |
URL url = new URL(urlString); |
HttpURLConnection conn = (HttpURLConnection) url.openConnection(); |
conn.setDoOutput(true); |
conn.setRequestProperty("Pragma", "no-cache"); |
conn.setRequestProperty("Cache-Control", "no-cache"); |
int temp = Integer.parseInt(Math.round(Math.random() |
* (UserAgent.length - 1)) |
+ ""); |
conn.setRequestProperty("User-Agent", UserAgent[temp]); // 模拟手机系统 |
conn.setRequestProperty("Accept", |
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");// 只接受text/html类型，当然也可以接受图片,pdf,*/*任意，就是tomcat/conf/web里面定义那些 |
conn.setConnectTimeout(timeout); |
try { |
if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) { |
return ""; |
} |
} catch (Exception e) { |
try { |
System.out.println(e.getMessage()); |
} catch (Exception e2) { |
e2.printStackTrace(); |
} |
return ""; |
} |
InputStream input = conn.getInputStream(); |
BufferedReader reader = new BufferedReader(new InputStreamReader(input, |
charset)); |
String line = null; |
StringBuffer sb = new StringBuffer(""); |
while ((line = reader.readLine()) != null) { |
sb.append(line).append("\r\n"); |
} |
if (reader != null) { |
reader.close(); |
} |
if (conn != null) { |
conn.disconnect(); |
} |
return sb.toString(); |
} |
public static String[] UserAgent = { |
"Mozilla/5.0 (Linux; U; Android 2.2; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.2", |
"Mozilla/5.0 (iPad; U; CPU OS 3_2_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B500 Safari/531.21.11", |
"Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18121", |
"Nokia5700AP23.01/SymbianOS/9.1 Series60/3.0", |
"UCWEB7.0.2.37/28/998", "NOKIA5700/UCWEB7.0.2.37/28/977", |
"Openwave/UCWEB7.0.2.37/28/978", |
"Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/989" }; |
} |
中级程序员
by: 中国人在美国 发表于：2013-08-01 05:36:37 顶(0) | 踩(0) 回复
谢谢分享
回复评论