代理下载网页，挖掘数据

原创于 2012-09-25 13:51:35 发布 · 102 阅读

0 ·

CC 4.0 BY-SA版权

java 专栏收录该内容

61 篇文章

订阅专栏

URL url = new URL("http://blog.youkuaiyun.com/mywait_00/article/details/1698627");

//设置代理
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("openproxy.fsfd.com", 8080));
//打开代理
URLConnection coon = url.openConnection(proxy);
//访问的时候需要设置 user-agent
coon.setRequestProperty("User-Agent","Mozila/4.0(compatible;MSIE 5.0;Windows XP;DigExt");

BufferedReader in = new BufferedReader(new InputStreamReader(coon.getInputStream()));

String inputLine; StringBuffer html = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
    html.append(inputLine);
}