利用cookie爬取新浪微博place

最新推荐文章于 2020-12-08 07:26:51 发布

原创最新推荐文章于 2020-12-08 07:26:51 发布 · 413 阅读

0 ·

CC 4.0 BY-SA版权

java 专栏收录该内容

10 篇文章

订阅专栏

本文介绍了一种利用Cookie进行网页爬取的方法。通过设置HTTP请求头中的Cookie字段，可以模拟登录状态访问网站并获取受保护的内容。示例代码展示了如何使用Java实现这一过程，并附带了处理异常的具体步骤。

package cookies;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;

import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.AbstractHttpClient;
import org.apache.http.impl.client.DefaultHttpClient;

public class CookieCrawler {

static DefaultHttpClient client = null;
public static HttpPost post = null;
public static HttpResponse response = null;
public static HttpGet getMethod = null;
private AbstractHttpClient client2;

public void sendCookie() {
String str_cookie = "";//填写自己的cookie可以通过浏览器查看，chrome用F12，network里面查看，具体参见资料
// 连接到页面
String path = "test.txt";

try {

URL url2 = new URL("http://weibo.com/p/100101120.181358_30.244699?pids=Pl_Core_MixFeed__7&feed_page=4#feedtop");//定位信息页面
URLConnection conn2 = url2.openConnection();
conn2.setRequestProperty("Cookie", str_cookie);
BufferedReader reader = new BufferedReader(new InputStreamReader(
conn2.getInputStream()));// 仅仅返回登录页面内容
String line = null;
while ((line = reader.readLine()) != null) {
FileOperate.AddWrite(line + "\n", path);
System.out.print(line + "\n");
// document.append(line + "\n");
}

} catch (MalformedURLException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}

public static void main(String[] args) throws IOException {
CookieCrawler cm = new CookieCrawler();
cm.sendCookie();
}

}

参考资料：http://www.douban.com/note/264976536/

http://www.douban.com/note/265628450/