Java 爬虫常用的方式

主要有两种解决方式：

一种方式是手动设置 cookie 

第二种方式就是使用程序模拟登陆，通过模拟登陆获取到 cookies

 

 Jsoup 模拟一下手动设置 cookie 方式

/**
 * 手动设置 cookies
 * 先从网站上登录，然后查看 request headers 里面的 cookies
 * @param url
 * @throws IOException
 */
public void setCookies(String url) throws IOException {

    Document document = Jsoup.connect(url)
            // 手动设置cookies
            .header("Cookie", "your cookies")
            .get();
    //
    if (document != null) {
        // 获取豆瓣昵称节点
        Element element = document.select(".info h1").first();
        if (element == null) {
            System.out.println("没有找到 .info h1 标签");
            return;
        }
        // 取出豆瓣节点昵称
        String userName = element.ownText();
        System.out.println("豆瓣我的网名为：" + userName);
    } else {
        System.out.println("出错啦！！！！！");
    }
}

Jsoup 模拟登录

/**
 * Jsoup 模拟登录豆瓣 访问个人中心
 * 在豆瓣登录时先输入一个错误的账号密码，查看到登录所需要的参数
 * 先构造登录请求参数，成功后获取到cookies
 * 设置request cookies，再次请求
 * @param loginUrl 登录url
 * @param userInfoUrl 个人中心url
 * @throws IOException
 */
public void jsoupLogin(String loginUrl,String userInfoUrl)  throws IOException {

    // 构造登陆参数
    Map<String,String> data = new HashMap<>();
    data.put("name","your_account");
    data.put("password","your_password");
    data.put("remember","false");
    data.put("ticket","");
    data.put("ck","");
    Connection.Response login = Jsoup.connect(loginUrl)
            .ignoreContentType(true) // 忽略类型验证
            .followRedirects(false) // 禁止重定向
            .postDataCharset("utf-8")
            .header("Upgrade-Insecure-Requests","1")
            .header("Accept","application/json")
            .header("Content-Type","application/x-www-form-urlencoded")
            .header("X-Requested-With","XMLHttpRequest")
            .header("User-Agent","Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36")
            .data(data)
            .method(Connection.Method.POST)
            .execute();
    login.charset("UTF-8");
    // login 中已经获取到登录成功之后的cookies
    // 构造访问个人中心的请求
    Document document = Jsoup.connect(userInfoUrl)
            // 取出login对象里面的cookies
            .cookies(login.cookies())
            .get();
    if (document != null) {
        Element element = document.select(".info h1").first();
        if (element == null) {
            System.out.println("没有找到 .info h1 标签");
            return;
        }
        String userName = element.ownText();
        System.out.println("豆瓣我的网名为：" + userName);
    } else {
        System.out.println("出错啦！！！！！");
    }
}

httpclient 模拟登录

/**
 * httpclient 的方式模拟登录豆瓣
 * httpclient 跟jsoup差不多，不同的地方在于 httpclient 有session的概念
 * 在同一个httpclient 内不需要设置cookies ，会默认缓存下来
 * @param loginUrl
 * @param userInfoUrl
 */
public void httpClientLogin(String loginUrl,String userInfoUrl) throws Exception{

    CloseableHttpClient httpclient = HttpClients.createDefault();
    HttpUriRequest login = RequestBuilder.post()
            .setUri(new URI(loginUrl))// 登陆url
            .setHeader("Upgrade-Insecure-Requests","1")
            .setHeader("Accept","application/json")
            .setHeader("Content-Type","application/x-www-form-urlencoded")
            .setHeader("X-Requested-With","XMLHttpRequest")
            .setHeader("User-Agent","Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36")
            // 设置账号信息
            .addParameter("name","your_account")
            .addParameter("password","your_password")
            .addParameter("remember","false")
            .addParameter("ticket","")
            .addParameter("ck","")
            .build();
    // 模拟登陆
    CloseableHttpResponse response = httpclient.execute(login);
    if (response.getStatusLine().getStatusCode() == 200){
        // 构造访问个人中心请求
        HttpGet httpGet = new HttpGet(userInfoUrl);
        CloseableHttpResponse user_response = httpclient.execute(httpGet);
        HttpEntity entity = user_response.getEntity();
        //
        String body = EntityUtils.toString(entity, "utf-8");

        // 偷个懒，直接判断 缺心眼那叫单纯 是否存在字符串中
        System.out.println("缺心眼那叫单纯是否查找到？"+(body.contains("缺心眼那叫单纯")));
    }else {
        System.out.println("httpclient 模拟登录豆瓣失败了!!!!");
    }
}