HttpClient+Jsoup实现爬虫数据

最新推荐文章于 2022-08-27 21:19:02 发布

原创

最新推荐文章于 2022-08-27 21:19:02 发布 · 370 阅读

1 ·

CC 4.0 BY-SA版权

本文介绍了如何利用Httpclient进行网络请求，并结合Jsoup解析网页数据，包括Httpclient的简单入门，如Get和Post请求的操作，以及HttpClient连接池的配置。同时讲解了Jsoup的使用和Httpclient工具包在实际项目中的应用。

一：Httpclient 简单入门
抓数据
简单4个步骤

 public static void main(String[] args) throws Exception {
   
   
       //1.打开浏览器:创建httpclient对象 2.输入网址 3.按回车，使用httpclient发起请求 4.解析响应，获取数据
        CloseableHttpClient httpClient=HttpClients.createDefault();
       HttpGet httpGet=new HttpGet("http://www.itcast.cn");
       CloseableHttpResponse response=httpClient.execute(httpGet);
       if(response.getStatusLine().getStatusCode()==200){
   
   
           HttpEntity httpEntity=response.getEntity();
           String content=EntityUtils.toString(httpEntity,"utf8");
           System.out.println(content);

       }

Get请求

public class HttpGetTest {
   
   
    public static void main(String[] args) {
   
   
        /*创建HttpClient对象
        创建HttpGet，设置url
         获取response  解析相应
         */
       CloseableHttpClient httpClient=HttpClients.createDefault();
       HttpGet httpGet=new HttpGet("http://www.itcast.cn");
       CloseableHttpResponse response=null;
        try {
   
   
             response=httpClient.execute(httpGet);
 if(response.getStatusLine().getStatusCode()==200){
   
   
     String content=EntityUtils.toString(response.getEntity(),"utf8");
     System.out.println(content.length());
 }
        } catch (IOException e) {
   
   
            e.printStackTrace();
        }finally {
   
   
            try {
   
   
                response.close();
            } catch (IOException e) {
   
   
                e.printStackTrace();
            }
            try {
   
   
                httpClient.close();
            } catch (IOException e) {
   
   
                e.printStackTrace();
            }

        }

    }
}

get请求带参数

public class HttpGetParamTest {
   
   
    public static void main(String[] args) throws Exception {
   
   
        /*创建HttpClient对象
        创建HttpGet，设置url
         获取response  解析相应
         */
        CloseableHttpClient httpClient=HttpClients.createDefault();
        //设置请求地址：http://yun.itheima.com/search?keys=Java
        //创建URIBuilder
        URIBuilder uriBuilder=new URIBuilder("http://yun.itheima.com/search");
        //设置参数
        uriBuilder.setParameter("keys","Java");

        HttpGet httpGet=new HttpGet(uriBuilder.build());
        System.out.println("发起请求信息："+httpGet);
        CloseableHttpResponse response=null;
        try {
   
   
            response=httpClient.execute(httpGet);
            if(response.getStatusLine().getStatusCode()==200){
   
   
                String content=EntityUtils.toString(response.getEntity(),"utf8");
                System.out.println(content.length());
            }
        } catch (IOException e) {
   
   
            e.printStackTrace();
        }finally {
   
   
            try {
   
   
                response.close();
            } catch (IOException e) {
   
   
                e.printStackTrace();
            }
            try {
   
   
                httpClient.close();
            } catch (IOException e) {
   
   
                e.printStackTrace();
            }

Post请求

public class HttpPostTest {
   
   
    public static void main(String[] args) {
   
   
        /*创建HttpClient对象
        创建HttpGet，设置url
         获取response  解析相应
         */
       CloseableHttpClient httpClient=HttpClients.createDefault();
       HttpPost httpPost=new HttpPost("http://www.itcast.cn");
       CloseableHttpResponse response=null;
        try {
   
   
             response=httpClient.execute(httpPost);
 if(response.getStatusLine().getStatusCode()==200){
   
   
     String content=EntityUtils.toString(response.getEntity(),"utf8");
     System.out.println(content.length