【爬虫】——模拟请求

需求

我们将要爬取人民网某些新闻中的数据，那我们该如何实现呢？

首先我们先实现第一步：根据url，模拟url请求，根据url获取网页中的内容。

模拟请求

public class HttpClientUtil {

    static final int timeOut = 30 * 10000;  //  连接超时时间为3分钟
    /** 
     * 模拟请求 ,支持 https
     *  
     * @param url       资源地址 
     * @param map   参数列表 
     * @param encoding  编码 
     * @return 
     * @throws NoSuchAlgorithmException  
     * @throws KeyManagementException  
     * @throws IOException  
     * @throws ClientProtocolException  
     */  
    public static String post(String url, Map<String,String> map,String encoding) throws KeyManagementException, NoSuchAlgorithmException, ClientProtocolException, IOException {  
        String results = "";  
        //采用绕过验证的方式处理https请求  
        SSLContext sslcontext = SSLClient.createIgnoreVerifySSL();  
           // 设置协议http和https对应的处理socket链接工厂的对象  
        HostnameVerifier hv = new HostnameVerifier() {
            public boolean verify(String urlHostName, SSLSession session) {
                System.out.println("Warning: URL Host: " + urlHostName + " vs. "
                        + session.getPeerHost());
                return true;
            }
        };

        LayeredConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslcontext, hv);

        Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()  
               .register("http", PlainConnectionSocketFactory.INSTANCE)  
               .register("https", sslsf)  
               .build();  

         PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);  

         HttpClients.custom().setConnectionManager(connManager);  

         //设置全局的标准cookie策略  
         RequestConfig config = RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES).build();
           //创建自定义的httpclient对象  
         CloseableHttpClient client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).build();  

         //创建post方式请求对象  
         HttpPost httpPost = new HttpPost(url); 
       //设置请求和传输超时时间  毫秒
         RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeOut).setConnectTimeout(timeOut).build();
         httpPost.setConfig(requestConfig); 

        //装填参数  
         List<NameValuePair> nvps = new ArrayList<NameValuePair>();  
         if(map!=null){  
            for (Entry<String, String> entry : map.entrySet()) {  
                nvps.add(new BasicNameValuePair(entry.getKey(), entry.getValue()));  
            }  
         }  
        //设置参数到请求对象中  
         httpPost.setEntity(new UrlEncodedFormEntity(nvps, encoding));  

         System.out.println("请求地址："+url);  
         System.out.println("请求参数："+nvps.toString());  

        //设置header信息  
        //指定报文头【Content-type】、【User-Agent】  
         httpPost.setHeader("Content-type", "application/x-www-form-urlencoded");  
         httpPost.setHeader("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");  

         //执行请求操作，并拿到结果（同步阻塞）  
         CloseableHttpResponse response = client.execute(httpPost); 

         if(response.getStatusLine().getStatusCode()!=200){//
             System.out.println("*******response.getStatusLine().getStatusCode()!=200*****");
             try {
                Thread.sleep(3000);
                 response = client.execute(httpPost);
            } catch (Exception e) {
                e.printStackTrace();
            }

         }

         /* //
         if (response.getStatusLine().getStatusCode() != 200){
             System.out.println("****************请求返回失败******************"+response.getStatusLine().getStatusCode());
             try {
                Thread.sleep(8000);
                response = client.execute(httpPost); //再连一次
            } catch (InterruptedException e) {
                e.printStackTrace();
            }

         }*/

        //获取结果实体  
         HttpEntity entity = response.getEntity();  
         if (entity != null) {  
            //按指定编码转换结果实体为String类型  
             results = EntityUtils.toString(entity, encoding);  
         }  
         EntityUtils.consume(entity);  
        //释放链接  
         response.close();  
         return results;  
    }  


/**
 * 
 * @param url
 * @param params 可变参数，设置网页编码
 * @return
 * @throws KeyManagementException
 * @throws NoSuchAlgorithmException
 * @throws ClientProtocolException
 * @throws IOException
 */
    public static String get(String url,String ... params) {
        String results = "";
        try {
            String encoding = CommonVariable.encoding_default;
            if (params.length > 0) {
                encoding = params[0];
            }


            //采用绕过验证的方式处理https请求
            SSLContext sslcontext = SSLClient.createIgnoreVerifySSL();
            // 设置协议http和https对应的处理socket链接工厂的对象
            HostnameVerifier hv = new HostnameVerifier() {
                public boolean verify(String urlHostName, SSLSession session) {
                /*System.out.println("Warning: URL Host: " + urlHostName + " vs. "
                        + session.getPeerHost());*/
                    return true;
                }
            };

            LayeredConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslcontext, hv);

            Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
                    .register("http", PlainConnectionSocketFactory.INSTANCE)
                    .register("https", sslsf)
                    .build();

            PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);

            HttpClients.custom().setConnectionManager(connManager);

            //设置全局的标准cookie策略
            RequestConfig config = RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES).build();
            //创建自定义的httpclient对象
            CloseableHttpClient client = HttpClients.custom().setConnectionManager(connManager).setDefaultRequestConfig(config).build();

            //创建GET方式请求对象

            HttpGet httpGet = new HttpGet(url);
            //设置请求和传输超时时间  毫秒
            RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeOut).setConnectTimeout(timeOut).build();
            httpGet.setConfig(requestConfig);

            //设置header信息
            //指定报文头【Content-type】、【User-Agent】
            httpGet.setHeader("Content-type", "application/x-www-form-urlencoded");
            httpGet.setHeader("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");

            //执行请求操作，并拿到结果（同步阻塞）
            CloseableHttpResponse response = client.execute(httpGet);

            if (response.getStatusLine().getStatusCode() != 200) {//
                //System.out.println("*******response.getStatusLine().getStatusCode()!=200*****重连");
               /* try {
                    Thread.sleep(3000);
                    response = client.execute(httpGet);
                } catch (Exception e) {
                    e.printStackTrace();
                }*/
            }

            //获取结果实体
            HttpEntity entity = response.getEntity();
            if (entity != null) {
                //按指定编码转换结果实体为String类型
                results = EntityUtils.toString(entity, encoding);
            }
            EntityUtils.consume(entity);
            //释放链接
            response.close();
        }
        catch (Exception ex)
        {
            ex.printStackTrace();
        }
         return results;  
    }  

}

当然我们这步的实现也可以采取很多优秀的框架，就不用自己写工具类来进行实现啦！

下一节会具体介绍，怎么获取想要的具体内容