java爬虫采集某保险公司产品数据

最新推荐文章于 2025-01-05 16:17:53 发布

Ayub

最新推荐文章于 2025-01-05 16:17:53 发布

阅读量875

点赞数

CC 4.0 BY-SA版权

分类专栏： Java 文章标签：队列爬虫 java css jquery

本文链接：https://blog.youkuaiyun.com/AyubLIbra/article/details/106943936

1、主方法

package com.sprider.pingan;

import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.nio.charset.Charset;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import static com.sprider.utils.SetGetHead.setPinanHead;

/***  财产险
 *
 */
public class Caichan {
   
   

    private final static ArrayBlockingQueue<String> arrayBlockingQueue = new ArrayBlockingQueue<String>(100);
    private final static ExecutorService threadPool = Executors.newFixedThreadPool(10);
    //抓取首页
    private final static  String indexUrl = "http://baoxian.pingan.com/product/allbaoxianlist.shtml";

    public static void main(String[] args) throws Exception {
   
   

        // 使用线程技术消费队列的数据。
        threadPool.execute(new Runnable() {
   
   
            public void run() {
   
   
                while (true) {
   
   
                    System.out.println("等待~~~，还有商品抓取：:" + arrayBlockingQueue.size());
                    try {
   
   
                        Thread.sleep(1000);