java 多线程爬虫

java 多线程爬虫

Executors类 提供工厂方法用来创建不同类型的线程池。Executors是工具类，他提供对ThreadPoolExecutor的封装，会产生几种线程池供大家使用。

• 线程池demo

package spider;
import java.util.concurrent.Executorservice;
import java.util.concurrent.Executors ;
class Spd{
	public static void gets(int page){
		System.out.printin(page);
	}
}
public class MultiThreadExample{
	public static void main(string[] args){
		// 创建固定大小的线程池，包含5个线程
	ExecutorService executorservice = Executors.newFixedThreadPool(5);
		long startTime =System.currentTimeMillis();
		//提交100个任务给线程池执行
		for(int i=1:i<= 100:i++){
			// 需要使用 final 或 effectively final 的变量
			final int page =i;
			executorService.submit(()->Spd.gets(page));
		}
		long endTime =System.currentTimeMillisO;
		long executionTime =endTimestartTime;
		System.out.printIn("Total execution time:" + executionTime + " milliseconds");
		//关闭线程池
		executorService.shutdown();
		}
	}

• 线程池提交任务写法

import java.util.concurrent.Executorservice:
import java.util.concurrent.Executors;
public static void getTest(){
	//创建固定大小的线程池
	String base_url = "https://wzzdg.sun0769.com/political/index/politicsNewest";
	//1、使用工厂类--获职线程池对象
	ExecutorService executorService = Executors.newFixedThreadpo61(5);
	for(int i=1;i<20;i++){
		final int page =i;// 使用fina]修饰符声明不可变的局部变量
		System.out.print1n("开始第几页:"+ page);//打印当前页数
		//提交任务给线程池执行
		executorservice.submit(()->{
			try {
				String url=base_url+"?id=1&page=" + page;//执行爬取逻辑
				String content=crawl(url);
				//处理爬取结果
				process(content);
			}catch(Exception e){
				e.printstackTrace();
				}
			});
		}
		//关闭线程池
		executorservice.shutdown():
	}

• 异步线程执行

public static void getTest(){
	ExecutorService executorService = Executors.newFixedThreadPool(5);
	
	for (int i=1;i<20:i++){
		final int page =i;
		System.out.printIn("开始第几页:"+page);
		CompletableFuture.runAsync(()->{
			try {
				String url=BASE_URL +"?id=1&page=" + page;
				String content =crawl(url);
				process(content);
			} catch(xceptione) {
				e.printStackTrace()
			}
		}，executorService);
	}
	executorService.shutdown();
}