HttpClient4.x 多线程请求

最新推荐文章于 2025-07-01 17:50:54 发布

原创最新推荐文章于 2025-07-01 17:50:54 发布 · 2.7k 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#java #多线程

java 专栏收录该内容

4 篇文章

订阅专栏

本文介绍如何使用HttpClient4.x在Java中实现多线程进行HTTP请求，探讨线程池的配置与并发控制，提升请求效率。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

基本过程：使用httpclient发起多线程的请求，将返回的数据临时放在ArrayBlockingQueue中，单独开启一个线程读取ArrayBlockingQueue中的数据并写入磁盘上的文件。

启动请求线程

import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;

public class MultiReq {

//数据返回是否完毕标志位
	public static volatile int status =0;
	public static BlockingQueue<String> bqueue = new ArrayBlockingQueue<String>(1000);
	static ExecutorService exec = Executors.newCachedThreadPool();
	private static String[] urls = {};
	public void req() {
		
		
		
		PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();

//最大线程数
		cm.setMaxTotal(140);
		CloseableHttpClient httpClient = HttpClients.custom()
				.setConnectionManager(cm).build();
		
		for(int i=0,n=urls.length;i<n;i++){
			HttpGet httpget = new HttpGet(urls[i]);
			exec.execute(new GetThread(httpClient,httpget));
		}
		
	}
	public static void main(String[] args){
		MultiReq req = new MultiReq();
		MultiReq.exec.execute(new getData());
		req.req();
		
	}
}

发起请求的线程

package newreq;

import java.io.IOException;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;

import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;

public class GetThread implements Runnable{
    private final CloseableHttpClient httpClient;
    private final HttpContext context;
    private final HttpGet httpget;

    public GetThread(CloseableHttpClient httpClient, HttpGet httpget) {
        this.httpClient = httpClient;
        this.context = HttpClientContext.create();
        this.httpget = httpget;
    }
	@Override
	public void run(){
		try {
            CloseableHttpResponse response = httpClient.execute(
                    httpget, context);
            
            try {
            	
                HttpEntity entity = response.getEntity();
                String res = EntityUtils.toString(entity);
                if(!"[]".equals(res)){
                	MultiReq.bqueue.add(res);
                }
                if(MultiReq.status==0){
	                if(response.getStatusLine().getStatusCode()!=HttpStatus.SC_OK || entity==null || "[]".equals(res)){
	                	synchronized(httpClient){
	                		MultiReq.status=1;
	                	}
	                }
                }
                
            } finally {
                response.close();
            }
        } catch (ClientProtocolException ex) {
            // Handle protocol errors
        } catch (IOException ex) {
            // Handle I/O errors
        }
	}
}

处理返回的数据

import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;

public class getData implements Runnable{

	);
	
	private static BufferedWriter bw = null;
	static{
	try {

		bw =new BufferedWriter(new FileWriter(
				"D:\\file\\0723.txt"));
	} catch (IOException e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	}
	}
	@Override
	public void run(){
		
		while(true){
				if (MultiReq.bqueue.size() > 0 && MultiReq.status == 0) {
					String readed = MultiReq.bqueue.poll();
					bw.write(readed);
                                        bw.flush();
}}}}