这两天研究了一下异步的httpclient ---- httpAsyncClient
原来使用httpclient都是同步的,如果项目中有大量的httpclient的话,可能会造成阻塞,如果使用异步请求的话可以避免这些问题
可以用在调用第三方接口或者不需要知道请求返回结果的场景下
于是写了一个工具类来封装了同步异步httpclient
github地址:https://github.com/a63881763/HttpAsyncClientUtils
1.首先需要在pom中添加需要的jar
<properties>
<!-- log4j日志包版本号 -->
<slf4j.version>1.7.25</slf4j.version>
<log4j.version>2.8.2</log4j.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.1</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
<version>4.4.6</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId>
<version>4.3.1</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpasyncclient</artifactId>
<version>4.1.3</version>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>2.6</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.slf4j/jcl-over-slf4j -->
<!--用log4j接管commons-logging-->
<!-- log配置:Log4j2 + Slf4j -->
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>${log4j.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>${log4j.version}</version>
</dependency>
<dependency> <!-- 桥接:告诉Slf4j使用Log4j2 -->
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<version>${log4j.version}</version>
</dependency>
<dependency> <!-- 桥接:告诉commons logging使用Log4j2 -->
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-jcl</artifactId>
<version>${log4j.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.39</version>
</dependency>
</dependencies>
import org.apache.http.Consts;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthSchemeProvider;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.MalformedChallengeException;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.AuthSchemes;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.config.ConnectionConfig;
import org.apache.http.config.Lookup;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.ssl.SSLContexts;
import org.apache.http.impl.auth.*;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
import org.apache.http.impl.nio.client.HttpAsyncClients;
import org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager;
import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor;
import org.apache.http.impl.nio.reactor.IOReactorConfig;
import org.apache.http.nio.conn.NoopIOSessionStrategy;
import org.apache.http.nio.conn.SchemeIOSessionStrategy;
import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
import org.apache.http.nio.reactor.ConnectingIOReactor;
import org.apache.http.nio.reactor.IOReactorException;
import javax.net.ssl.SSLContext;
import java.nio.charset.CodingErrorAction;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.UnrecoverableKeyException;
/**
* 异步的HTTP请求对象,可设置代理
*/
public class HttpAsyncClient {
private static int socketTimeout = 1000;// 设置等待数据超时时间5秒钟 根据业务调整
private static int connectTimeout = 2000;// 连接超时
private static int poolSize = 3000;// 连接池最大连接数
private static int maxPerRoute = 1500;// 每个主机的并发最多只有1500
// http代理相关参数
private String host = "";
private int port = 0;
private String username = "";
private String password = "";
// 异步httpclient
private CloseableHttpAsyncClient asyncHttpClient;
// 异步加代理的httpclient
private CloseableHttpAsyncClient proxyAsyncHttpClient;
public HttpAsyncClient() {
try {
this.asyncHttpClient = createAsyncClient(false);
this.proxyAsyncHttpClient = createAsyncClient(true);
} catch (Exception e) {
e.printStackTrace();
}
}
public CloseableHttpAsyncClient createAsyncClient(boolean proxy)
throws KeyManagementException, UnrecoverableKeyException,
NoSuchAlgorithmException, KeyStoreException,
MalformedChallengeException, IOReactorException {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(connectTimeout)
.setSocketTimeout(socketTimeout).build();
SSLContext sslcontext = SSLContexts.createDefault();
UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(
username, password);
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, credentials);
// 设置协议http和https对应的处理socket链接工厂的对象
Registry<SchemeIOSessionStrategy> sessionStrategyRegistry = RegistryBuilder
.<SchemeIOSessionStrategy> create()
.register("http", NoopIOSessionStrategy.INSTANCE)
.register("https", new SSLIOSessionStrategy(sslcontext))
.build();
// 配置io线程
IOReactorConfig ioReactorConfig = IOReactorConfig.custom()
.setIoThreadCount(Runtime.getRuntime().availableProcessors())
.build();
// 设置连接池大小
ConnectingIOReactor ioReactor;
ioReactor = new DefaultConnectingIOReactor(ioReactorConfig);
PoolingNHttpClientConnectionManager conMgr = new PoolingNHttpClientConnectionManager(
ioReactor, null, sessionStrategyRegistry, null);
if (poolSize > 0) {
conMgr.setMaxTotal(poolSize);
}
if (maxPerRoute > 0) {
conMgr.setDefaultMaxPerRoute(maxPerRoute);
} else {
conMgr.setDefaultMaxPerRoute(10);
}
ConnectionConfig connectionConfig = ConnectionConfig.custom()
.setMalformedInputAction(CodingErrorAction.IGNORE)
.setUnmappableInputAction(CodingErrorAction.IGNORE)
.setCharset(Consts.UTF_8).build();
Lookup<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder
.<AuthSchemeProvider> create()
.register(AuthSchemes.BASIC, new BasicSchemeFactory())
.register(AuthSchemes.DIGEST, new DigestSchemeFactory())
.register(AuthSchemes.NTLM, new NTLMSchemeFactory())
.register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory())
.register(AuthSchemes.KERBEROS, new KerberosSchemeFactory())
.build();
conMgr.setDefaultConnectionConfig(connectionConfig);
if (proxy) {
return HttpAsyncClients.custom().setConnectionManager(conMgr)
.setDefaultCredentialsProvider(credentialsProvider)
.setDefaultAuthSchemeRegistry(authSchemeRegistry)
.setProxy(new HttpHost(host, port))
.setDefaultCookieStore(new BasicCookieStore())
.setDefaultRequestConfig(requestConfig).build();
} else {
return HttpAsyncClients.custom().setConnectionManager(conMgr)
.setDefaultCredentialsProvider(credentialsProvider)
.setDefaultAuthSchemeRegistry(authSchemeRegistry)
.setDefaultCookieStore(new BasicCookieStore()).build();
}
}
public CloseableHttpAsyncClient getAsyncHttpClient() {
return asyncHttpClient;
}
public CloseableHttpAsyncClient getProxyAsyncHttpClient() {
return proxyAsyncHttpClient;
}
}
3.httpclient的工厂类
/**
*
* httpclient 工厂类
* */
public class HttpClientFactory {
private static HttpAsyncClient httpAsyncClient = new HttpAsyncClient();
private static HttpSyncClient httpSyncClient = new HttpSyncClient();
private HttpClientFactory() {
}
private static HttpClientFactory httpClientFactory = new HttpClientFactory();
public static HttpClientFactory getInstance() {
return httpClientFactory;
}
public HttpAsyncClient getHttpAsyncClientPool() {
return httpAsyncClient;
}
public HttpSyncClient getHttpSyncClientPool() {
return httpSyncClient;
}
}
4.httpclient的util
2017.11.22
发现每次异步连接回调成功后,连接总是延迟很久关,查了不少资料,
发现httpAsyncClient默认是长连接,所以在不需要长连接的时候记得要把
请求头中的connection设成false
/**
*
* http client 业务逻辑处理类
* */
public class HttpClientUtil {
private static Logger LOG = LoggerFactory
.getLogger(HttpClientUtil.class);
private static String utf8Charset = "utf-8";
/**
* 向指定的url发送一次post请求,参数是List<NameValuePair>
* @param baseUrl 请求地址
* @param list 请求参数,格式是List<NameValuePair>
* @return 返回结果,请求失败时返回null
* @apiNote http接口处用 @RequestParam接收参数
*/
public static String httpSyncPost(String baseUrl,List<BasicNameValuePair> list) {
CloseableHttpClient httpClient = HttpClientFactory.getInstance().getHttpSyncClientPool().getHttpClient();
HttpPost httpPost = new HttpPost(baseUrl);
//Parameters
LOG.warn("==== Parameters ======" +list);
CloseableHttpResponse response = null;
try {
httpPost.setEntity(new UrlEncodedFormEntity(list));
response = httpClient.execute(httpPost);
LOG.warn("========HttpResponseProxy:========"+response.getStatusLine());
HttpEntity entity = response.getEntity();
String result = null;
if(entity != null){
result = EntityUtils.toString(entity, "UTF-8");
LOG.warn("========Response=======" +result);
}
EntityUtils.consume(entity);
return result;
} catch (Exception e) {
e.printStackTrace();
}finally {
if(response != null){
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return null;
}
/**
* 向指定的url发送一次post请求,参数是字符串
* @param baseUrl 请求地址
* @param postString 请求参数,格式是json.toString()
* @return 返回结果,请求失败时返回null
* @apiNote http接口处用 @RequestBody接收参数
*/
public static String httpSyncPost(String baseUrl, String postString) {
CloseableHttpClient httpClient = HttpClientFactory.getInstance().getHttpSyncClientPool().getHttpClient();
HttpPost httpPost = new HttpPost(baseUrl);
//parameters
LOG.warn("==== Parameters ======" +postString);
CloseableHttpResponse response = null;
try {
if(postString == null || "".equals(postString)){
throw new Exception("missing post String");
}
StringEntity stringEntity = new StringEntity(postString.toString(), utf8Charset);
stringEntity.setContentEncoding("UTF-8");
stringEntity.setContentType("application/json");
httpPost.setEntity(stringEntity);
response = httpClient.execute(httpPost);
LOG.warn("========HttpResponseProxy:========"+response.getStatusLine());
HttpEntity entity = response.getEntity();
String result = null;
if(entity != null){
result = EntityUtils.toString(entity, "UTF-8");
LOG.warn("========Response=======" +result);
}
EntityUtils.consume(entity);
return result;
} catch (Exception e) {
e.printStackTrace();
}finally {
if(response != null){
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return null;
}
/**
* 向指定的url发送一次get请求,参数是List<NameValuePair>
* @param baseUrl 请求地址
* @param list 请求参数,格式是List<NameValuePair>
* @return 返回结果,请求失败时返回null
* @apiNote http接口处用 @RequestParam接收参数
*/
public static String httpSyncGet(String baseUrl, List<BasicNameValuePair> list) {
CloseableHttpClient httpClient = HttpClientFactory.getInstance().getHttpSyncClientPool().getHttpClient();
HttpGet httpGet = new HttpGet(baseUrl);
//Parameters
LOG.warn("==== Parameters ======" +list);
CloseableHttpResponse response = null;
try {
if(list != null){
String getUrl = EntityUtils
.toString(new UrlEncodedFormEntity(list));
httpGet.setURI(new URI(httpGet.getURI().toString()
+ "?" + getUrl));
}
else{
httpGet.setURI(new URI(httpGet.getURI().toString()));
}
response = httpClient.execute(httpGet);
LOG.warn("========HttpResponseProxy:========"+response.getStatusLine());
HttpEntity entity = response.getEntity();
String result = null;
if(entity != null){
result = EntityUtils.toString(entity, "UTF-8");
LOG.warn("========Response=======" +result);
}
EntityUtils.consume(entity);
return result;
} catch (Exception e) {
e.printStackTrace();
}finally {
if(response != null){
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return null;
}
/**
* 向指定的url发送一次get请求,参数是字符串
* @param baseUrl 请求地址
* @param urlParams 请求参数,格式是String
* @return 返回结果,请求失败时返回null
* @apiNote http接口处用 @RequestParam接收参数
*/
public static String httpSyncGet(String baseUrl,String urlParams) {
CloseableHttpClient httpClient = HttpClientFactory.getInstance().getHttpSyncClientPool().getHttpClient();
HttpGet httpGet = new HttpGet(baseUrl);
//Parameters
LOG.warn("==== Parameters ======" +urlParams);
CloseableHttpResponse response = null;
try {
if (null != urlParams || "".equals(urlParams)) {
httpGet.setURI(new URI(httpGet.getURI().toString()
+ "?" + urlParams));
}
else{
httpGet.setURI(new URI(httpGet.getURI().toString()));
}
response = httpClient.execute(httpGet);
LOG.warn("========HttpResponseProxy:========"+response.getStatusLine());
HttpEntity entity = response.getEntity();
String result = null;
if(entity != null){
result = EntityUtils.toString(entity, "UTF-8");
LOG.warn("========Response=======" +result);
}
EntityUtils.consume(entity);
return result;
} catch (Exception e) {
e.printStackTrace();
}finally {
if(response != null){
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return null;
}
/**
* 向指定的url发送一次get请求,参数是字符串
* @param baseUrl 请求地址
* @return 返回结果,请求失败时返回null
* @apiNote http接口处用 @RequestParam接收参数
*/
public static String httpSyncGet(String baseUrl) {
CloseableHttpClient httpClient = HttpClientFactory.getInstance().getHttpSyncClientPool().getHttpClient();
HttpGet httpGet = new HttpGet(baseUrl);
CloseableHttpResponse response = null;
try {
httpGet.setURI(new URI(httpGet.getURI().toString()));
response = httpClient.execute(httpGet);
LOG.warn("========HttpResponseProxy:========"+response.getStatusLine());
HttpEntity entity = response.getEntity();
String result = null;
if(entity != null){
result = EntityUtils.toString(entity, "UTF-8");
LOG.warn("========Response=======" +result);
}
EntityUtils.consume(entity);
return result;
} catch (Exception e) {
e.printStackTrace();
}finally {
if(response != null){
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return null;
}
/**
* 向指定的url发送一次异步post请求,参数是字符串
* @param baseUrl 请求地址
* @param postString 请求参数,格式是json.toString()
* @param urlParams 请求参数,格式是String
* @param callback 回调方法,格式是FutureCallback
* @return 返回结果,请求失败时返回null
* @apiNote http接口处用 @RequestParam接收参数
*/
public static void httpAsyncPost(String baseUrl,String postString,
String urlParams,FutureCallback callback) throws Exception {
if (baseUrl == null || "".equals(baseUrl)) {
LOG.warn("we don't have base url, check config");
throw new Exception("missing base url");
}
CloseableHttpAsyncClient hc = HttpClientFactory.getInstance().getHttpAsyncClientPool()
.getAsyncHttpClient();
try {
hc.start();
HttpPost httpPost = new HttpPost(baseUrl);
httpPost.setHeader("Connection","close");
if (null != postString) {
LOG.debug("exeAsyncReq post postBody={}", postString);
StringEntity entity = new StringEntity(postString.toString(), utf8Charset);
entity.setContentEncoding("UTF-8");
entity.setContentType("application/json");
httpPost.setEntity(entity);
}
if (null != urlParams) {
httpPost.setURI(new URI(httpPost.getURI().toString()
+ "?" + urlParams));
}
LOG.warn("exeAsyncReq getparams:" + httpPost.getURI());
hc.execute(httpPost, callback);
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 向指定的url发送一次异步post请求,参数是字符串
* @param baseUrl 请求地址
* @param urlParams 请求参数,格式是List<BasicNameValuePair>
* @param callback 回调方法,格式是FutureCallback
* @return 返回结果,请求失败时返回null
* @apiNote http接口处用 @RequestParam接收参数
*/
public static void httpAsyncPost(String baseUrl, List<BasicNameValuePair> postBody,
List<BasicNameValuePair> urlParams, FutureCallback callback ) throws Exception {
if (baseUrl == null || "".equals(baseUrl)) {
LOG.warn("we don't have base url, check config");
throw new Exception("missing base url");
}
try {
CloseableHttpAsyncClient hc = HttpClientFactory.getInstance().getHttpAsyncClientPool()
.getAsyncHttpClient();
hc.start();
HttpPost httpPost = new HttpPost(baseUrl);
httpPost.setHeader("Connection","close");
if (null != postBody) {
LOG.debug("exeAsyncReq post postBody={}", postBody);
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(
postBody, "UTF-8");
httpPost.setEntity(entity);
}
if (null != urlParams) {
String getUrl = EntityUtils
.toString(new UrlEncodedFormEntity(urlParams));
httpPost.setURI(new URI(httpPost.getURI().toString()
+ "?" + getUrl));
}
LOG.warn("exeAsyncReq getparams:" + httpPost.getURI());
hc.execute(httpPost, callback);
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 向指定的url发送一次异步get请求,参数是String
* @param baseUrl 请求地址
* @param urlParams 请求参数,格式是String
* @param callback 回调方法,格式是FutureCallback
* @return 返回结果,请求失败时返回null
* @apiNote http接口处用 @RequestParam接收参数
*/
public static void httpAsyncGet(String baseUrl,String urlParams,FutureCallback callback) throws Exception {
if (baseUrl == null || "".equals(baseUrl)) {
LOG.warn("we don't have base url, check config");
throw new Exception("missing base url");
}
CloseableHttpAsyncClient hc = HttpClientFactory.getInstance().getHttpAsyncClientPool()
.getAsyncHttpClient();
try {
hc.start();
HttpGet httpGet = new HttpGet(baseUrl);
httpGet.setHeader("Connection","close");
if (null != urlParams || "".equals(urlParams)) {
httpGet.setURI(new URI(httpGet.getURI().toString()
+ "?" + urlParams));
}
else{
httpGet.setURI(new URI(httpGet.getURI().toString()));
}
LOG.warn("exeAsyncReq getparams:" + httpGet.getURI());
hc.execute(httpGet, callback);
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 向指定的url发送一次异步get请求,参数是List<BasicNameValuePair>
* @param baseUrl 请求地址
* @param urlParams 请求参数,格式是List<BasicNameValuePair>
* @param callback 回调方法,格式是FutureCallback
* @return 返回结果,请求失败时返回null
* @apiNote http接口处用 @RequestParam接收参数
*/
public static void httpAsyncGet(String baseUrl, List<BasicNameValuePair> urlParams, FutureCallback callback) throws Exception {
if (baseUrl == null || "".equals(baseUrl)) {
LOG.warn("we don't have base url, check config");
throw new Exception("missing base url");
}
try {
CloseableHttpAsyncClient hc = HttpClientFactory.getInstance().getHttpAsyncClientPool()
.getAsyncHttpClient();
hc.start();
HttpPost httpGet = new HttpPost(baseUrl);
httpGet.setHeader("Connection","close");
if (null != urlParams) {
String getUrl = EntityUtils
.toString(new UrlEncodedFormEntity(urlParams));
httpGet.setURI(new URI(httpGet.getURI().toString()
+ "?" + getUrl));
}
LOG.warn("exeAsyncReq getparams:" + httpGet.getURI());
hc.execute(httpGet, callback);
} catch (Exception e) {
e.printStackTrace();
}
}
}
本文章参考了http://blog.youkuaiyun.com/angjunqiang/article/details/55259170