httpclient4.3下载远程附件，设置user-agent和伪装成google的refer，并设置timeout。

最新推荐文章于 2021-09-11 07:30:27 发布

原创最新推荐文章于 2021-09-11 07:30:27 发布 · 596 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#JAVA后端

JAVA后端专栏收录该内容

4 篇文章

订阅专栏

本文介绍如何在Java后端使用HttpClient4.3库下载远程附件，同时设置User-Agent为Google浏览器的标识，并指定Referer字段，以模拟来自Google的请求。此外，还将讲解如何为HTTP请求设置超时限制，确保网络延迟或不稳定情况下的操作可靠性。

<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.3.6</version>
</dependency>

<dependency>
    <groupId>org.slf4j</groupId>
    <artifactId>slf4j-api</artifactId>
    <version>1.7.10</version>
</dependency>

<dependency>
    <groupId>org.apache.commons</groupId>
    <artifactId>commons-io</artifactId>
    <version>1.3.2</version>
</dependency>

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;

import org.apache.commons.io.IOUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class ImageDownloader {

    private static final String USER_AGENT = "Mozilla/5.0 Firefox/26.0";

    private static Logger logger = LoggerFactory.getLogger(ImageDownloader.class);

    private static final int TIMEOUT_SECONDS = 120;

    private static final int POOL_SIZE = 120;

    private static CloseableHttpClient httpclient;

    public static void main(String[] args) throws ClientProtocolException, IOException {

        ImageDownloader imageDownloader = new ImageDownloader();
        imageDownloader.initApacheHttpClient();

        String imageUrl = "https://ibclaty.j.com/niku/ui/uitk/images/s.gif";
        String filePath = "D:\\test.jpg";

        imageDownloader.fetchContent(imageUrl, filePath);

        imageDownloader.destroyApacheHttpClient();
    }

    public void initApacheHttpClient() {
        // Create global request configuration
        RequestConfig defaultRequestConfig = RequestConfig.custom().setSocketTimeout(TIMEOUT_SECONDS * 1000)
                .setConnectTimeout(TIMEOUT_SECONDS * 1000).build();

        // Create an HttpClient with the given custom dependencies and
        // configuration.
        httpclient = HttpClients.custom().setUserAgent(USER_AGENT).setMaxConnTotal(POOL_SIZE)
                .setMaxConnPerRoute(POOL_SIZE).setDefaultRequestConfig(defaultRequestConfig).build();
    }

    private void destroyApacheHttpClient() {
        try {
            httpclient.close();
        } catch (IOException e) {
            logger.error("httpclient close fail", e);
        }
    }

    public void fetchContent(String imageUrl, String filePath) throws ClientProtocolException, IOException {

        HttpGet httpget = new HttpGet(imageUrl);
        httpget.setHeader("Referer", "http://www.google.com");

        System.out.println("executing request " + httpget.getURI());
        CloseableHttpResponse response = httpclient.execute(httpget);

        try {
            HttpEntity entity = response.getEntity();

            if (response.getStatusLine().getStatusCode() >= 400) {
                throw new IOException("Got bad response, error code = " + response.getStatusLine().getStatusCode()
                        + " imageUrl: " + imageUrl);
            }
            if (entity != null) {
                InputStream input = entity.getContent();
                OutputStream output = new FileOutputStream(new File(filePath));
                IOUtils.copy(input, output);
                output.flush();
            }
        } finally {
            response.close();

        }

    }
}