并发编程之实现图片下载

并发编程之实现图片下载

本文主要利用java中java.utils.concurrent包中的Future和Callable类来实现图片的下载。本文主要分为三步：

1. 实现ImgUrl类，其主要作用是获取html文件中图片地址，利用Jsoup技术来实现html的解析，需要下载Jsoup的jar包，maven项目需要dependency下；
2. 实现DownLoadImgUrl类，该类实现Callable类，根据图片的url把图片下载到本地；
3. 实现测试类，测试代码的有效性，需要下载Junit的jar包，如果是maven项目需要dependency下。

首先，实现ImgUrl类。

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Created by lionel on 16/8/11.
 */
public class ImgUrl {
    /**
     * 获取网页下所有图片的地址列表
     * @param url 网页地址
     * @return
     */
    public static List<String>  getImgUrl(String url){
        Document document = null;
        try {
            document = Jsoup.connect(url).get();
        } catch (IOException e) {
            e.printStackTrace();
        }

        String html = document.html();
        String img = "";
        String regEx_img = "<img.*src=(.*?)[^>]*?>";
        Pattern pattern = Pattern.compile(regEx_img,Pattern.CASE_INSENSITIVE);
        Matcher matcher = pattern.matcher(html);
        List<String> pics = new ArrayList<String>();

        while (matcher.find()) {
            img = img + "," + matcher.group();
            Matcher m = Pattern.compile("src=\"?(.*?)(\"|>|\\s+)").matcher(img);
            while (m.find()) {
                pics.add(m.group(1));
            }
        }

        return pics;
    }
}

其次，实现DownLoadImgUrl类

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.concurrent.Callable;

/**
 * Created by lionel on 16/8/12.
 */
public class DownPicTask implements Callable<Boolean> {
    String url;
    String fileName;

    public DownPicTask(String url, String fileName) {
        this.url = url;
        this.fileName = fileName;
    }

    @Override
    public Boolean call() throws Exception {
        return getImgFile(url, fileName);
    }

    /**
     * 下载文件到本地
     * @param url 图片地址
     * @param fileName 下载到本地的文件地址
     * @return
     */
    public static Boolean getImgFile(String url, String fileName) {
        OutputStream os = null;
        InputStream is = null;
        try {
            URL url1 = new URL(url);
            URLConnection urlConnection = url1.openConnection();
            is = urlConnection.getInputStream();
            byte[] bs = new byte[1024];
            int len;
            os = new FileOutputStream(fileName);

            while ((len = is.read(bs)) != -1) {
                os.write(bs, 0, len);
            }
            return Boolean.TRUE;
        } catch (MalformedURLException me) {
            me.printStackTrace();
        } catch (Exception e) {
            e.printStackTrace();

        } finally {
            if (os != null) {
                try {
                    os.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (is != null) {
                try {
                    is.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        return Boolean.FALSE;
    }
}

最后，写一个测试类

import org.junit.Test;

import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

/**
 * Created by lionel on 16/8/11.
 */
public class DownloadPictureTest {
    @Test
    public void test(){
        ExecutorService executorService = Executors.newFixedThreadPool(5);
        String pagehtml = "http://mp.weixin.qq.com/s?__biz=MzA4MTk3NTgyMQ==&mid=401795482&idx=1&sn=016237aa3979221741235214c1764c4c&scene=21#wechat_redirect";
        List<String> imgUrlList = ImgUrl.getImgUrl(pagehtml);
        int i=0;
        for (String imgUrl:imgUrlList){
            String filename="data/picture/"+i+".jpg";
            DownPicTask downPicTask=new DownPicTask(imgUrl,filename);
            executorService.submit(downPicTask1);
            i++;
        }
        executorService.shutdown();
        try {
            executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        System.out.println("all thread complete");
    }
}