java 爬虫：开源java爬虫 swing工具 Imgraber

1实现点：

• 1.返回给定URL网页内，所有图像url list
• 2.返回给定URL网页内，自动生成图像文件路径.txt 文件
• 3.返回给定URL网页内，下载txt文件指定的图片url,并将所有图像保存在 ./img文件夹下
• 4.实现简易swing 界面，有空再改造

2基于开源jsoup实现，鸣谢！

效果

github

imgraber

package himi.crawler;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.jsoup.Jsoup;
import org.jsoup.helper.StringUtil;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

/**
 * 
 * @ClassName: ImgTool
 * @Description:IMG tool ，请先赋值URL 获取指定ULR的某个网页里面 
 *                  1.返回所有图像url list 
 *                  2.图像文件夹路径.txt
 *                  3.所有图像保存在img文件夹里面
 * @author penny
 * @date 2018年3月7日 下午10:07:49
 *
 */
public class ImgTool {
    // 参数域
    /** 指定URL */
    public static String URL = "https://www.oschina.net";
    private int imgNumbs = 0;
    public static List<String> downloadMsg=new ArrayList<String>();
    public String imgUrlTxt = "imgURLs.txt";
    public static String regex= "^((https|http|ftp|rtsp|mms)?://)" 
                                + "?(([0-9a-z_!~*'().&=+$%-]+: )?[0-9a-z_!~*'().&=+$%-]+@)?" //ftp的user@ 
                                + "(([0-9].)[0-9]" // IP形式的URL- 199.194.52.184 
                                + "|" // 允许IP和DOMAIN（域名）
                                + "([0-9a-z_!~*'()-]+.)*" // 域名-  
                                + "([0-9a-z][0-9a-z-])?[0-9a-z]." // 二级域名 
                                + "[a-z])" // first level domain- .com or .museum 
                                + "(:[0-9])?" // 端口- :80 
                                + "((/?)|" // a slash isn't required if there is no file name 
                                + "(/[0-9a-z_!~*'().;?:@&=+$,%#-]+)+/?)$";
    private ImgTool() {
    };
    
    private static ImgTool instance = new ImgTool();

    /** 获取ImgTool 单例 */
    public static ImgTool getInstance() {
        return instance;
    }

    public List<String> getURLs() {
        return getURLs(null);
    }
    public boolean isURL(String str) {
        if(StringUtil.isBlank(str)){
            return false;
        }else{
//      String regex = "^(?:https?://)?[\\w]{1,}(?:\\.?[\\w]{1,})+[\\w-_/?&=#%:]*$";
//      String regex = "^([hH][tT]{2}[pP]:/*|[hH][tT]{2}[pP][sS]:/*|[fF][tT][pP]:/*)(([A-Za-z0-9-~]+).)+([A-Za-z0-9-~\\/])+(\\?{0,1}(([A-Za-z0-9-~]+\\={0,1})([A-Za-z0-9-~]*)\\&{0,1})*)$";
        Pattern pattern = Pattern.compile(regex);
        Matcher matcher = pattern.matcher(str);
        if(matcher.matches()){
            return true;
        }else{
            return false;
        }
    }};
    /***
     * @Title: getURLs
     * @Description: 给定cssQuery对象
     * @param @param cssQuery HTML中的CSS(或者 JQuery)选择器语法，更多详细用法见Jsoup介绍 < a
     *        href="https://jsoup.org/apidocs/org/jsoup/select/Selector.html"
     *        ></a>
     * @param @return List
     * @throws
     * 
     */
    public List<String> getURLs(String cssQuery) {
        List<String> urls = null;
        Document doc;
        Elements imgElements ;
        if (!isURL(URL)) {
            return null;
        }
        if(StringUtil.isBlank(cssQuery)){
            cssQuery="img";
        }
        try {
            doc = Jsoup.connect(URL).get();
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
        if(doc==null)return null;
        imgElements = doc.select(cssQuery);
        urls = new ArrayList<String>();
        for (Object eleObj : imgElements) {
            //"(https?|ftp|http)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]"
            Pattern pattern = Pattern.compile("(https?|http)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]");
            Matcher matcher = pattern.matcher(eleObj.toString());
            if (matcher.find()) {
                String url = matcher.group();
                urls.add(url);
            }
        }
        imgNumbs = imgElements.size();
        return urls;
    }


    /**
     * 
     * @Title: createImgURLTxt
     * @Description:
     * @param @param cssQuery：默认使用img HTML中的CSS(或者 JQuery)选择器语法，更多详细用法见Jsoup介绍 <
     *        a href="https://jsoup.org/apidocs/org/jsoup/select/Selector.html">
     *        </a>
     * @throws 生成imgURLs.txt
     */
    public String createImgURLTxt(String cssQuery) {
        long start = System.currentTimeMillis();
        List<String> urls;
        urls = getURLs(cssQuery);
        BufferedWriter os = null;
        File urlsFiles = new File("imgURLs.txt");
        if(urlsFiles.exists()){
            try {
                urlsFiles.createNewFile();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        try {
            os = new BufferedWriter(new FileWriter(urlsFiles));
            if(urls==null)return null;
            for (int i = 0; i < urls.size(); i++) {
                os.write(urls.get(i) + "\n");
            }
            String result = "执行完毕,生成imgURLs.txt,耗时"
                    + (System.currentTimeMillis() - start) / 1000 + "s";
            System.out.println(result);
            return result;
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            try {
                if (os != null)
                    os.close();

            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return null;
    }

    /**
     * 
     * @Title: createImgs 
     * @Description: 
     * @param @param cssQuery
     * @param @throws IOException    
     * @throws
     */
    public void createImgs(String cssQuery) throws IOException {
        long startTime = System.currentTimeMillis();
        downloadMsg.add("Your images is downloading ");
        BufferedReader br = null;
        OutputStream out = null;
        InputStream in = null;
        ArrayList<String> imgList = null;
        HttpURLConnection con = null;
        
        String url;             //待下载文件url
        int fileSize = 0;       //单个文件大小
        int totalFileNum=0;     //总文件数
        int downLoadFileNum=0;  //已下载文件
        long totalTime=0;       //总耗时/s
        long singleTime=0;      //单个文件耗时/ms
        
        br = new BufferedReader(new FileReader(imgUrlTxt));
        imgList = new ArrayList<String>();
        while ((url = br.readLine()) != null) {
            imgList.add(url);
        }
        downLoadFileNum=totalFileNum= imgList.size();
        downloadMsg.add("总文件数"+(totalFileNum));
        for (String listUrl : imgList) {
            startTime = System.currentTimeMillis();
            String fileName = listUrl.substring(listUrl.lastIndexOf('/') + 1);// 截取文件名
            URL imgUrl = new URL(listUrl.trim());
            if (con != null)
                con.disconnect();
            con = (HttpURLConnection) imgUrl.openConnection();
            con.setRequestMethod("GET");
            con.setDoInput(true);
            con.setConnectTimeout(1000 * 30);
            con.setReadTimeout(1000 * 30);
            fileSize = con.getContentLength();
            con.connect();
            try {
                in = con.getInputStream();
                File file = new File("img" + File.separator, fileName);
                if (!file.exists()) {
                    file.getParentFile().mkdirs();
                    file.createNewFile();
                }
                out = new BufferedOutputStream(new FileOutputStream(file));
                int len = 0;
                byte[] buff = new byte[1024 * 1024];
                while ((len = new BufferedInputStream(in).read(buff)) != -1) {
                    out.write(buff, 0, len);
                }
                out.flush();
                downLoadFileNum--;
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                if (br != null)
                    try {
                        br.close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                if (in != null)
                    try {
                        in.close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                if (out != null)
                    try {
                        out.close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                singleTime=System.currentTimeMillis() - startTime;
                totalTime+=singleTime;
                downloadMsg.add("文件名" + fileName + "  文件大小" + fileSize
                                  +"  未下载文件数"+(downLoadFileNum)
                                  +"  下载耗时"+ singleTime + "ms" );
                System.out.println(downloadMsg.get(downloadMsg.size()-1));
            }
        }
        downloadMsg.add("总耗时"+totalTime/1000+"s");
    }

    /**
     * @throws IOException
     * 
     * @Title: main
     * @Description: test
     * @param @param args
     * @throws
     */
    public static void main(String[] args) throws IOException {
        ImgTool img = ImgTool.getInstance();
         img.createImgURLTxt("img");
         List<String> urls = img.getURLs("img");
         for (Object str : urls) {
            System.out.println(str.toString());
        }
        img.createImgs(null);
    }
}

转载于:https://www.cnblogs.com/humi/p/8555000.html