百度百科页面解析抓取（java版）

最新推荐文章于 2023-02-23 10:33:41 发布

Apoca_lypse

最新推荐文章于 2023-02-23 10:33:41 发布

阅读量808

点赞数

CC 4.0 BY-SA版权

文章标签： java 爬虫

本文链接：https://blog.youkuaiyun.com/Apoca_lypse/article/details/116265695

前言：最近做个需求，就是把部分植物的百科内容抓取清洗，并且保存展示。
对比了几大百科网站，还是百度百科的比较好解析，写完了记录分享一下。

应用到的第三方包：

		<!-- hutool工具包 -->
		<dependency>
            <groupId>cn.hutool</groupId>
            <artifactId>hutool-all</artifactId>
            <version>5.5.2</version>
        </dependency>
        <!-- jsoup工具包 -->
		<dependency>
			<groupId>org.jsoup</groupId>
			<artifactId>jsoup</artifactId>
			<version>1.11.3</version>
		</dependency>

代码如下：

public class DataAcquisitionService {

    private final static String BAIDU_BAIKE = "https://baike.baidu.com/item/{}";


    /**
     * 根据词条检索百度百科，然后返回结构化的百科html结构
     *
     * @param entry 词条
     * @return {@link String}
     */
    public static List<String> baiduBaikeContent(String entry) {
        String html = HttpRequest
                .get(StrUtil.format(BAIDU_BAIKE, entry))
                // 这个UserAgentConstants.userAgents是一个List<String>，保存着useragent的列表
                .header("user-agent", RandomUtil.randomEle(UserAgentConstants.userAgents))
                .timeout(3000)
                .execute()
                .body();
        if (html.equals("")) {
            throw new RuntimeException("抓到的页面为空");
        }
        Document page = Jsoup.parse(html);
        // 词条不存在
        if (page.head().selectFirst("title").text().contains("302")) {
            throw new RuntimeException("ip已被拦截，无法继续爬了");
        }
        if (page.selectFirst(".searchResult") != null) {
            return null;
        }
        List<BaiduBkCont> contentList = new ArrayList<>();
        // 抓取摘要
        contentList.add(new BaiduBkCont(true, 2, "摘要"));
        contentList.add(new BaiduBkCont(false, null,
                getElStr(page.selectFirst(".lemma-summary"))));
        Element cur = page.selectFirst(".para-title.level-2");
        while (inMainBody(cur)) {
            // 4级标题
            if (cur.tagName().equals("ul")) {
                contentList.add(new BaiduBkCont(true, 4, cur.text()));
            }// 分割线不做任何处理
            else if (!cur.hasAttr("label-module")) {

            }
            // 2级标题和三级标题
            else if (cur.attr("label-module").equals("para-title")) {
                String level = ReUtil.get("(?<=level-)\\d", cur.attr("class"), 0);
                contentList.add(new BaiduBkCont(true, Convert.toInt(level), getElStr(cur)));
            }// 正文
            else if (cur.attr("label-module").equals("para")) {
                contentList.add(new BaiduBkCont(false, null, getElStr(cur)));
            }
            cur = cur.nextElementSibling();
        }
        // 图片
        Element img = page.selectFirst(".summary-pic img");
        String url = img == null ? null : img.attr("src");

        StringBuffer result = new StringBuffer();
        for (BaiduBkCont cont : contentList) {
            result.append(cont.toHtml());
        }
        return ListUtil.toList(result.toString(), url);
    }

    /**
     * 清洗element，清理掉里面的describes，title-prefix，和sup，清除掉多余的换行符
     * 并返回element里的所有字符串
     *
     * @param el
     * @return
     */
    private static String getElStr(Element el) {
        if (el == null) {
            System.out.println(el);
        }
        Elements prefixes = el.select(".title-prefix");
        prefixes.addAll(el.select(".edit-icon"));
        prefixes.addAll(el.select(".description"));
        prefixes.addAll(el.select("sup"));
        for (Element element : prefixes) {
            element.remove();
        }
        return el.text().trim().replaceAll("\\n(?=。)", "");
    }

    /**
     * 判断Element是不是为正文部分
     *
     * @param el
     * @return
     */
    private static boolean inMainBody(Element el) {
        return el != null
                && (el.hasClass("para-title")  // 2级标题和3级标题
                || el.hasClass("para")  // 正文内容
                || el.hasClass("para-list")  // 4级标题
                || el.hasClass("anchor-list"));  // 2级分割线
    }

    public static void main(String[] args) {
        List<String> results = baiduBaikeContent("水稻显纹纵卷叶螟");
        System.out.println(results.get(0));
        System.out.println(results.get(1));
    }

    /**
     * 抓取文本实体
     */
    @Data
    @AllArgsConstructor
    private static class BaiduBkCont {
        // 是否是标题
        private Boolean isTitle;
        // 标题级别
        private Integer titleLevel;
        // 内容
        private String inner;

        /**
         * 转成html展示
         *
         * @return
         */
        private String toHtml() {
            if (!isTitle) {
                return StrUtil.format("<p class=\"content-line\">{}</p>", inner);
            }
            return StrUtil.format("<div class=\"h{}\">{}</div>", titleLevel, inner);
        }
    }
}

附：UserAgentConstants.userAgents

public interface UserAgentConstants {
	// userAgent 列表
    List<String> userAgents = ListUtil.of(
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
            "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
            "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0",
            "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.30729; .NET CLR 3.5.30729; InfoPath.3; rv:11.0) like Gecko",
            "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)",
            "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)",
            "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)",
            "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
            "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
            "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11",
            "Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
            "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)",
            "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)",
            "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
            "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)",
            "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)",
            "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)",
            "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser)",
            "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
            "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
            "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
            "Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
            "Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
            "MQQBrowser/26 Mozilla/5.0 (Linux; U; Android 2.3.7; zh-cn; MB200 Build/GRJ22; CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
            "Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945; U; en-GB) Presto/2.8.149 Version/11.10",
            "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13",
            "Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; en) AppleWebKit/534.1+ (KHTML, like Gecko) Version/6.0.0.337 Mobile Safari/534.1+",
            "Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.0; U; en-US) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/233.70 Safari/534.6 TouchPad/1.0",
            "Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18124",
            "Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; HTC; Titan)",
            "UCWEB7.0.2.37/28/999",
            "NOKIA5700/ UCWEB7.0.2.37/28/999",
            "Openwave/ UCWEB7.0.2.37/28/999",
            "Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/999",
            "Mozilla/6.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/8.0 Mobile/10A5376e Safari/8536.25"
    );
}