Java实现历史上的今天

关注

原创最新推荐文章于 2026-03-25 22:16:52 发布 715 阅读 AI 写同款· GEO 优化 ›

网页爬虫--历史上的今天网页爬虫–历史上的今天上篇博客中我们知道了，调用第三方的API获得我们想要的数据，如果我们要的数据没有现成的API怎么办？那就需要我们去网站上爬数据了。下面就教大家怎么去网页上爬我们需要的数据。一.使用Java JDK中的工具自己实现HTTP和HTTPS请求写一个HttpUtils 工具类：package org.sunming.util;import java.io.BufferedReader; 阅读详情

一、controller层

package com.hd.history.controller;
import com.hd.history.dto.HistoryDto;
import com.hd.history.entity.History;
import com.hd.history.service.HistoryService;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

@RestController
@RequestMapping("/findHistoryday")
public class HistoryController {
    @Autowired
    private HistoryService historyService;
      @Scheduled(cron = "0 0 7 1 * ? ")//定时每个月1号7点执行一次
    public void getHistoryMonthDay() throws IOException {
        Integer[] dayNumber = {31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
        //循环每一天
        for (int i = 1; i <= 12; i++) {
            for (int j = 1; j <= dayNumber[i - 1]; j++) {
                getgetHistoday(i, j);
            }
        }
    }
    public void getgetHistoday(Integer month, Integer day) throws IOException {
    //爬虫目标源网页的url路径要求不够10的0补齐
        String monthDay = "";
        String monthNumber = String.valueOf(month);
        String dayNumber = String.valueOf(day);
        if (month < 10) {
            monthNumber = "0" + month;
        }
        if (day < 10) {
            dayNumber = "0" + day;
        }
        monthDay = monthNumber + dayNumber;
        Document document = Jsoup.connect("http://hao.360.com/histoday/" + monthDay + ".html").get();
        Element body = document.body();
        Elements lie12 = body.getElementsByClass("tih-item");
        List<History> historyList = new ArrayList<>();
        //这方面我们主要需要三个数据 标题图片和具体内容
        for (Element element : lie12) {
            //导入标题
            Elements titleData = element.select("dt");
            String title = titleData.text();
            //导入图片
            Elements imageData = element.getElementsByClass("item-img");
            String image = imageData.attr("src");
            String imageNumber = imageData.attr("data-src");
            String imageUrl = "";
            if (null!=imageNumber&&imageNumber!=""){
                imageUrl = "http:" + imageNumber;
            }else if (null != image && image != "") {
                imageUrl = "http:" + image;
            }

            //具体内容
            Elements contentData = element.getElementsByClass("desc");
            String content = contentData.text();
            History history = new History();
            history.setTitle(title);
            history.setImg(imageUrl);
            history.setContent(content);
            historyList.add(history);
        }
        String finalMonthDay = monthDay;
        historyList.forEach(history -> history.setHistoryDate(finalMonthDay));
        historyService.insertHistory(historyList);
    }
    //查找接口
    @PostMapping("/getHistoryDay")
    public List<History> getHistoryList(@RequestBody HistoryDto historyDto) {
        return historyService.getHistoday(historyDto);
    }
}

二、service层

package com.hd.history.service.impl;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.hd.history.dto.HistoryDto;
import com.hd.history.entity.History;
import com.hd.history.mapper.HistoryMapper;
import com.hd.history.service.HistoryService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.util.List;
@Service
public class HistoryServiceImpl implements HistoryService {
    @Autowired
    private HistoryMapper historyMapper;

    @Override
    public List<History> getHistoday(HistoryDto historyDto) {
        QueryWrapper<History> queryWrapper = new QueryWrapper<>();
        queryWrapper.lambda().eq(History::getHistoryDate, historyDto.getHistoryDate());
        List<History> histories = historyMapper.selectList(queryWrapper);
        return histories;
    }
//采取的是先删除在插入，因为不好考虑唯一索引，而且整体删除也用不了多少时间
    @Override
    public void insertHistory(List<History> historyList) {
        findHistory(historyList.get(0).getHistoryDate());
        for (History history : historyList) {
            historyMapper.insert(history);
        }
    }

    public void findHistory(String historyDate) {
        QueryWrapper<History> queryWrapper = new QueryWrapper<>();
        queryWrapper.lambda().eq(History::getHistoryDate, historyDate);
        List<History> histories = historyMapper.selectList(queryWrapper);
        if (histories.size() > 0) {
            historyMapper.delete(queryWrapper);
        }
    }
}

三、entity层

package com.hd.history.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.io.Serializable;
@Data
@TableName("hd_history")
public class History implements Serializable {
    private static final long serialVersionUID = 1L;
    @TableId(value = "id", type = IdType.AUTO)
    private Long id;
    private String historyDate;
    private String title;
    private String img;
    private String content;
}

四、pom文件

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.hd.history</groupId>
    <artifactId>maven-history</artifactId>
    <version>1.0-SNAPSHOT</version>
    <name>maven_history</name>
    <description>Demo project for Spring Boot</description>
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.2.1.RELEASE</version>
    </parent>

    <properties>
        <java.version>1.8</java.version>
        <cloud.version>Hoxton.RELEASE</cloud.version>
        <cloud.alibaba.version>0.2.2.RELEASE</cloud.alibaba.version>
        <gmall.version>1.0</gmall.version>
        <mybatis-plus.version>3.0.5</mybatis-plus.version>
        <mysql.version>5.1.46</mysql.version>
        <swagger.version>2.7.0</swagger.version>
        <fastdfs.version>1.27.0.0</fastdfs.version>
        <lombok.version>1.18.10</lombok.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.springframework.cloud</groupId>
            <artifactId>spring-cloud-dependencies</artifactId>
            <version>Hoxton.RELEASE</version>
            <type>pom</type>
            <scope>import</scope>
        </dependency>

        <dependency>
            <groupId>org.springframework.cloud</groupId>
            <artifactId>spring-cloud-alibaba-dependencies</artifactId>
            <version>0.2.2.RELEASE</version>
            <type>pom</type>
            <scope>import</scope>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>
        <!--mybatis-plus 持久层-->
        <dependency>
            <groupId>com.baomidou</groupId>
            <artifactId>mybatis-plus-boot-starter</artifactId>
            <version>3.0.5</version>
        </dependency>

        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.46</version>
        </dependency>


        <!-- jsoup HTML parser library @ http://jsoup.org/ -->
        <dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.11.2</version>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-tomcat</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
            </plugin>
        </plugins>
    </build>


</project>

五、application.properties

# 服务端口
server.port=8088
# 服务名
spring.application.name=maven-history

# 环境设置：dev、test、prod
spring.profiles.active=dev

# mysql数据库连接
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
spring.datasource.url=jdbc:mysql://localhost:3306/media?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai&rewriteBatchedStatements=true&useServerPrepStmts=true&allowMultiQueries=true

spring.datasource.username=root
spring.datasource.password=root
#mybatis日志
mybatis-plus.configuration.log-impl=org.apache.ibatis.logging.stdout.StdOutImpl

六、表结构

DROP TABLE IF EXISTS `hd_history`;
CREATE TABLE `hd_history`  (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `history_date` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '' COMMENT '当前日期',
  `title` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '' COMMENT '标题',
  `img` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '图像',
  `content` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '内容',
  PRIMARY KEY (`id`) USING BTREE
)

七、爬取的网页HTML

<dl class="tih-item open"> 
  <dt>
   <em>1</em>. 1981年-我国正式实行学位制度
  </dt> 
  <dd> 
   <img class="item-img" src="//p7.qhmsg.com/t01fd70f46cc423a8d7.jpg"> 
   <div class="clearfix"> 
    <div class="desc">
      1981年1月1日，我国正式实行学位制度。我国从50年代开始，就试图建立学位制，因受极“左”思潮的影响和“文革”的干扰，一直未能实行。1976年，教育部和国务院科技干部局联合组织的学位小组，拟订了《中华人民共和国学位条例（草案）》，经审议修改，1980年2月15日由人大常委会第十三次会议讨论通过。学位制规定，我国设学士、硕士、博士3级学位。这些当时还看不出分量的名称后来却极大地规范了人才选拔制度。 
    </div> 
    <div class="right-btn-container"> 
     <a href="http://baike.so.com/doc/6222143.html" target="_blank" class="read-btn">阅读全文</a> 
    </div> 
   </div> 
  </dd> 
 </dl> 
 <dl class="tih-item"> 
  <dt>
   <em>2</em>. 1814年-太平天国农民起义领袖洪秀全诞辰
  </dt> 
  <dd> 
   <img class="item-img" data-src="//p8.qhmsg.com/t014369be2b3746e6d2.jpg" src="//hao3.qhimg.com/t0194aea54158184820.gif"> 
   <div class="clearfix"> 
    <div class="desc">
      洪秀全（1814年1月1日－1864年6月1日），原名洪仁坤，小名火秀，客家人，是太平天国以宗教名义发动民变的领袖。清嘉庆十八年十二月初十生于广东花县（今广州市花都区）福源水村，后来移居到官禄布村。道光年间屡应科举不中，遂吸取早期基督教义中的平等思想，创立拜上帝会，撰《原道救世歌》以布教，主张建立远古“天下为公”盛世。道光三十年十二月初十（1851年1月11日）洪秀全发动金田起义，建国号太平天国，自称天王。1853年以南京作为首都，改名天京，1864年在天京病逝，太平天国在他死后不久灭亡。 
    </div> 
    <div class="right-btn-container"> 
     <a href="http://baike.so.com/doc/4371050.html" target="_blank" class="read-btn">阅读全文</a> 
    </div> 
   </div> 
  </dd> 
 </dl>