SpringBoot +Jpa+webmagic入门学习案例

在b站学习的webmagic代码

整体结构图

#=

maven依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.0.3.RELEASE</version>

    </parent>
    <groupId>cn.itcast</groupId>
    <artifactId>itcast-crawler-webmagic</artifactId>
    <version>1.0-SNAPSHOT</version>
    <modelVersion>4.0.0</modelVersion>

    <properties>
            <java.version>1.8</java.version>
    </properties>
    <dependencies>
        <!--SpringMVC-->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
            <version>2.0.3.RELEASE</version>
        </dependency>
        <!--SpringData Jpa-->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-jpa</artifactId>
        </dependency>
        <!--MySQL连接包-->
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
        </dependency>
        <!--Webmagic 核心包-->
        <dependency>
            <groupId>us.codecraft</groupId>
            <artifactId>webmagic-core</artifactId>
            <version>0.7.3</version>
            <!--排除日志-->
            <exclusions>
                <exclusion>
                    <groupId>org.slf4j</groupId>
                    <artifactId>slf4j-log4j12</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <!--Webmagic 扩展包-->
        <dependency>
            <groupId>us.codecraft</groupId>
            <artifactId>webmagic-extension</artifactId>
            <version>0.7.3</version>
        </dependency>
        <!--布隆过滤器的包-->
        <dependency>
            <groupId>com.google.guava</groupId>
            <artifactId>guava</artifactId>
            <version>16.0</version>
        </dependency>
        <!--工具包-->
        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-lang3</artifactId>
        </dependency>
    </dependencies>


</project>

配置

#DB Configuration：
spring.datasource.driverClassName=com.mysql.jdbc.Driver
spring.datasource.url=jdbc:mysql://127.0.0.1:3306/crawler
spring.datasource.username=root
spring.datasource.password=woshiliujie1

#JPA configuration：
# spring.jpa.show-sql 配置在日志中打印出执行的 SQL 语句信息。
spring.jpa.database=mysql
spring.jpa.show-sql=true  
#springboot jpa 会将驼峰式 自动修改为增加下划线
spring.jpa.hibernate.naming.physical-strategy=org.hibernate.boot.model.naming.PhysicalNamingStrategyStandardImpl

pojo层

package cn.itcast.job.pojo;

import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
@Entity
//@Entity(name="EntityName") 必须，用来标注一个数据库对应的实体，数据库中创建的表名默认和类名一致。
// 其中，name 为可选，对应数据库中一个表，使用此注解标记 Pojo 是一个 JPA 实体。
    public class JobInfo {

// @Table(name = "AUTH_USER") 是一个可选的注解。
// 声明了数据库实体对应的表信息。包括表名称、索引信息等。这里声明这个实体类对应的表名是 AUTH_USER。
// 如果没有指定，则表名和实体的名称保持一致。
    @Id
    @GeneratedValue(strategy = GenerationType.IDENTITY)
    private  Long id;
    private String companyName;
    private  String companyAddr;
    private  String companyInfo;
    private  String jobName;
    private String jobAddr;
    private  String jobInfo;
    private String url;

    public Long getId() {
        return id;
    }

    public void setId(Long id) {
        this.id = id;
    }

    public String getCompanyName() {
        return companyName;
    }

    public void setCompanyName(String companyName) {
        this.companyName = companyName;
    }

    public String getCompanyAddr() {
        return companyAddr;
    }

    public void setCompanyAddr(String companyAddr) {
        this.companyAddr = companyAddr;
    }

    public String getCompanyInfo() {
        return companyInfo;
    }

    public void setCompanyInfo(String companyInfo) {
        this.companyInfo = companyInfo;
    }

    public String getJobName() {
        return jobName;
    }

    public void setJobName(String jobName) {
        this.jobName = jobName;
    }

    public String getJobAddr() {
        return jobAddr;
    }

    public void setJobAddr(String jobAddr) {
        this.jobAddr = jobAddr;
    }


    public String getJobInfo() {
        return jobInfo;
    }

    public void setJobInfo(String jobInfo) {
        this.jobInfo = jobInfo;
    }



    public String getUrl() {
        return url;
    }

    public void setUrl(String url) {
        this.url = url;
    }

    @Override
    public String toString() {
        return "JobInfo{" +
                "id=" + id +
                ", companyName='" + companyName + '\'' +
                ", companyAddr='" + companyAddr + '\'' +
                ", companyInfo='" + companyInfo + '\'' +
                ", jobName='" + jobName + '\'' +
                ", jobAddr='" + jobAddr + '\'' +
                ", jobInfo='" + jobInfo + '\'' +
                ", url='" + url + '\'' +
                '}';
    }
}

dao层

package cn.itcast.job.dao;

import cn.itcast.job.pojo.JobInfo;
import org.springframework.data.jpa.repository.JpaRepository;
//其中 T 是数据库实体类(JobInfo)，ID 是数据库实体类的主键
public interface JobInfoDao  extends JpaRepository<JobInfo,Long> {
}

service层

package cn.itcast.job.service;

import cn.itcast.job.pojo.JobInfo;

import java.util.List;

public interface JobInfoService {
    /**
     * 保存工作信息
     */
    public void save(JobInfo jobInfo);

    /**
     *根据条件查询工作信息
     */
    public List<JobInfo> fineJobInfo(JobInfo jobInfo);
}

`
###

JobInfoSErviceImpl

package cn.itcast.job.service.impl;

import cn.itcast.job.dao.JobInfoDao;
import cn.itcast.job.pojo.JobInfo;
import cn.itcast.job.service.JobInfoService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Example;
import org.springframework.stereotype.Service;

import javax.transaction.Transactional;
import java.util.List;

@Service
public class JobInfoServiceImpl  implements JobInfoService {

//    注入dao

    @Autowired
    private JobInfoDao jobInfoDao;


    @Override
    @Transactional
    public void save(JobInfo jobInfo) {
        //根据url和发布时间查询数据
        JobInfo param =new JobInfo();
        param.setUrl(jobInfo.getUrl());
        //查询
       List<JobInfo> list = this.fineJobInfo(param);

       //判断查询结果数据是否为空
        if(list.size() == 0){
            //如果查询结果为空 表示招聘信息不存在 或者已经更新 需要新增或者更新数据
            this.jobInfoDao.saveAndFlush(jobInfo);
        }

    }

    @Override
    public List<JobInfo> fineJobInfo(JobInfo jobInfo) {
        //设置查询条件
        Example example = Example.of(jobInfo);
        //执行查询
        List list=this.jobInfoDao.findAll(example);

        return list;
    }
}

task

package cn.itcast.job.task;

import cn.itcast.job.pojo.JobInfo;
import org.jsoup.Jsoup;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.util.List;

@Component
public class JobProcessor implements PageProcessor {

    private  String url="https://search.51job.com/list/000000,000000,0000,00,9,99,java,2,1.html?lang=c&stype=1&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=01&companysize=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=22&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=%E5%81%9A%E4%B8%80%E4%BC%91%E4%B8%80";
    @Override
    public void process(Page page) {
            // 解析页面获取招聘信息详情的url地址
        List<Selectable> list = page.getHtml().css("div#resultList div.el").nodes();
        // 判断获取到的集合是否为空
        if (list.size() == 0){
            //如果为空表示招聘页 解析页面 获取招聘详情信息保存数据
            this.saveJobInfo(page);
        }else {
            //不为空 解析url放到任务中
            for (Selectable selectable:list){
                //url
              String jobInfourl = selectable.links().toString();
              //把获取到的url放到任务列表中
                page.addTargetRequest(jobInfourl);
            }
            //获取下一页的url
            String bkUrl = page.getHtml().css("div.p_in li.bk").nodes().get(1).links().toString();
           //将下一页的url放到任务队列中
            page.addTargetRequest(bkUrl);
        }
        String html = page.getHtml().toString();


    }
    //解析获取招聘详情信息保存数据
    private void saveJobInfo(Page page) {
        //创建招聘详情对象
        JobInfo jobInfo = new JobInfo();

        //解析页面
        Html html = page.getHtml();

        //获取数据，封装到对象中
        jobInfo.setCompanyName(html.css("div.cn p.cname a","text").toString());
        jobInfo.setCompanyAddr(Jsoup.parse(html.css("div.bmsg").nodes().get(1).toString()).text() );
        jobInfo.setCompanyInfo(Jsoup.parse(html.css("div.tmsg").toString()).text());//因为就一个 所以不用写是第几个
        jobInfo.setJobName(html.css("div.cn h1","text").toString());
        jobInfo.setJobAddr(Jsoup.parse(html.css("div.bmsg").nodes().get(1).toString()).text());
        jobInfo.setJobInfo(Jsoup.parse(html.css("div.job_msg").toString()).text());
        jobInfo.setUrl(page.getUrl().toString());
        //获取发布时间 需要用到正则
//          String time= Jsoup.parse(html.css("div.cn p.ltype").toString()).text();
//          jobInfo.setTime(time.substring(0,time.length()-2));
          //结果保存
        page.putField("jobInfo",jobInfo);
//        jobInfo.setId();
//        jobInfo.setTime();



    }

    private Site site = Site.me()
            .setCharset("gbk")
            .setTimeOut(10000)//设置超时时间 单位是ms毫秒
            .setRetrySleepTime(30000) //设置重试的间隔时间 1000为默认值
            .setRetrySleepTime(3)
            ;
    @Override
    public Site getSite() {
        return site;
    }

    @Autowired
    private  SpringDataPipeline springDataPipeline;


    //initialDelay  当任务启动时候 等多久执行方法
    //fixedDelay    m每隔10秒 执行方法
    @Scheduled(initialDelay = 1000,fixedDelay = 100*1000)
    public  void process(){
        Spider.create(new JobProcessor())
                .addUrl(url)
                .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000)))
                .thread(10)
                .addPipeline(this.springDataPipeline)
                .run();
    }

}

package cn.itcast.job.task;

import cn.itcast.job.pojo.JobInfo;
import cn.itcast.job.service.JobInfoService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.pipeline.Pipeline;
//Component  添加这个为创建实例
@Component
public class SpringDataPipeline  implements Pipeline {

    @Autowired
    private JobInfoService jobInfoService;

    // 处理最后的持久化操作，或者业务操作  实现Pipeline即可   处理结果集
    @Override
    public void process(ResultItems resultItems, Task task) {
        //获取封装好的招聘详情
        JobInfo jobInfo =resultItems.get("jobInfo");
        if (jobInfo !=null){
                this.jobInfoService.save(jobInfo);
        }

        //判断数据是否不为空
    }
}

数据库查询结果

附：源码下载地址