JAVA爬虫（三）-51招聘信息

最新推荐文章于 2023-02-18 19:50:32 发布

qq_40860185

最新推荐文章于 2023-02-18 19:50:32 发布

阅读量246

点赞数

分类专栏： Java爬虫文章标签： java 爬虫

本文链接：https://blog.youkuaiyun.com/qq_40860185/article/details/118464700

版权

Java爬虫专栏收录该内容

3 篇文章

订阅专栏

一.环境搭建

1. 创建数据库表

// 创建crawler数据库, 再创建表
CREATE TABLE `job_info` (
  `id` bigint(0) NOT NULL AUTO_INCREMENT COMMENT '主键id',
  `company_name` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '公司名称',
  `company_addr` varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '公司联系方式',
  `company_info` text CHARACTER SET utf8 COLLATE utf8_general_ci NULL COMMENT '公司信息',
  `job_name` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '职位名称',
  `job_addr` text CHARACTER SET utf8 COLLATE utf8_general_ci NULL COMMENT '工作地点',
  `job_info` text CHARACTER SET utf8 COLLATE utf8_general_ci NULL COMMENT '职位信息',
  `salary_min` int(0) NULL DEFAULT NULL COMMENT '薪资范围，最小',
  `salary_max` int(0) NULL DEFAULT NULL COMMENT '薪资范围，最大',
  `url` varchar(150) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '招聘信息详情页',
  `time` varchar(30) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '职位最近发布时间',
  PRIMARY KEY (`id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COMMENT='招聘信息';

2.导入pom.xml依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.example</groupId>
    <artifactId>itcast_third</artifactId>
    <version>1.0-SNAPSHOT</version>

    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.0.2.RELEASE</version>
    </parent>


    <dependencies>
        <!--SpringMVC-->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

        <!--SpringData Jpa-->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-jpa</artifactId>
        </dependency>

        <!--MySQL连接包-->
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>8.0.22</version>
        </dependency>

        <!-- HttpClient -->
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
        </dependency>

        <!--Jsoup-->
        <dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.10.2</version>
        </dependency>

        <!--工具包-->
        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-lang3</artifactId>
        </dependency>
        <dependency>
            <groupId>us.codecraft</groupId>
            <artifactId>webmagic-core</artifactId>
            <version>0.7.4</version>
        </dependency>
        <dependency>
            <groupId>us.codecraft</groupId>
            <artifactId>webmagic-extension</artifactId>
            <version>0.7.4</version>
        </dependency>
    </dependencies>
</project>

3.配置配置application.properties文件

spring.datasource.url=jdbc:mysql://localhost:3306/itcast?useUnicode=true&characterEncoding=UTF-8&useSSL=false&serverTimezone=UTC
spring.datasource.username=root
spring.datasource.password=xxxxxx
spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver

spring.jpa.database=MySQL
spring.jpa.show-sql=true

配置log4j

log4j.rootLogger=DEBUG,A1
log4j.logger.cn.itcast=DEBUG

log4j.appender.A1=org.apache.log4j.ConsoleAppender
log4j.appender.A1.layout=org.apache.log4j.PatternLayout
log4j.appender.A1.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss,SSS} [%t] [%c]-[%p] %m%n

4.创建JobInfo实体类



import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;

@Entity
public class JobInfo {

    @Id
    @GeneratedValue(strategy = GenerationType.IDENTITY)
    private Long id;
    private String companyName;
    private String companyAddr;
    private String companyInfo;
    private String jobName;
    private String jobAddr;
    private String jobInfo;
    private Integer salaryMin;
    private Integer salaryMax;
    private String url;
    private String time;

    public Long getId() {
        return id;
    }

    public void setId(Long id) {
        this.id = id;
    }

    public String getCompanyName() {
        return companyName;
    }

    public void setCompanyName(String companyName) {
        this.companyName = companyName;
    }

    public String getCompanyAddr() {
        return companyAddr;
    }

    public void setCompanyAddr(String companyAddr) {
        this.companyAddr = companyAddr;
    }

    public String getCompanyInfo() {
        return companyInfo;
    }

    public void setCompanyInfo(String companyInfo) {
        this.companyInfo = companyInfo;
    }

    public String getJobName() {
        return jobName;
    }

    public void setJobName(String jobName) {
        this.jobName = jobName;
    }

    public String getJobAddr() {
        return jobAddr;
    }

    public void setJobAddr(String jobAddr) {
        this.jobAddr = jobAddr;
    }

    public String getJobInfo() {
        return jobInfo;
    }

    public void setJobInfo(String jobInfo) {
        this.jobInfo = jobInfo;
    }

    public Integer getSalaryMin() {
        return salaryMin;
    }

    public void setSalaryMin(Integer salaryMin) {
        this.salaryMin = salaryMin;
    }

    public Integer getSalaryMax() {
        return salaryMax;
    }

    public void setSalaryMax(Integer salaryMax) {
        this.salaryMax = salaryMax;
    }

    public String getUrl() {
        return url;
    }

    public void setUrl(String url) {
        this.url = url;
    }

    public String getTime() {
        return time;
    }

    public void setTime(String time) {
        this.time = time;
    }

    @Override
    public String toString() {
        return "JobInfo{" +
                "id=" + id +
                ", companyName='" + companyName + '\'' +
                ", companyAddr='" + companyAddr + '\'' +
                ", companyInfo='" + companyInfo + '\'' +
                ", jobName='" + jobName + '\'' +
                ", jobAddr='" + jobAddr + '\'' +
                ", jobInfo='" + jobInfo + '\'' +
                ", salaryMin=" + salaryMin +
                ", salaryMax=" + salaryMax +
                ", url='" + url + '\'' +
                ", time='" + time + '\'' +
                '}';
    }
}

5.编写JobInfoDao 接口

import jd.pojo.JobInfo;
import org.springframework.data.jpa.repository.JpaRepository;

// Item的Dao接口
public interface JobInfoDao extends JpaRepository<JobInfo, Long> {

}

6.编写JobInfoService接口

import jd.pojo.JobInfo;

import java.util.List;

// Item的Service接口
public interface JobInfoService {

    /**
     * 保存商品
     * @param jobInfo
     */
    public void save(JobInfo jobInfo);

    /**
     * 根据条件查询商品
     * @param jobInfo
     * @return
     */
    public List<JobInfo> findAll(JobInfo jobInfo);
}

7.编写JobInfoServiceImpl实现类

import jd.Dao.JobInfoDao;
import jd.pojo.JobInfo;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Example;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

import java.util.List;

@Service
public class JobInfoServiceImp implements JobInfoService {
    @Autowired
    private JobInfoDao jobInfoDao;

    @Override
    @Transactional
    public void save(JobInfo jobInfo) {

        this.jobInfoDao.save(jobInfo);
    }

    @Override
    public List<JobInfo> findAll(JobInfo jobInfo) {
        //明查询条件
        Example<JobInfo> example=Example.of(jobInfo);
        //根据查询条件进行查询数据
        List<JobInfo> list=this.jobInfoDao.findAll(example);

        return list;
    }
}

8.编写爬虫实现类JobProcessor

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;

import jd.pojo.JobInfo;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.selector.Html;

import java.text.SimpleDateFormat;
import java.util.Date;

//@Component
public class JobProcessor implements PageProcessor {
 private int count = 1;
    private String url = "https://search.51job.com/list/030200,000000,0000,00,9,99,java,2,1.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=";
    private SimpleDateFormat formatter = new SimpleDateFormat("yyyy-");
    private String today = formatter.format(new Date());
    @Override
    public void process(Page page) {
        // 因为在51招聘的页面中不能直接拿到页面元素, 数据在页面的js中
        // 于将js对象(json数据)解析出来, 获取到详情页面的链接放到任务队列中
        // 解析页面, 获取script中招聘详情信息
        String dataJs = page.getHtml().css("script").regex(".*SEARCH_RESULT.*").get();
        // 判断获取到的页面是否为空
        if (!StringUtils.isEmpty(dataJs)) {
            System.out.println("开始抓取第" + count++ + "页");
            // 如果不为空, 表示这是列表页
            // 解析拿到json字符串
            dataJs = dataJs.substring(dataJs.indexOf("{"), dataJs.lastIndexOf("}") + 1);
            // 创建json对象
            JSONObject jsonObject = (JSONObject) JSONObject.parse(dataJs);
            // 根据分析拿到放置信息的数组
            JSONArray resArr = jsonObject.getJSONArray("engine_search_result");
            // 判断数组中是否存在数据
            if (resArr.size() > 0) {
                for (int i = 0; i < resArr.size(); i++) {
                    // 获取数组中的每一个对象
                    JSONObject resObj = (JSONObject) resArr.get(i);
                    //把获取到的url地址放到任务队列中
                    page.addTargetRequest(String.valueOf(resObj.get("job_href")));
                }
                // 获取下一页的url
                String bkUrl = "https://search.51job.com/list/030200,000000,0000,00,9,99,java,2," + (count + 1) + ".html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=";
                // 把url放到任务队列中
                page.addTargetRequest(bkUrl);
            } else {
                // 设置变量为初始值
                count = 0;
                // 如果没有数据那么爬虫结束
                return;
            }
        } else {
            // 如果为空, 表示这是招聘详情页, 解析页面, 获取招聘详情信息, 保存数据
            this.saveJobInfo(page);
        }

    }

    // 解析页面, 获取招聘详情信息, 保存数据
    private void saveJobInfo(Page page) {
        // 创建招聘详情对象
        JobInfo jobInfo = new JobInfo();

        // 解析页面
        Html html = page.getHtml();

        // 获取数据, 封装到对象中
        // 拿到有地址和发布时间的那条文字
        String desc = Jsoup.parse(html.css("p.msg.ltype").toString()).text();
        if (desc == null) {
            // 有极少数非51内部网站无法找到
            return;
        }
        // 为了能够找到发布时间截取了字符串
        desc = desc.substring(0, desc.lastIndexOf("发布"));
        // 设置公司名称
        jobInfo.setCompanyName(html.css("div.cn p.cname a", "text").toString().trim());
        // 设置公司地址
        jobInfo.setCompanyAddr(desc.substring(0, desc.indexOf("|")).trim());
        // 设置公司信息
        jobInfo.setCompanyInfo(Jsoup.parse(html.css("div.tmsg").toString()).text());
        // 设置职位名称
        jobInfo.setJobName(html.css("div.cn h1", "text").toString());
        // 设置工作地址
        jobInfo.setJobAddr(html.css("div.bmsg>p.fp", "text").toString());
        // 设置工作信息(要求)
        jobInfo.setJobInfo(Jsoup.parse(html.css("div.job_msg").toString()).text());
        // 设置当前链接
        jobInfo.setUrl(page.getUrl().toString());
        // 获取薪资
        // 有的没有写薪资, 先获取薪资的字符串
        String salaryText = html.css("div.cn strong", "text").toString();
        // 看看是否没有薪资这个字段
        if (!StringUtils.isEmpty(salaryText)) {
            // 使用工具类转换薪资字符串
            Integer[] salary = MathSalary.getSalary(salaryText);
            jobInfo.setSalaryMax(salary[0]);
            jobInfo.setSalaryMin(salary[1]);
        } else {
            // 没有则设为零
            jobInfo.setSalaryMax(0);
            jobInfo.setSalaryMin(0);
        }
        // 获取发布时间
        // 获取发布时间
        String time = desc.substring(desc.lastIndexOf("|") + 3);
        jobInfo.setTime(today + time.trim());
        // 把结果保存起来
        page.putField("jobInfo", jobInfo);
    }

    private Site site = Site.me()
            .setCharset("gbk") // 设置编码
            .setTimeOut(10*1000) // 设置超时时间
            .setRetrySleepTime(1000) // 设置重试的间隔时间
            .setRetryTimes(10); // 设置重试的次数

    @Override
    public Site getSite() {
        return site;
    }

    @Autowired
    private SpringDataPipeline springDataPipeline;

    // initialDelay当任务启动后, 等多久执行方法
    // fixedDelay每隔多久执行方法
    @Scheduled(initialDelay = 1000, fixedDelay = 1000 * 1000)
    public void process() {
        Spider.create(new JobProcessor())
                .addUrl(url)
                // 设置Secheduler
                .setScheduler(new QueueScheduler()
                        // 设置Bloom去重
                        .setDuplicateRemover(new BloomFilterDuplicateRemover(100000)))
                .thread(10)
                // 设置自定义的Pipeline储存数据
                .addPipeline(this.springDataPipeline)
                .run();
    }
}

9.添加工具类MathSalary

// 转换薪资
public class MathSalary {
    /**
     * 获取薪水范围
     */
    public static Integer[] getSalary(String salaryStr) {
        //声明存放薪水范围的数组
        Integer[] salary = new Integer[2];

        //"500/天"
        //0.8-1.2万/月
        //5-8千/月
        //5-6万/年
        String date = salaryStr.substring(salaryStr.length() - 1, salaryStr.length());
        //如果是按天，则直接乘以240进行计算
        if (!"月".equals(date) && !"年".equals(date)) {
            salaryStr = salaryStr.substring(0, salaryStr.length() - 2);
            salary[0] = salary[1] = str2Num(salaryStr, 240);
            return salary;
        }

        String unit = salaryStr.substring(salaryStr.length() - 3, salaryStr.length() - 2);
        String[] salarys = salaryStr.substring(0, salaryStr.length() - 3).split("-");


        salary[0] = mathSalary(date, unit, salarys[0]);
        salary[1] = mathSalary(date, unit, salarys[1]);

        return salary;


    }

    //根据条件计算薪水
    private static Integer mathSalary(String date, String unit, String salaryStr) {
        Integer salary = 0;

        //判断单位是否是万
        if ("万".equals(unit)) {
            //如果是万，薪水乘以10000
            salary = str2Num(salaryStr, 10000);
        } else {
            //否则乘以1000
            salary = str2Num(salaryStr, 1000);
        }

        //判断时间是否是月
        if ("月".equals(date)) {
            //如果是月，薪水乘以12
            salary = str2Num(salary.toString(), 12);
        }

        return salary;
    }


    private static int str2Num(String salaryStr, int num) {
        try {
            // 把字符串转为小数，必须用Number接受，否则会有精度丢失的问题
            Number result = Float.parseFloat(salaryStr) * num;
            return result.intValue();
        } catch (Exception e) {
        }
        return 0;
    }
}

10.定义SpringDataPipeline类存储数据

import jd.pojo.JobInfo;
import jd.service.JobInfoService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.pipeline.Pipeline;

@Component
public class SpringDataPipeline implements Pipeline {

    @Autowired
    private JobInfoService jobInfoService;
    @Override
    public void process(ResultItems resultItems, Task task) {

        // 获取封装好的招聘详情对象
        JobInfo jobInfo = resultItems.get("jobInfo");

        // 判断是否不为空
        if (jobInfo != null) {
            // 如果不为空把数据保存到数据库中
            this.jobInfoService.save(jobInfo);
        }
    }
}

学习视频来源于：https://www.bilibili.com/video/BV1cE411u7RA?p=41
相关学习资料来源于：https://www.bilibili.com/video/BV1cE411u7RA?p=41