jsoup爬虫

1.找URL

    String url = "";

2.响应是页面就用jsoup，是json数据就用httpclient

3.设置请求标头和负载并获取对应界面的html

Document document = .header("cookie","")
                    .header("User-Agent", "")
                    .get();

4.select、getElementsByClass、getElementsById

    Elements list1 = document.getElementsByClass("position-wrap");
    Elements list = list1.select("li");

5.遍历数据

for (Element element : list){
​
    String jobName = element.select("div.job-card-wrapper > span.job-name").text();
    String salary = element.select("div.position_item_inner > p:nth-child(1)").text();
​
    JiuYeJob.setJobName(jobName);
    JiuYeJob.setSalary(salary);
​
}

例子

public void JiuYe(String kd) {
    String url ="https://www.jobonline.cn/position?q="+kd;
    try {
        //获取对应界面的html
        Document document = Jsoup.connect(url)
        .header("cookie","")
        .header("User-Agent", "")
        .get();
​
        Elements list1 = document.getElementsByClass("position-wrap");
        Elements list = list1.select("li");
        for (Element element : list){
        
            String jobName = element.select("div.job-card-wrapper > div.job-card-body clearfix > a.job-card-left > div.job-title clearfix > span.job-name").text();
            String salary = element.select("div.position_item_inner > div.position_item_left > p:nth-child(1) > span.salary").text();
​
            JiuYeJob.setJobName(jobName);
            JiuYeJob.setSalary(salary);
​
        }
        jobDao.jiuYe(JiuYeJob); //插入数据库
    } catch (Exception e) {
        e.printStackTrace();
    }
}