webMagic--注解爬取数据

最新推荐文章于 2022-03-01 12:50:11 发布

weixin_30345577

最新推荐文章于 2022-03-01 12:50:11 发布

阅读量493

点赞数

CC 4.0 BY-SA版权

文章标签： json 爬虫数据库

原文链接：http://www.cnblogs.com/mxh-java/p/10906420.html

此博客为转载内容，原链接为https://www.cnblogs.com/mxh-java/p/10906420.html ，标签涉及JSON、爬虫和数据库等信息技术领域内容。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.model.ConsolePageModelPipeline;
import us.codecraft.webmagic.model.OOSpider;
import us.codecraft.webmagic.model.annotation.ExtractBy;
import us.codecraft.webmagic.model.annotation.TargetUrl;
import us.codecraft.webmagic.pipeline.JsonFilePipeline;
import us.codecraft.webmagic.scheduler.RedisScheduler;

/**
 * @auther mxh
 * @time 2019/5/22 10:22
 */
@TargetUrl("https://b.faloo.com/y/0/0/0/0/0/0/[\\d+].html")
public class Demo {

    @ExtractBy("//div[@class=\"l_bar\"]/div[@class=\"l_rc\"]/div[@class=\"l_title a_666\"]/h1/a/text()")
    private String name;

    @ExtractBy("//div[@class=\"l_bar\"]/div[@class=\"l_rc\"]/div[@class=\"l_title a_666\"]/span/allText()")
    private String auther;

    @ExtractBy("//div[@class=\"l_bar\"]/div[@class=\"l_rc\"]/div[@class=\"l_con\"]/a/text()")
    private String introduce;

    /*==============get/set=============*/

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

    public String getAuther() {
        return auther;
    }

    public void setAuther(String auther) {
        this.auther = auther;
    }

    public String getIntroduce() {
        return introduce;
    }

    public void setIntroduce(String introduce) {
        this.introduce = introduce;
    }

    public static void main(String[] args) {
        OOSpider.create(Site.me(), new ConsolePageModelPipeline(), Demo.class)
                .addUrl("https://b.faloo.com/y/0/0/0/0/0/0/1.html")
                // new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(10000000))
                .setScheduler(new RedisScheduler("127.0.0.1"))
                .addPipeline(new JsonFilePipeline("D:\\webmagic\\"))
                .thread(4)
                .run();
    }

}