scrapy 定时执行的两种方法

定时爬虫任务调度

最新推荐文章于 2025-09-18 04:22:23 发布

原创最新推荐文章于 2025-09-18 04:22:23 发布 · 4.7k 阅读

6 ·

CC 4.0 BY-SA版权

文章标签：

#scrapy

python 专栏收录该内容

54 篇文章

订阅专栏

本文介绍了一种使用Python实现的定时爬虫任务调度方案。通过schedule库和多进程结合Scrapy框架，实现了对不同爬虫任务的周期性调度执行。每个爬虫任务可以根据其特定的需求设置不同的执行频率。

# -*- coding: utf-8 -*-
import subprocess
import schedule
import time
import datetime
from multiprocessing import Process
from scrapy import cmdline
import logging
def crawl_work():
    # subprocess.Popen('scrapy crawl it')
    print('-'*100)
    # args = ["scrapy", "crawl", 'it']
    # while True:
    #     start = time.time()
    #     p = Process(target=cmdline.execute, args=(args,))
    #     p.start()
    #     p.join()
    #     logging.debug("### use time: %s" % (time.time() - start))
if __name__=='__main__':
    print('*'*10+'开始执行定时爬虫'+'*'*10)
    schedule.every(1).minutes.do(crawl_work)
    print('当前时间为{}'.format(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
    print('*' * 10 + '定时爬虫开始运行' + '*' * 10)
    while True:
        schedule.run_pending()
        time.sleep(10)

# -*- coding: utf-8 -*-
from multiprocessing import Process
from scrapy import cmdline
import time
import logging

# 配置参数即可, 爬虫名称，运行频率
confs = [
    {
        "spider_name": "it",
        "frequency": 2,
    },
]


def start_spider(spider_name, frequency):
    args = ["scrapy", "crawl", spider_name]
    while True:
        start = time.time()
        p = Process(target=cmdline.execute, args=(args,))
        p.start()
        p.join()
        logging.debug("### use time: %s" % (time.time() - start))
        time.sleep(frequency)


if __name__ == '__main__':
    for conf in confs:
        process = Process(target=start_spider,args=(conf["spider_name"], conf["frequency"]))
        process.start()
        time.sleep(86400)