scrapy <一>_____开启多个爬虫-优快云博客

本文介绍了使用Scrapy框架批量运行爬虫的实现方法，包括命令语法、参数配置及环境设置，适合Python开发者快速上手。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

1 spiders同级目录创建commands

crawlall.py

#-*-coding:utf-8-*-

from scrapy.commands import ScrapyCommand
from scrapy.crawler import CrawlerRunner
from scrapy.utils.conf import arglist_to_dict
from scrapy.exceptions import UsageError


class Command(ScrapyCommand):
    requires_project = True

    def syntax(self):
        return '[options]'

    def short_desc(self):
        return 'Runs all of the spiders'

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
                          help="set spider argument (may be repeated)")
        parser.add_option("-o", "--output", metavar="FILE",
                          help="dump scraped items into FILE (use - for stdout)")
        parser.add_option("-t", "--output-format", metavar="FORMAT",
                          help="format to use for dumping items with -o")

    def process_options(self, args, opts):
        ScrapyCommand.process_options(self, args, opts)
        try:
            opts.spargs = arglist_to_dict(opts.spargs)
        except ValueError:
            raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)

    def run(self, args, opts):
        # settings = get_project_settings()

        spider_loader = self.crawler_process.spider_loader
        for spidername in args or spider_loader.list():
            print("*********cralall spidername************" + spidername)
            self.crawler_process.crawl(spidername, **opts.spargs)

        self.crawler_process.start()

setup.py

#-*-coding:utf-8-*-
from setuptools import setup, find_packages

setup(name='scrapy-mymodule',
  entry_points={
    'scrapy.commands': [
      'crawlall=cnblogs.commands:crawlall',
    ],
  },
 )

setting.py

#-------同时启动多个爬虫
COMMANDS_MODULE = 'BaiduSpider.commands'#爬虫名

cmd 运行 scrapy crawlall

windows设置计划任务创建run.py 加入基本任务中

#-*-coding:utf-8-*-
from scrapy import cmdline

cmdline.execute(['scrapy','crawlall'])

转载于:https://my.oschina.net/u/3771014/blog/1932601