oo1

Scrapy爬虫配置

最新推荐文章于 2024-11-01 15:23:40 发布

nio006

最新推荐文章于 2024-11-01 15:23:40 发布

阅读量245

点赞数

CC 4.0 BY-SA版权

分类专栏： scrapy

本文链接：https://blog.youkuaiyun.com/weixin_43890188/article/details/88879180

scrapy 专栏收录该内容

16 篇文章

订阅专栏

本文介绍了一个使用Python Scrapy框架的爬虫配置示例。该爬虫能够针对不同的目标进行网页抓取，如百度关键词搜索和百科页面。通过命令行参数可以灵活地设定爬虫类型、关键词、最大抓取深度及数量等。

import argparse,os,datetime,logging,time
from urllib.parse import quote
from scrapy.cmdline import execute
from crawler import settings


now=datetime.datetime.now()
# 设置日志
logger=logging.getLogger('crawler.py')
logger.setLevel(logging.INFO)
rq=time.strftime('%Y%m%d%H%M',time.localtime(time.time()))
log_path=os.path.dirname(os.getcwd())+'/Logs/'
log_name=log_path+rq+'.log'
logfile=log_name
fh=logging.FileHandler(logfile,mode='w')
fh.setLevel(logging.DEBUG)
formatter=logging.Formatter("%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s")
fh.setFormatter(formatter)
logger.addHandler(fh)


def main():
    # 判断所选择的爬虫是哪一个
    if FLAGS.type=='BaiduKeySearch':
        # 如果要求限制爬取的最大数量，执行scrapy自带的限制爬取数量的命令
        if FLAGS.max_nb_docs:
            execute("scrapy crawl {} -a key_name={} -s CLOSESPIDER_ITEMCOUNT={}".format(FLAGS.type,quote(FLAGS.key),FLAGS.max_nb_docs).split())
        else:
            execute("scrapy crawl {} -a key_name={}".format(FLAGS.type,quote(FLAGS.key)).split())
    elif FLAGS.type=='baike':
        if FLAGS.max_nb_docs:
            execute('scrapy crawl baike -s CLOSESPIDER_ITEMCOUNT={}'.format(FLAGS.max_nb_docs).split())
        else:
            execute('scrapy crawl {}'.format(FLAGS.type).split())
    else:
        logger.warning('Wrong Type!')

if __name__ == '__main__':
    # 创建命令行解析对象
    parser=argparse.ArgumentParser()
                                                                                                                         
                                                                                                                          # 添加命令行参数
    parser.add_argument('-t','--type',choices=['BaiduKeySearch','baike'],dest='type',help='choose one spider in choices list')
    parser.add_argument('-k','--key',dest='key',type=str,help='add the key you want to search')
    parser.add_argument('-m','--max-deepth',dest='max_deepth',type=int,help='set the maximum deepth you want to crawl')
    parser.add_argument('-max-nb-docs',dest='max_nb_docs',type=int,help='set the maximum quantity of items you want to crawl')
    parser.add_argument('-o','--output',dest='output',type=str,default=os.path.abspath('.')+'/crawled_data/',help='set the downloading output path')
    parser.add_argument('-l','--log',dest='log',type=str,default=os.path.abspath('.')+'/log/',help='set the log file path ')
    parser.add_argument('-f','--filename',dest='filename',type=str,help='recording the downloaded filename in this file')
    # 解析
    FLAGS, unparsed = parser.parse_known_args()
    settings.KEY_NAME=FLAGS.key
    settings.DEPTH_LIMIT=FLAGS.max_deepth
    settings.OUT_PUT=FLAGS.output
    settings.SHA_ONE=FLAGS.filename
    settings.LOG_FILE=FLAGS.log+'scrapy {} {} {}.log'.format(now.year,now.month,now.day)
    main()