scrapy爬取王者荣耀皮肤

Scrapy爬取王者荣耀皮肤

一、项目需求

爬取王者荣耀官网（https://pvp.qq.com/）所有皮肤图片

要求：

• 英雄名作为文件夹名
• 皮肤名作为图片名称
• 皮肤图片按所属英雄存入对应的文件夹

二、项目解析

• 定义items文件，设置hero_name、pf_names、image_urls、images字段
• 分析网页，确定抓取思路
• 设置spider文件，抓取所有英雄名称、皮肤名称、图片URL
• 数据过多，设置下载中间件，伪装成浏览器
• 设置item管道，重写管道类，继承ImagesPipeline，根据需求改变下载路径与图片名称
• 设置整理settings文件
• 处理bug

三、项目代码

items.py

import scrapy

class PvpqqItem(scrapy.Item):
    # define the fields for your item here like:
    hero_name = scrapy.Field()      # 英雄名称
    pf_names = scrapy.Field()       # 皮肤名称
    image_urls = scrapy.Field()     # 项目的图像URL
    images = scrapy.Field()         # 有关下载图像的信息字段

pf.py

import scrapy
from ..items import PvpqqItem

class PfSpider(scrapy.Spider):
    name = 'pf'
    # allowed_domains = ['https://pvp.qq.com']
    start_urls = ['https://pvp.qq.com/web201605/herolist.shtml']

    def parse(self, response):
        pf_urls = response.xpath('//ul[@class="herolist clearfix"]/li/a/@href').extract()
        for pf_url in pf_urls:
            yield scrapy.Request(url='https://pvp.qq.com/web201605/%s' % pf_url, callback=self.pf_parse)

    def pf_parse(self, response):
        item = PvpqqItem()
        item['hero_name'] = response.xpath('//h2[@class="cover-name"]/text()').extract_first()
        # '圣骑之力&0|死亡骑士&1|狮心王&13|心灵战警&12' ==》 ['圣骑之力', '死亡骑士', '狮心王', '心灵战警']
        item['pf_names'] = response.xpath('//ul[@class="pic-pf-list pic-pf-list3"]/@data-imgname').re('(.*?)\&\\d+\|?')
        item['image_urls'] = []
        for num in range(1, len(item['pf_names'])+1):
            # //game.gtimg.cn/imgs/yxzj/img201606/heroimg/166/166-mobileskin-1.jpg
            # 去除-后面的字符，再重新进行拼接
            image_url_head = response.xpath('//a[@class="hero-video"]/img/@src').extract_first()[:-5]
            image_url = "https:{}{}.jpg".format(image_url_head, num)
            item['image_urls'].append(image_url)
        yield item

middlewares.py

import random

class RandomUserAgentMiddleware(object):

    def __init__(self, user_agents):
        self.user_agents = user_agents

    @classmethod
    def from_crawler(cls, crawler):
        # 在settings.py文件中加载MY_USER_AGENTS的值
        s = cls(user_agents=crawler.settings.get('MY_USER_AGENTS'))
        return s

    def process_request(self, request, spider):
        # 随机设置User-Agent的值
        agent = random.choice(self.user_agents)
        # 将其赋给Request
        request.headers['User-Agent'] = agent
        # proxy = random.choice(self.proxy)
        # request.meta['proxy'] = proxy
        return None

pipelines.py

import os
from scrapy.pipelines.images import ImagesPipeline
from . import settings


# 继承ImagesPipeline类
class PvpqqPipeline(ImagesPipeline):

    # 此方法是在发送下载请求之前调用，其实此方法本身就是去发送下载请求
    def get_media_requests(self, item, info):
        # 调用原父类方法，发送下载请求并获取返回的结果(request的列表)
        request_objs = super().get_media_requests(item, info)
        # 给每个request对象带上meta属性传入hero_name、pf_name参数，并返回
        for request_obj, num in zip(request_objs, range(0, len(item['pf_names']))):
            request_obj.meta['hero_name'] = item['hero_name']
            request_obj.meta['pf_name'] = item['pf_names'][num]
        return request_objs

    # 此方法是在图片将要被存储的时候调用，用来获取这个图片存储的全部路径
    def file_path(self, request, response=None, info=None):
        # 获取request的meta属性的hero_name作为文件夹名称
        hero_name = request.meta.get('hero_name')
        # 获取request的meta属性的pf_name并拼接作为文件名称
        image_name = request.meta.get('pf_name') + '.jpg'
        # 获取IMAGES_STORE图片的默认地址并拼接
        image_store = settings.IMAGES_STORE
        hero_name_path = os.path.join(image_store, hero_name)
        # 判断地址是否存在，不存则创建
        if not os.path.exists(hero_name_path):
            os.makedirs(hero_name_path)
        # 拼接文件夹地址与图片名图片存储的全部路径并返回
        image_path = os.path.join(hero_name_path, image_name)
        return image_path

settings.py

import os

BOT_NAME = 'pvpqq'
SPIDER_MODULES = ['pvpqq.spiders']
NEWSPIDER_MODULE = 'pvpqq.spiders'

ROBOTSTXT_OBEY = False

DOWNLOADER_MIDDLEWARES = {
   'pvpqq.middlewares.RandomUserAgentMiddleware': 543,
}
# 自定义的图片处理管道
ITEM_PIPELINES = {
   'pvpqq.pipelines.PvpqqPipeline': 300,
}
# 设置所有图片默认地址,必须设置
# IMAGES_STORE = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'imgs')
IMAGES_STORE = 'C:\\Users\\lenovo\\Desktop\\爬虫\\spider_demo\\pvpqq\\imgs'
# 设置图片通道失效时间
IMAGES_EXPIRES = 90
# 设置允许重定向，否则可能找不到图片
MEDIA_ALLOW_REDIRECTS = True
# 切换User_Agent
MY_USER_AGENTS = [
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 "
        "(KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
        "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 "
        "(KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 "
        "(KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
        "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 "
        "(KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
        "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 "
        "(KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 "
        "(KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
        "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 "
        "(KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "
        "(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
        "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 "
        "(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 "
        "(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 "
        "(KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "
        "(KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 "
        "(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "
        "(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 "
        "(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 "
        "(KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 "
        "(KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
        "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 "
        "(KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"
]

start.py

from scrapy import cmdline

# 运行此文件代替命令行运行scrapy项目
cmdline.execute("scrapy crawl pf".split(" "))

四、效果截图