python scrapy爬虫整合chrome驱动

安装scrap我就不说了，
python 安装 selenium 库
pip install selenium

# -*- coding: utf-8 -*-
import scrapy
import logging
from selenium import webdriver
from selenium.webdriver.chrome.options import Options

class MydomainSpider(scrapy.Spider):
    name = 'mydomain'
    allowed_domains = ['mydomain.com']
    start_urls = ['http://www.baidu.com']

    def __init__(self):
        #这个是一个用来控制chrome以无界面模式打开的浏览器
        #创建一个参数对象，用来控制chrome以无界面的方式打开
        chrome_options = Options()
        #后面的两个是固定写法 必须这么写
        chrome_options.add_argument('--headless')
        chrome_options.add_argument('--disable-gpu')
        self.browser = webdriver.Chrome(executable_path = r'C:\document\python\chromedriver.exe',chrome_options=chrome_options)

    def parse(self, response):
      #  logging.getLogger(__name__).debug("11")
        logging.debug(response.body)


        pass

    def closed(self,spider):
        print("关闭浏览器")
        self.browser.quit() 

下面就算设置下载中间件，如果你设置了，但是你返回的是none，系统仍然会执行自己的下载器，所以你返回none，发现系统还是执行自己的下载器，这里你就不要误会，是设置失效了，其实是对的

DOWNLOADER_MIDDLEWARES = {
   'myscrapy.middlewares.MyscrapyDownloaderMiddleware': 543,
   'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': None, // 关闭系统的下载器
   
}

下面就是自定义下载器

class MyscrapyDownloaderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider): #重新这个方法

        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of

        
        #   installed downloader middleware will be called

        #spider.logger.debug("11")


        print('即将返回一个新的响应对象')
        bro = spider.browser
        bro.get(url=request.url)
        #注意 这两个写法是一致的
        # bro.get(url='http://war.163.com/')
        # bro.get(url=spider.bro.current_url)
        bro.implicitly_wait(10)
        #包含了动态加载出来的数据   获取到页面源码数据
        page_text = bro.page_source
        #body 响应体
        return HtmlResponse(url=request.url,body=page_text,encoding='utf-8',request=request) #这里返回none，系统仍然会调用自己的下载器

    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.

        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
        return response

    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain
        pass

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)