scrapy与selenium结合爬取网易新闻

# -*- coding: utf-8 -*-
import scrapy
from selenium import webdriver
from wangyiPeo.items import WangyipeoItem
class WangyiSpider(scrapy.Spider):
    name = 'wangyi'
    # allowed_domains = ['www.xxx.com']
    start_urls = ['https://news.163.com']
    model_urls = []
    bro = webdriver.Chrome(executable_path=r'chromedriver.exe')

    def parse(self, response):
        #解析出5个板块对应的url
        li_list = response.xpath('//*[@id="index2016_wrap"]/div[1]/div[2]/div[2]/div[2]/div[2]/div/ul/li')
        model_index = [3,4,6,7,8]
        for index in model_index:
            #li依次表示的是5个板块对应的li标签
            li = li_list[index]
            #5个板块对应的url
            model_url = li.xpath('./a/@href').extract_first()
            self.model_urls.append(model_url)
            #对每一个板块的url进行手动请求的发送
            yield scrapy.Request(model_url,callback=self.parse_model)
    def parse_model(self,response):#用作于解析每一个板块对应页面数据中的新闻标题和新闻详情页的url
        #该方法中获取的response对象是没有包含动态加载出的新闻数据（是一个不满足需求的response）
        div_list = response.xpath('/html/body/div/div[3]/div[4]/div[1]/div/div/ul/li/div/div')#1+5+n
        for div in div_list:
            title = div.xpath('./div/div[1]/h3/a/text()').extract_first()
            detail_url = div.xpath('./a/@href').extract_first()
            item = WangyipeoItem()
            item['title'] = title
            yield scrapy.Request(detail_url,callback=self.parse_new_detail,meta={'item':item})
    def parse_new_detail(self,response):#解析新闻内容
        item = response.meta['item']
        content = response.xpath('//*[@id="endText"]//text()').extract()
        content = ''.join(content)
        item['content'] = content

        yield item
    #改方法只会在整个程序结束时执行一次
    def closed(self,spider):
        self.bro.quit()

 

下载中间件：

 

# -*- coding: utf-8 -*-

# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html

from scrapy import signals
from scrapy.http import HtmlResponse
from time import sleep


class WangyipeoDownloaderMiddleware(object):

    #参数：
    #reuqest:拦截到请求对应的响应对象
    #response：拦截到所有的响应对象（1+5+n）
    #spider：爬虫类实例化的对象，可以实现爬虫类和中间件类的数据交互
    def process_response(self, request, response, spider):
        #拦截到5个板块对应的响应对象，将其替换成5个符合需求的新的响应对象进行返回
        #1.找出5个板块对应的5个不符合需求的响应对象
        if request.url in spider.model_urls:
            #就是满足需求的五个板块对应的响应对象
            #url:响应对象对应的请求对象的url
            #body:响应数据,可以由selenium中的page_source返回
            bro = spider.bro
            bro.get(request.url)
            sleep(3)
            page_text = bro.page_source #包含了动态加载的新闻数据
            new_response = HtmlResponse(url=request.url,body=page_text,encoding='utf-8',request=request)
            return new_response
        else:
            return response