存入mongo数据库

• 普通存

from selenium import webdriver
from lxml import etree
import pymongo

# 创建连接
conn = pymongo.MongoClient('localhost',27017)
# 建库
db = conn.wangyi
# 建表
table = db.wang

def save_data(dic):
    table.insert_one(dic)

def save_wangyi(url):
    broswer = webdriver.Chrome('./chromedriver.exe')
    broswer.get(url=url)

    html = broswer.page_source
    tree = etree.HTML(html)
    title = ''.join(tree.xpath('//ul[@class="newsdata_list fixed_bar_padding noloading"]/li//h3/a/text()'))
    dic = {'title':title}
    save_data(dic)

if __name__ == '__main__':
    url = 'https://news.163.com/domestic/'
    save_wangyi(url)

• 框架存
• 爬虫文件中

import scrapy
from selenium import webdriver
from ..items import WyxwItem

class WySpider(scrapy.Spider):
    name = 'wy'
    # allowed_domains = ['baidu.com']
    start_urls = ['https://news.163.com/domestic']
    broswer = webdriver.Chrome('./chromedriver.exe')


    def parse(self, response):
        div_list = response.xpath('//div[@class="ndi_main"]/div')
        for div in div_list:
            title = div.xpath('.//h3/a/text()').extract_first()
            # print(title)
            item = WyxwItem()
            item['title'] = title
            yield item

• 管道中

import pymongo
class WyxwPipeline(object):
    conn = pymongo.MongoClient()
    db = conn.wynews
    table = db.news
    def process_item(self, item, spider):
        self.table.insert_one(dict(item))
        return item

执行代码前需要将settings配置文件中的下载器和item管道打开注释