python+scrapy+mysql+css/xpath搭建爬取豆瓣的学习笔记（亲测编写css和xpath均可用）-优快云博客

本文链接：https://blog.youkuaiyun.com/qingshu0400/article/details/89184786

一.创建工程（前提是scrapy和pymysql成功安装）（参考博客：https://www.jianshu.com/p/b8c0e6b5cf9e?utm_campaign=maleskine&utm_content=note&utm_medium=seo_notes&utm_source=recommendation和 https://www.jianshu.com/p/79af7806d01a）

scrapy startproject doubanmove

进入 doubanmove 创建spider：

cd douban 执行命令 scrapy genspider douban douban.com

二.项目文件主要有：

scrapy.cfg：项目的配置文件

items.py：项目中的 item 文件

pipelines.py：项目中的 pipelines 文件

settings.py：项目的设置文件

spiders/：放置 spider 代码的文件夹

三.创建main.py在setting同目录下,以便开发测试内容如下：

from scrapy.cmdline import execute

execute("scrapy crawl douban".split())

编写 items.py:

import scrapy

class DoubanmovieItem(scrapy.Item):
    title = scrapy.Field()  # 电影名字
    movieInfo = scrapy.Field()  # 电影的描述信息，包括导演、主演、电影类型等等
    star = scrapy.Field()  # 电影评分
    quote = scrapy.Field()  # 电影中最经典或者说脍炙人口的一句话
    pass
编写spider：（亲测编写css和xpath均可用）

# -*- coding: utf-8 -*-
import scrapy
from scrapy.http import Request
from urllib.parse import urljoin
from doubanmovie.items import DoubanmovieItem
from scrapy.selector import Selector
class DoubanSpider(scrapy.Spider):
    name = 'douban'
    allowed_domains = ['douban.com']
    start_urls = ['https://movie.douban.com/top250']
    def parse(self,response):
        item=DoubanmovieItem()
        # selector = Selector(response)
        # Movies = selector.xpath('//div[@class="info"]')
        Movies=response.css('ol[class=grid_view] li')
        for movie in Movies:
            print(movie)
            # title = movie.xpath('div[@class="hd"]/a/span/text()').extract()  # 多个span标签
            title=movie.css('div[class="hd"] a span::text').extract()
            fullTitle = "".join(title)  # 将多个字符串无缝连接起来
           # movieInfo = movie.xpath('div[@class="bd"]/p/text()').extract()
            movieInfo=movie.css('div[class="bd"] p::text').extract()
            # star = movie.xpath('div[@class="bd"]/div[@class="star"]/span/text()').extract()[0]
            star =movie.css('span[class="rating_num"]::text').extract()
            # quote = movie.xpath('div[@class="bd"]/p[@class="quote"]/span/text()').extract()
            quote =movie.css('span[class="inq"]::text').extract()
            if quote:
                quote=quote[0]
            else:
                quote=""
            item['title'] = fullTitle
            item['movieInfo'] = ';'.join(movieInfo)
            item['star'] = star
            item['quote'] = quote
            yield item
        # nextLink = selector.xpath('//span[@class="next"]/link/@href').extract()
            nextLink= response.css('span[class="next"] a::attr(href)').extract()
        # # 第10页是最后一页，没有下一页的链接
        if nextLink:
            nextLink = nextLink[0]
            yield Request(urljoin(response.url, nextLink), callback=self.parse)

编写pip：

import pymysql
from doubanmovie import settings

class DoubanmoviePipeline(object):
    def __init__(self):
        self.connect = pymysql.connect(
            host=settings.MYSQL_HOST,
            db=settings.MYSQL_DBNAME,
            user=settings.MYSQL_USER,
            passwd=settings.MYSQL_PASSWD,
            # charset='utf8',
            #use_unicode=True
            )
        self.cursor = self.connect.cursor()
    def process_item(self, item, spider):
        try:
            self.cursor.execute(
                """insert into doubantop250(title,movieInfo,star,quote)
                  value (%s,%s,%s,%s)""",
                (item['title'],
                 item['movieInfo'],
                 item['star'],
                 item['quote']))
            self.connect.commit()
        except Exception as err:
            print("重复插入了==>错误信息为：" + str(err))
        return item

设置setting：

# -*- coding: utf-8 -*-
import random
# Scrapy settings for doubanmovie project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://doc.scrapy.org/en/latest/topics/settings.html
#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'doubanmovie'

SPIDER_MODULES = ['doubanmovie.spiders']
NEWSPIDER_MODULE = 'doubanmovie.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
# USER_AGENT = 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50'
# user agent 列表
USER_AGENT_LIST = [
    'MSIE (MSIE 6.0; X11; Linux; i686) Opera 7.23',
    'Opera/9.20 (Macintosh; Intel Mac OS X; U; en)',
    'Opera/9.0 (Macintosh; PPC Mac OS X; U; en)',
    'iTunes/9.0.3 (Macintosh; U; Intel Mac OS X 10_6_2; en-ca)',
    'Mozilla/4.76 [en_jp] (X11; U; SunOS 5.8 sun4u)',
    'iTunes/4.2 (Macintosh; U; PPC Mac OS X 10.2)',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:5.0) Gecko/20100101 Firefox/5.0',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0) Gecko/20100101 Firefox/9.0',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:16.0) Gecko/20120813 Firefox/16.0',
    'Mozilla/4.77 [en] (X11; I; IRIX;64 6.5 IP30)',
    'Mozilla/4.8 [en] (X11; U; SunOS; 5.7 sun4u)'
]
# 随机生成user agent
USER_AGENT = random.choice(USER_AGENT_LIST)
# Obey robots.txt rules
ROBOTSTXT_OBEY = True

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
#   'Accept-Language': 'en',
#}

# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'doubanmovie.middlewares.DoubanmovieSpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
#    'doubanmovie.middlewares.DoubanmovieDownloaderMiddleware': 543,
#}

# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
#ITEM_PIPELINES = {
#    'doubanmovie.pipelines.DoubanmoviePipeline': 300,
#}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
MYSQL_HOST = 'localhost'  # 数据库地址
MYSQL_DBNAME = 'python'  # 数据库名字
MYSQL_USER = 'root'  # 数据库登录名
MYSQL_PASSWD = '123456'  # 数据库登录密码
# 数据传输
ITEM_PIPELINES = {
    'doubanmovie.pipelines.DoubanmoviePipeline': 301,
}

可用资源链接：https://download.youkuaiyun.com/download/qingshu0400/11103187