爬取豆瓣

'''
爬取豆瓣网站上
经典电影的   名字  和剧情简介
'''
#encoding:utf-8
import requests
from lxml import etree


class  DoubanSpider(object):
    def __init__(self):
        self.url =  "https://movie.douban.com/explore#!type=movie&tag=%E7%BB%8F%E5%85%B8"
        self.headers = {
            "User-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36"
        }
        self.data_list = []

    #发起请求
    def get_response(self,url):
        response = requests.get(url,headers = self.headers)
        data = response.content.decode("utf-8")
        return data

    #解析数据
    def parse_data(self,data,i):
        xpath_data = etree.HTML(data)
        #获取电影名
        name_list = xpath_data.xpath('//div[@class="hd"]/a/span[1]/text()')
        #每部电影的url
        url_list = xpath_data.xpath('//div[@class="hd"]/a/@href')

        print(len(name_list ))
        print(name_list )
        print(len(url_list))
        print(url_list)
        #将电影名字和url以字典的形式存储
        # for index,name in enumerate(name_list):
        #     dict_film = {}
        #     dict_film['name'] =name
        #     dict_film['url'] = url_list[index]
        #     self.data_list.append(dict_film)
        # print(self.data_list)
        self.save(i,name_list,url_list)


    def save(self,i,name_list,url_list):
        j=i+1
        # with open('第{}页.html'.format(j),'w',encoding='utf-8') as f:
        #     f.write(data)
        f = open("./豆瓣爬取/"+"第{}页.txt".format(j),'w')
        for index, name in enumerate(name_list):
            f.write(name)
            f.write('\n')
            f.write(url_list[index])
            f.write('\n')
        f.close()


    #启动
    def  run(self):
        page = input("请输入要爬取的页数：")
        for i in range(int(page)):
            self.url = "https://movie.douban.com/top250?start={}".format(str(i*25))
            print(self.url)
            data = self.get_response(self.url)
            self.parse_data(data,i)




DoubanSpider().run()