爬去top250的电影标题,导演,评分,评价,详细信息。
代码
#!/user/bin/env python
#-*- coding:utf-8 -*-
import requests
from bs4 import BeautifulSoup
import re
import time
def get_detail():#获取1-10页的数据
#获取电影名,导演,评分,关键词
for k in range(0, 250, 25):
url1 = 'https://movie.douban.com/top250?start={}&filter='.format(k)
for i in range(0,25):
try:#有些电影链接失效
list=[]
data = requests.get(url1).text
soup = BeautifulSoup(data, 'lxml')
name=soup.select('div > div.info > div.hd > a > span:nth-of-type(1)')[i].text#获取电影题目
director_starring = soup.select('li > div.item > div.info > div.bd > p')[2*i].text.strip()#获取导演与演员
director = re.split(r'\s+',director_starring)[1]
mark = soup.select('div.bd > div.star > span.rating_num')[i].text#获取评分
key_word = soup.select('div > div.info > div.bd > p.quote > span')[i].text#电影关键词
link = soup.select('li > div.item > div.pic > a')[i].get('href')#获取本页每个电影的链接
time.sleep(2)#防止被封
url = link
data1 = requests.get(url).text
soup1 = BeautifulSoup(data1, 'lxml')
introduction = soup1.select('#link-report > span:nth-of-type(1)')[0].text.strip()#获取每个电影的详情介绍
a = ''.join(introduction.split())#除去其中的大量空格
for b in (name,director,mark,key_word,a):
list.append(b)
print(list)
except:
print('error')
get_detail()
结果如下: