用xpath爬取豆瓣

最新推荐文章于 2024-04-02 00:01:19 发布

原创最新推荐文章于 2024-04-02 00:01:19 发布 · 647 阅读

1 ·

CC 4.0 BY-SA版权

爬虫专栏收录该内容

8 篇文章

订阅专栏

（继上次正则的爬取豆瓣的另一种方法）

import requests
from lxml import html
url = "https://movie.douban.com/chart"
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36','Cookie':'ll="118088"; bid=pEvZmYymg9w; __yadk_uid=MBVdmoWQpfuaAduWgI3sr27xdVb2SpxL; _vwo_uuid_v2=D48A302F0862238CF3F23ACA0683DA5FD|96bd09ebd9b5983c45ac0b80abdd5580; trc_cookie_storage=taboola%2520global%253Auser-id%3Daeb8d923-8e31-45ff-9a11-7a4a24c2fd2a-tuct46c9718; viewed="6898903"; gr_user_id=3ca08a9d-8af2-4ae5-b72c-1bd6343499e3; __utmc=30149280; __utmz=30149280.1569503137.3.3.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; __utmc=223695111; __utmz=223695111.1569503137.2.2.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; acw_tc=276082a815695948946125465eed822b29faf81d8ad94d5cf0d821287a5f92; ap_v=0,6.0; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1569594896%2C%22https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3D9Tr6b3hwv1bkFIt9iIEG9pjP2rKLHFpsgJYvz_JZ4P-gjFxDN7hKNDQUVT_-57rw%26wd%3D%26eqid%3D99624ff700502a24000000035d8cb79c%22%5D; _pk_id.100001.4cf6=0d8c49bca23e986a.1568892239.4.1569594896.1569541094.; __utma=30149280.1815959067.1568892238.1569541094.1569594896.5; __utma=223695111.1519397804.1568892239.1569541094.1569594896.4'}
response=requests.get(url,headers=headers,timeout=2)
response.encoding = response.apparent_encoding
html=response.text
selector=lxml.html.fromstring(html)

titles= selector.xpath("//a[@class='nbg']/@title")
webs=selector.xpath("//a[@class='nbg']/@href") 
scores=selector.xpath("//span[@class='rating_nums']/text()")
contents=selector.xpath("//p[@class='pl']/text()")
for i in range(0,11):
    print("片名:",titles[i])
    print("评分:",scores[i]) 
    print("详情:",contents[i])
    print("网址:",webs[i])
    print('\n')