lxml和Xpath实战

百度页面部分爬取

import requests
from lxml import etree

#定义被爬取的网页
url = "https://www.baidu.com/"

#定义请求头
headers = {
        "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
        }
#response对象，得到返回内容
response = requests.get(url, headers= headers)
content = response.content.decode('utf8')

#将html字符串解析
html = etree.HTML(content)
#使用xpath语法
contents = html.xpath("//div[@id='u1']/a/text()")#一定要加text()才能是字符串，否则返回对象。
#print(contents)
urls = html.xpath("//div[@id='u1']/a/@href")
#print(urls)

egs = []
for content,url in zip(contents,urls):
    eg = {}#每次清空字典，再添加新的字典
    eg = {
            "content":content,
            "url":url
            }
    egs.append(eg)

豆瓣影评爬取

import requests
from lxml import etree

headers = {
        "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
        
        }
"""
1.构造十页的url
"""
urls = []
for i in range(0,3,1):
    i = i*20
    url = "https://movie.douban.com/review/best/?start={}".format(i)
    urls.append(url)

"""
2.获取每页所有电影的详细url
"""
#list of list格式   列表嵌套列表
detail_urls = []
for url in urls:
    response = requests.get(url,headers=headers)
    #对response进行解码
    content = response.content.decode('utf8')
    #解析HTML字符串
    html = etree.HTML(content)
    #利用Xpath语法提取详情页的url[list形式的结果]
    detail_url = html.xpath('//h2/a/@href')
#    print(detail_url)
    detail_urls.append(detail_url)#列表嵌套列表
    
print(detail_urls)


"""
3.获取每一部电影的影评数据
"""
movies = []
i = 0
for page in detail_urls:#得到每一页的url
    for url in page:#得到每一部影评的url
        try:#为了防止有一个网页不是按照之前分析的格式，出错致使程序停止，使用try，except跳过异常网页
            #发送请求
            response = requests.get(url, headers=headers)
            content = response.content.decode('utf8')
            #利用etree进行解析
            html = etree.HTML(content)
            #提取电影名
            title = html.xpath('//div[@class="subject-title"]/a/text()')[0][2:]
            #提取评论人
            commenter = html.xpath('//header/a/span/text()')[0]
            #电影评分
            rank = html.xpath('//header//span/@title')[0]
            #电影评论
            comment = html.xpath('//div[@id="link-report"]//p//text()')
            comment = ''.join(comment)
            movie = {
            "title":title,
            "commenter":commenter,
            "rank":rank,
            "comment":comment,
            }
            movies.append(movie)
        except:
            continue
    i += 1
    print("第{}页已爬取完毕！！".format(i))