抓取糗事百科

# -*- coding:utf-8 -*-
import requests
from lxml import etree
page = 1
url = "https://www.qiushibaike.com/text/page/" + str(page)
headers = {"User-Agent" : "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0;"}
try:
    response = requests.get(url, headers=headers)
    resHtml = response.text
    html = etree.HTML(resHtml)
    # print etree.tostring(html)
    result = html.xpath('//div[contains(@id,"qiushi_tag")]')
    for site in result:
        item = {}
        imgUrl = site.xpath('./div/a/img/@src')[0]
        username = site.xpath('./div/a/h2')[0].text.strip()
        rank = site.xpath('./div/div')[0].text
        content = site.xpath('.//div[@class="content"]/span')[0].text.strip()
        vote = site.xpath('.//span[@class="stats-vote"]/i')[0].text
        comments = site.xpath('.//span[@class="stats-comments"]//i')[0].text
        print imgUrl, username, rank, content, vote, comments
except Exception, e:
    print e