抓取糗事百科段子代码

#代码来源 雨敲窗博客
# -*- coding: utf-8 -*-
import re
import requests
import html
import time

def crawl_joke_list(page=1):
    url = "http://www.qiushibaike.com/8hr/page/" + str(page)
    res = requests.get(url)
    #抽取每个段子的div的正则
    pattern = re.compile("<div class=\"article block untagged mb15 typs_long.*?<div class=\"content\">.*?</div>",re.S) #re.S匹配多行
    #把<br/>替换成换行
    body = html.unescape(res.text).replace("<br/>","\n")
    body = re.sub(r'<span>',"",body)
    body = re.sub(r'</span>', "", body)
    body = re.sub(r'<span class="contentForAll">查看全文</span>',"", body)

    m = pattern.findall(body)
    #抽取用户名的正则
    user_pattern = re.compile("<div class=\"author clearfix\">.*?<h2>(.*?)</h2>",re.S)
    #抽取段子的正则
    content_pattern = re.compile("<div class=\"content\">(.*?)</div>",re.S)
    for joke in m:
        user = user_pattern.findall(joke)

        output = []
        if len(user) > 0:
            output.append(user[0])

        content = content_pattern.findall(joke)
        if len(content) > 0:
            output.append(content[0].replace("\n",""))
        print("\t".join(output))
    time.sleep(1)

if __name__ == '__main__':
    for i in range(1,10):
        crawl_joke_list(i)

转载于:https://www.cnblogs.com/lili414/p/8901993.html