爬虫之requests+lxml+json获取糗百某分类数据（返回数据是html）

最新推荐文章于 2023-09-18 16:02:38 发布

一起学python吧

最新推荐文章于 2023-09-18 16:02:38 发布

阅读量593

点赞数

CC 4.0 BY-SA版权

分类专栏：爬虫文章标签：爬虫

本文链接：https://blog.youkuaiyun.com/myli_binbin/article/details/93906794

爬虫专栏收录该内容

27 篇文章

订阅专栏

本文介绍了一个使用Python的糗事百科爬虫项目，通过requests和lxml库抓取热门页面的内容，包括用户昵称、年龄、性别、头像、评论和点赞数等信息，并将数据保存到本地文件。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

import requests, json
from lxml import etree


class QiubaiSpider(object):

    # 初始化url
    def __init__(self):
        self.origin_url = "https://www.qiushibaike.com/hot/page/{}/"
        self.headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36"}

    # 因为这里发现url规律比较好找，可以事先构造url列表
    def get_url_list(self):
        url_list = [self.origin_url.format(page_num) for page_num in range(1, 13)]
        return url_list

    # 发送请求获取数据
    def parse_url(self, url):
        response = requests.get(url, headers=self.headers)
        return response.text

    # 提取数据
    def get_content_list(self, html_str):
        html = etree.HTML(html_str)
        # 经过分析发现，每一条数据都是存在一个div标签中的，所以这里按照div分组
        group_div_list = html.xpath("//div[@id='content-left']/div")
        content_list = []
        for div in group_div_list:
            content_dict = {}
            content_dict["user_name"] = div.xpath(".//h2/text()")[0].strip() if len(div.xpath("//h2/text()")) != 0 else None
            content_dict["user_age"] = div.xpath(".//div[@class='articleGender womenIcon']/text()")[0] if len(div.xpath(".//div[@class='articleGender womenIcon']/text()")) != 0 else None
            content_dict["user_gender"] = div.xpath(".//div[@class='author clearfix']/div/@class")[0] if len(div.xpath(".//div[@class='author clearfix']/div/@class")) != 0 else None
            content_dict["user_image"] = div.xpath(".//div[@class='author clearfix']/a/img/@src")
            content_dict["user_image"] = "https" + content_dict["user_image"][0] if len(div.xpath(".//div[@class='author clearfix']/a/img/@src")) != 0 else None
            content_dict["comments"] = div.xpath(".//div[@class='content']/span/text()") if len(div.xpath(".//div[@class='content']/span/text()")) != 0 else None
            content_dict["comments"] = [i.strip() for i in content_dict["comments"]]
            content_dict["stats-vote"] = div.xpath(".//div[@class='stats']/span[@class='stats-vote']/i/text()")[0] if len(div.xpath(".//div[@class='stats']/span[@class='stats-vote']/i/text()")) != 0 else None
            content_dict["stats-comments"] = div.xpath(".//div[@class='stats']/span[@class='stats-comments']//i/text()")[0] if len(div.xpath(".//div[@class='stats']/span[@class='stats-comments']//i/text()")) != 0 else None
            content_list.append(content_dict)
        return content_list

    # 保存数据
    def save_data(self,content_list):
        with open("qiubai.txt", "a", encoding="utf-8") as f:
            for content_dict in content_list:
                print(content_dict)
                f.write(json.dumps(content_dict, ensure_ascii=False))
                f.write("\n")
        print("保存成功")

    def run(self):
        # 1-获取url
        url_list = self.get_url_list()
        # 2-发送请求,获取数据
        for url in url_list:
            # 3-提取数据
            html_str = self.parse_url(url)
            content_list = self.get_content_list(html_str)
            # 4-保存
            self.save_data(content_list)


if __name__ == '__main__':
    qiubai = QiubaiSpider()
    qiubai.run()