Python学习笔记——20170829

最新推荐文章于 2024-12-29 11:51:07 发布

蒋含竹

最新推荐文章于 2024-12-29 11:51:07 发布

阅读量435

点赞数

CC 4.0 BY-SA版权

分类专栏： Python 文章标签： python HTMLParser 爬虫豆瓣

本文链接：https://blog.youkuaiyun.com/alionsss/article/details/77731637

Python 专栏收录该内容

39 篇文章

订阅专栏

本文介绍了一种使用Python中的HTMLParser模块抓取豆瓣电影信息的方法。通过定义特定的解析器类，可以有效地从网页中提取电影标题、评分、导演及演员等信息。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

HTMLParser

方法注解

from html.parser import HTMLParser
#
class MyParser(HTMLParser):
    """
    HTMLParser
    """

    def __init__(self):
        HTMLParser.__init__(self)

    def handle_startendtag(self, tag, attrs):
        super().handle_startendtag(tag, attrs)

    # 处理开始标签<a>
    def handle_starttag(self, tag, attrs):
        pass

    # 处理结束标签</a>
    def handle_endtag(self, tag):
        pass

    # 处理特殊字符串，例如$#开头的
    def handle_charref(self, name):
        pass

    # 处理标签中的内容，比如<a href="http://www.baidu.com">baidu<a>
    def handle_data(self, data):
        pass

    # 处理注释
    def handle_comment(self, data):
        pass

    # 处理以<!开头的，比如<!DOCTYPE HTML>
    def handle_decl(self, decl):
        pass

    # 处理特殊字符，例如&nbsp
    def handle_entityref(self, name):
        pass

    # 处理<?instruction>
    def handle_pi(self, data):
        pass

Douban电影内容爬取

import requests
from html.parser import HTMLParser
#
class MovieParser(HTMLParser):
    """
    电影解析器
    """

    def __init__(self):
        HTMLParser.__init__(self)
        self.moives = []

    def handle_starttag(self, tag, attrs):

        def _attr(attrList, attrName):
            for attr in attrList:
                if attr[0] == attrName:
                    return attr[1]
            return None

        if tag == 'li' and _attr(attrs, 'data-title'):
            movie = {}
            movie['title'] = _attr(attrs, 'data-title')
            movie['score'] = _attr(attrs, 'data-score')
            if movie['score'] is None:
                movie['score'] = "None"
            movie['director'] = _attr(attrs, 'data-director')
            movie['actors'] = _attr(attrs, 'data-actors')
            self.moives.append(movie)
            # print('{movie[title]} | {movie[score]} | {movie[director]} | {movie[actors]}'.format(movie=movie))

    def error(self, message):
            pass
#
#
def my_movies(url):
    """
    网络请求
    :param url: 地址
    :return: 解析好的内容
    """

    mp = None
    try:
        headers = {'User-Agent': 'Mozilla/5.0 (compatible; MSIE 6.0; Windows NT 5.1;SV1)'}
        response = requests.get(url, headers=headers)
        response.raise_for_status()
        mp = MovieParser()
        mp.feed(response.text)
        return mp.moives
    except:
        return print('发生异常')
    finally:
        if mp is not None:
            mp.close()
#
#
def save_file(path, text):
    """
    文本存储
    :param path: 存储路径
    :param text: 文本内容
    :return: None
    """

    with open(path, 'w', encoding='UTF-8') as file:
        file.write(text)
#
#
if __name__ == '__main__':
    url = "https://movie.douban.com/cinema/nowplaying/chongqing/"
    text = my_movies(url)
    save_file("d:/upload/movies.json", str(text).replace('\'', '\"'))