2021-03-24

爬取斗鱼并制图

from selenium import webdriver
import time
import matplotlib.pyplot as plt
import jieba
from wordcloud import WordCloud
from pyecharts.charts import Bar
from pyecharts import options as opts


class DouYuSpider():

    def __init__(self):
        self.url = 'https://www.douyu.com/g_LOL'
        self.driver = webdriver.Chrome(executable_path='F:\chromedriver.exe')
        self.dict = {"大于100w": 0, "50w-100w": 0, "30w-50w": 0, "10w-30w": 0, "1w-10w": 0, "小于1w": 0}
        self.titles = ''

        self.page = 1

    def send_request(self, url):
        self.driver.get(url)

    def parse_content(self):
        titles = self.driver.find_elements_by_css_selector('.DyListCover-intro')  # 取出标题
        hots = self.driver.find_elements_by_css_selector('.DyListCover-hot')  # 热度
        authors = self.driver.find_elements_by_css_selector('.DyListCover-userName')  # 热度
        for t in zip(titles, hots, authors):
            print(t[0].text, t[1].text, t[2].text)
            title = t[0].text
            hot = t[1].text
            if "万" in hot:
                hot = float(hot.split('万')[0]) * 10000
            else:
                hot = float(hot)
            self.cal_hot(hot)
            self.titles += title

    def create_word_cloud(self, word):
        word_list = list(jieba.cut(word))
        word_list = [word for word in word_list if len(word) > 1]  # 清洗数据
        wc = WordCloud(width=800, height=800, font_path='Fonts/font1.ttf')
        wc.generate(" ".join(word_list))
        wc.to_file('douyu.png')

    def cal_hot(self, hot):
        if hot >= 1000000:
            self.dict['大于100w'] = self.dict['大于100w'] + 1
        elif hot >= 500000 and hot < 1000000:
            self.dict['50w-100w'] = self.dict['50w-100w'] + 1
        elif hot >= 300000 and hot < 500000:
            self.dict['30w-50w'] = self.dict['30w-50w'] + 1
        elif hot >= 100000 and hot < 300000:
            self.dict['10w-30w'] = self.dict['10w-30w'] + 1
        elif hot >= 10000 and hot < 100000:
            self.dict['1w-10w'] = self.dict['1w-10w'] + 1
        else:
            self.dict['小于1w'] = self.dict['小于1w'] + 1

    def paint_table(self):
        bar = (
            Bar()
                .add_xaxis([str(key) for key in self.dict.keys()])
                .add_yaxis("斗鱼热度分析", list(self.dict.values()))
                .set_global_opts(title_opts=opts.TitleOpts(title="斗鱼热度分析"))
        )
        bar.render()

    def start(self):
        self.send_request(self.url)
        while True:
            time.sleep(1)
            self.driver.execute_script("window.scrollTo(0,document.body.scrollHeight)")  # selenium 支持js
            self.parse_content()
            time.sleep(1)
            next = self.driver.find_element_by_class_name('dy-Pagination-next')
            if next.get_attribute('aria-disabled') == 'false':
                next.click()
                # self.page += 1
                # if self.page == 2:
                #     self.paint_table()  # 制图
                #     self.create_word_cloud(self.titles)  # 绘制词云
                #     break
            else:
                self.paint_table()  # 制图
                self.create_word_cloud(self.titles)  # 绘制词云
                break


if __name__ == '__main__':
    dys = DouYuSpider()
    dys.start()