爬取河科院各个新闻内容，然后分词处理，取出来最高的前十个词（作业）

爬取新闻内容

import urllib.request
from bs4 import BeautifulSoup
response = urllib.request.urlopen('https://news.hist.edu.cn/kyyw/378.htm')
#print(response.read().decode('utf-8'))
content=response.read().decode('utf-8')
soup = BeautifulSoup(content, 'html.parser', from_encoding='utf-8')
# lis=soup.find_all('li')
# for li in lis:
# print(li)
divs=soup.find_all('div',{'class':"sec-a"})
# print(divs)
lis=divs[0].find_all('li')
num = 1

# 获取新闻内容
def content(href,title):
    response = urllib.request.urlopen('https://news.hist.edu.cn/' + href)
    content = response.read().decode('utf-8')
    soup = BeautifulSoup(content, 'html.parser', from_encoding='utf-8')
    divs = soup.find_all('div', {'id': "vsb_content_501"})
    # print(divs)
    if len(divs) == 0:
        return
    ps = divs[0].find_all('p')
	# 写入news目录下 
    file = "./news/" +title + ".txt"
    for p in ps:
        with open(file,'a',encoding='utf-8') as f:
            f.write(p.text + '\n')



for li in lis:
    href = li.find_all('a')[0].get("href")
    if len(href) == 0:
        continue
    title = li.find_all('a')[0].get("title")
    // 一个一个网页爬取
    content(href[2:],title)

分词处理

import jieba
#
# str="好好学习，天天向上"
# print("/".join(jieba.lcut(str)))
#
import os

ciyu ={}

rootdir = './news/'
list = os.listdir(rootdir)  # 列出文件夹下所有的目录与文件
for i in range(0, len(list)):
    path = os.path.join(rootdir, list[i])
    # if os.path.isfile(path):
    # 遍历所有文件
    with open(path,'r',encoding='utf-8') as f:
        data = f.readlines()
        list1 = jieba.lcut(str(data))
        for i in list1:
        	# 去掉小于等于1个的词
            if len(i) <= 1:
                continue
            if ciyu.get(i) == None:
                ciyu[i] = 1
            else:
                ciyu[i] += 1

# 排序
sort_ciyu = sorted(ciyu.items(),key=lambda x:x[1],reverse=True)[:10]
print(sort_ciyu)