数据解析 bs4 爬取三国演义文章

import requests
from bs4 import BeautifulSoup

# 将本地html文档中的数据加载到该对象中
# fp = open("./template", "r", encoding="utf-8")
# soup = BeautifulSoup(fp, "lxml")

# 从互联网中获取页面源码数据加载到对象中
url = 'https://www.shicimingju.com/book/sanguoyanyi.html'
# 进行UA伪装
header = {
    "User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36'
}
response = requests.get(url=url, headers=header)
# 解决乱码问题的方案二：
response.encoding = 'UTF-8'

soup = BeautifulSoup(response.text, "lxml")
search_list = soup.select(".book-mulu > ul > li > a")
file = open("txtdir/sanguoyanyi.txt", "w", encoding="utf-8")
for i in search_list:
    page_name = i.text
    page_url = "https://www.shicimingju.com/" + i["href"]
    page_content = requests.get(url=page_url, headers=header)
    # 解决乱码问题的方案二：
    page_content.encoding = "UTF-8"

    detail_soup = BeautifulSoup(page_content.text, "lxml")
    detail_content = detail_soup.find('div', class_='chapter_content').text
    file.write(page_name + ":" + detail_content + "\n\n")
    print(page_name + "         爬取成功！！！！！！！！！")
file.close()