scrapy_novel_python

最新推荐文章于 2024-09-07 10:22:27 发布

原创最新推荐文章于 2024-09-07 10:22:27 发布 · 134 阅读

0 ·

CC 4.0 BY-SA版权

python爬虫专栏收录该内容

4 篇文章

订阅专栏

本文介绍了一个使用Python Scrapy框架实现的小说爬虫项目。该爬虫能够从指定网站抓取《一念永恒》这部小说的所有章节链接及内容，并将其保存到本地文件中。通过分析网页结构并利用BeautifulSoup进行解析，爬虫实现了自动化下载。

scrapy_novel_python

 1 # _*_ coding:UTF _8_
 2 from bs4 import BeautifulSoup
 3 import requests,sys
 4 class downloader(object):
 5     def __init__(self):
 6         self.server = 'http://www.biqukan.com/'
 7         self.target = 'http://www.biqukan.com//1_1094/'
 8         self.names = []
 9         self.urls = []
10         self.nums = 0
11     #获取下载链接
12     def get_download_url(self):
13         reg = requests.get(url = self.target)
14         html = reg.text
15         div_bf = BeautifulSoup(html)
16         div = div_bf.find_all('div', class_ = 'listmain')
17         a_bf = BeautifulSoup(str(div[0]))
18         a = a_bf.find_all('a')
19         self.nums = len(a[15:])
20         for each in a[15:]:
21             self.names.append(each.string)
22             self.urls.append(self.server + each.get('href'))
23     #获取章节内容
24     def get_contents(self,target):
25         req = requests.get(url = target)
26         html = req.text
27         bf = BeautifulSoup(html)
28         texts = bf.find_all('div',class_ = 'showtxt')
29         texts = texts[0].text.replace('\xa0'*8,'\n\n')
30         return texts
31     #写入文件
32     def writer(self,name,path,text):
33         write_flag = True
34         with open(path,'a',encoding='utf-8') as f:
35             f.write(name + '\n')
36             f.writelines(text)
37             f.write('\n\n')
38 if __name__ == '__main__':
39     dl = downloader()
40     dl.get_download_url()
41     print('《一念永恒》开始下载：')
42     for i in range(dl.nums):
43         dl.writer(dl.names[i],'一念永恒.txt',dl.get_contents(dl.urls[i]))
44         sys.stdout.write("已下载：%.3f%%" % float(i/dl.nums) + '\r')
45         sys.stdout.flush()
46     print('《一念永恒》下载完成')

posted @ 2018-03-22 12:54 Edge_of_Eternity 阅读(...) 评论(...) 编辑收藏