爬虫----配合多线程的思路

最新推荐文章于 2025-09-10 17:05:30 发布

转载最新推荐文章于 2025-09-10 17:05:30 发布 · 70 阅读

0 ·

CC 4.0 BY-SA版权

原文链接：http://www.cnblogs.com/saintdingspage/p/10582296.html

文章标签：

#爬虫

本文介绍了一个使用 Python 实现的小说解析器，该解析器能够从指定文件夹读取 HTML 格式的小说章节，并通过 PyQuery 提取标题和正文内容，然后将内容保存为文本文件。解析器采用多线程方式提高处理效率。

from pyquery import PyQuery as pq
import os
from queue import Queue
from threading import Thread
class txtparser(Thread):
    def __init__(self,queue):
        Thread.__init__(self)
        self.queue = queue
        #文件夹目录

    def run(self):
        #path = "E:\辰东\ZheTian\\395020.html"
        while True:
            content = self.queue.get()
            html=""
            try:
                with open (content,"r",encoding='utf-8') as reader:
                     html = reader.read()
            except Exception:
                with open (content,"r") as reader:
                     html = reader.read()
            #print(html)
            try:
                doc = pq(html)
                title = doc("#main .content_read .box_con .bookname h1")
                print("标题=====",title.text())
                clipname = content.split("\\")[-2]
                #junkp = doc(".content").find('p').remove()
                passage = doc("#content").text()
            except Exception:
                continue
            print("正文======",str.replace(passage,"<br/>",""))
            try:
                clipname = str.replace(clipname,"《","")
                clipname = str.replace(clipname,"》","")
            except Exception:
                clipname = clipname
            if os.path.exists(clipname):
                pass
            else:
                os.mkdir(clipname)
            try:
                with open(clipname+"\\"+title.text()+".txt","w",encoding="gbk") as writer:
                    writer.write(passage)
                print("完成{}的写入".format(clipname+"\\"+title.text()+".txt"))
            except Exception:
                with open("errorecorder.log","a") as writer:
                    writer.write(clipname+"\\"+title.text()+".txt"+"\r")
            print("文件夹名称======",clipname)

def launchtxtparser(parentdir):
    rootdir = parentdir
    queue = Queue()
    print(rootdir)
    for i in os.listdir(rootdir):
        print(i)
        if os.path.isdir(rootdir+"\\"+i):
            print(rootdir+"\\"+i)
            g = (k for k in os.listdir(rootdir+"\\"+i))
            print(next(g))
            while True:
                 try:
                     filename = next(g)
                     fullfilename = rootdir+"\\"+i+"\\"+filename
                     queue.put(fullfilename)
                     print(fullfilename)
                 except StopIteration:
                     print("ooooophs~处理完毕")
                     break
    for i in range(10):
        cpc = txtparser(queue)
        cpc.daemon=True
        cpc.start()
    queue.join()
#print(os.listdir(rootdir))
launchtxtparser("E:\月关")