爬虫实战——爬取百思不得姐的段子

from queue import Queue
import requests
import threading
from lxml import etree
import csv
class Productor(threading.Thread):
    Base_url="http://www.budejie.com"
    headers={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36"}
    def __init__(self,page_url,joke_url,*arg,**kwargs):
        super(Productor,self).__init__(*arg,**kwargs)
        self.page_url=page_url
        self.joke_url=joke_url

    def run(self):

        while True:

            if self.page_url.empty():
                break
            url = self.page_url.get()
            #self.parse_url(url)
            response = requests.get(url, headers=self.headers)
            text = response.text
            html = etree.HTML(text)
            contents = html.xpath("//div[@class='j-r-list-c-desc']")
            for content in contents:
                jokes = content.xpath(".//text()")
                # print(jokes)
                joke = "\n".join(jokes).strip()
                # print(joke)
                link = self.Base_url + content.xpath(".//a/@href")[0]
                print(link)
                self.joke_url.put((joke, link))
            print('=' * 30 + "第%s页下载完成！" % url.split('/')[-1] + "=" * 30)

    # def parse_url(self,url):
        # response=requests.get(url,headers=self.headers)
        # text=response.text
        # html=etree.HTML(text)
        # contents=html.xpath("//div[@class='j-r-list-c-desc']")
        # for content in contents:
        #     jokes=content.xpath(".//text()")
        #     #print(jokes)
        #     joke="\n".join(jokes).strip()
        #     #print(joke)
        #     link=self.Base_url+content.xpath(".//a/@href")[0]
        #     print(link)
        #     self.joke_url.put((joke,link))
        # print('='*30+"第%s页下载完成！"%url.split('/')[-1]+"="*30)

class Costumer(threading.Thread):
    def __init__(self, joke_queue,writer,gLock ,*arg, **kwargs):
        super(Costumer, self).__init__(*arg, **kwargs)
        self.joke_queue = joke_queue
        self.writer = writer
        self.lock = gLock
    # def run(self):
    #     while True:
    #         if self.page_url.empty() and self.joke_url.empty():
    #             break
    #         joke,url=self.joke_url.get(timeout=40)
    #         self.lock.acquire()
    #         self.writer.writerow((joke,url))
    #         self.lock.release()
    #         print("保存1条")
    def run(self):
        while True:
            try:
                joke_info = self.joke_queue.get(timeout=40)
                joke, link = joke_info
                self.lock.acquire()
                self.writer.writerow((joke, link))
                self.lock.release()
                print('保存一条')
            except:
                break





def main():
    page_url=Queue(100)
    joke_url=Queue(500)
    gLock=threading.Lock()
    fp = open('bsbdj.csv', 'a', newline='', encoding='utf-8')
    writer = csv.writer(fp)
    writer.writerow(('content', 'link'))
    for i in range(1,11):
        url="http://www.budejie.com/text/%d"%i
        page_url.put(url)

    for i in range(5):
        t=Productor(page_url,joke_url)
        t.start()

    for i in range(5):
        t = Costumer(joke_url,writer,gLock)
        t.start()


if __name__=="__main__":
    main()

使用生产者消费者模式，并存到csv中