「python爬虫之路day15」:多线程爬取低俗段子（来看看你的fzl时代！）

今天学习了爬百思不得姐姐的段子，其实有点懵，也觉得自己技术之路的遥遥，所以记录一下，然后准备打王者荣耀。
来看代码：

import requests
from lxml import etree
import csv
import threading
from queue import Queue
class BSSpider(threading.Thread):
    headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.87 Safari/537.36"}
    def __init__(self,page_queue,joke_queue,*args,**kwargs):
        super(BSSpider,self).__init__(*args,**kwargs)
        self.base_domain='http://www.budejie.com'
        self.page_queue=page_queue
        self.joke_queue=joke_queue
    def run(self):
        while True:
            if self.page_queue.empty():
                break
            url=self.page_queue.get()
            response=requests.get(url,headers=self.headers)
            text=response.text
            html=etree.HTML(text)
            descs=html.xpath('//div[@class="j-r-list-c-desc"]')
            for desc in descs:
                jokes=desc.xpath(".//text()")
                joke="\n".join(jokes).strip()#各项链接，返回字符串
                link=self.base_domain+desc.xpath(".//a/@href")[0]
                self.joke_queue.put((joke,link))
            print("*"*30+"第%s页下载完成！"%url.split('/')[-1]+"*"*30)

class BSWriter(threading.Thread):
    def __init__(self,gLock,writer,joke_queue,*args,**kwargs):
        super(BSWriter, self).__init__(*args, **kwargs)
        self.writer = writer
        self.joke_queue = joke_queue
        self.lock=gLock
    def run(self):
        while True:
            try:
                joke_info = self.joke_queue.get(timeout=4)
                joke,link = joke_info
                self.lock.acquire()
                self.writer.writerow((joke,link))
                self.lock.release()
                print('保存一条')
            except:
                break

def main():
    page_queue=Queue(10)
    joke_queue=Queue(500)
    gLock=threading.Lock()
    fp=open('bsbs.csv','a',newline='',encoding='utf-8')
    writer=csv.writer(fp)
    writer.writerow(('content','link'))#写入标签
    for x in range(1,11):
        url='http://www.budejie.com/text/%d'%x
        page_queue.put(url)
    for x in range(5):
        t=BSSpider(page_queue,joke_queue)
        t.start()
    for x in range(5):
        t=BSWriter(gLock,writer,joke_queue)
        t.start()
if __name__ == '__main__':
    main()

来看我们保存的段子咯：

现在看看当年一直看的东西感觉索然无味，其实并不是它们变了，而是我们出现了其的替代品而且我们的要求越来越高了。