普通模式与多线程模式之爬取斗图拉网表情对比

普通模式：

import requests
import re
import urllib
import os

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36'}


def image_get(url):
    resp = requests.get(url=url, headers=HEADERS)
    text = resp.text
    image_url = re.findall(r'<a\sclass="col-xs-6 col-sm-3".*?data-original=\"(.*?)\"\salt', text, re.S)
    titles = re.findall(r'<a\sclass="col-xs-6 col-sm-3".*?alt=\"(.*?)\"\sclass', text, re.S)
    for index, url in enumerate(image_url):
        pure_title = re.sub(r"[\?？\.，。！\!]", '', titles[index])
        tail = os.path.splitext(url)[1]
        file_name = pure_title + tail
        print(file_name)
        urllib.request.urlretrieve(url,'image/'+file_name)


def url_make():
    for x in range(1, 101):
        url = 'https://www.doutula.com/photo/list/?page=%d' % x
        image_get(url)


if __name__ == '__main__':
    url_make()

多线程模式：

import requests
import re
import urllib
import os
import threading
from queue import Queue

HEADERS = {
‘User-Agent’: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36’}

class producer(threading.Thread):
def init(self, page_queue, image_queue, *args, **kwargs):
super(producer, self).init(*args, **kwargs)
self.page_queue = page_queue
self.image_queue = image_queue

def run(self):
    while True:
        if self.page_queue.empty():
            break
        image_url = self.page_queue.get()
        self.image_url_get(image_url)

def image_url_get(self, url):
    resp = requests.get(url=url, headers=HEADERS)
    text = resp.text
    image_url = re.findall(r'<a\sclass="col-xs-6 col-sm-3".*?data-original=\"(.*?)\"\salt', text, re.S)
    titles = re.findall(r'<a\sclass="col-xs-6 col-sm-3".*?alt=\"(.*?)\"\sclass', text, re.S)
    for index, url in enumerate(image_url):
        pure_title = re.sub(r"[\?？\.，。！\!]", '', titles[index])
        tail = os.path.splitext(url)[1]
        file_name = pure_title + tail
        self.image_queue.put((url, file_name))

class consumer(threading.Thread):
def init(self, page_queue, image_queue, *args, **kwargs):
super(consumer, self).init(*args, **kwargs)
self.page_queue = page_queue
self.image_queue = image_queue

def run(self):
    while True:
        image_url, file_name = self.image_queue.get()
        urllib.request.urlretrive(image_url, 'image/' + file_name)
        print("%s下载成功"%file_name)
        if self.image_queue.empty() and self.page_queue.empty():
            break

def main():
page_queue = Queue(100)
image_queue = Queue(1000)
for x in range(1, 101):
url = ‘https://www.doutula.com/photo/list/?page=%d’ % x
print(url)
page_queue.put(url)

for x in range(5):
    t = producer(page_queue, image_queue)
    t.start()

for x in range(5):
    t = consumer(page_queue, image_queue)
    t.start()

if name == ‘main’:
main()