多线程下载文件dome

最新推荐文章于 2025-08-05 17:20:40 发布

原创最新推荐文章于 2025-08-05 17:20:40 发布 · 73 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#python

由于未提供博客具体内容，无法给出包含关键信息的摘要。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

import os
import threading
import requests
import time

# 线程数
thread_num = 10
name = 0
cur_path = os.path.dirname(os.path.realpath(__file__))
log_path = os.path.join(cur_path, 'output')
if not os.path.exists(log_path): os.mkdir(log_path)


def download(img_url):
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0"
    }
    response = requests.get(url=img_url, headers=headers, timeout=5, stream=True)
    global name
    name = name + 1
    with open("./output/%d.jpg" % name, "wb") as f:
        f.write(response.content)
        time.sleep(0.01)


def get_imgurl_generate():
    with open('./url_new.txt', 'r') as f:
        for line in f:
            line = line.strip()
            if len(line) > 5:
                yield line
            else:
                continue


lock = threading.Lock()


def loop(imgs):
    print('thread %s is running...' % threading.current_thread().name)

    while True:
        try:
            with lock:
                # img_url, img_name = next(imgs)
                img_url = next(imgs)
        except StopIteration:
            break
        try:
            download(img_url)
        except:
            print('exceptfail\t%s' % img_url)
        # download(img_url)
    print('thread %s is end...' % threading.current_thread().name)


img_gen = get_imgurl_generate()

for i in range(0, thread_num):
    t = threading.Thread(target=loop, name='LoopThread %s' % i, args=(img_gen,))
    t.start()