python 多线程下载图片示例

import os
import time
import pymysql
import requests
import threading
from concurrent.futures import ThreadPoolExecutor, as_completed

def db(sql):
    conn = pymysql.connect(host="xxx", port=3306, user='xxx', password=xxxx
                           database='xx')
    cursor = conn.cursor(cursor=pymysql.cursors.DictCursor)
    cursor.execute(sql)
    result = cursor.fetchall()  # [{},{}]
    car_len = len(result)
    cursor.close()
    conn.close()
    return result, car_len


def car_inout_sql():
    # 偏移基准日间
    sql = f"""SELECT id,base_photo2 FROM building_door WHERE deleted=0 AND area_id=19;"""
    return sql


count = 1
count_lock = threading.Lock()

def download(url, id):
    global count
    my_header = "https://xxxx/minio/"
    path = "./偏移告警基准/"
    # print(path)
    # count_lock.acquire()
    try:
        # r = requests.get(my_header + url, timeout=10)

        # 偏移
        if not url.startswith("http"): # 有前缀
            r = requests.get(my_header + url, timeout=10)
        else:
            r = requests.get(url, timeout=10)
        if r.status_code != 404:
            count_lock.acquire() # 加锁
            with open(path + str(id) + "_" + str(count) + ".jpg", 'wb') as f:
                f.write(r.content)

                count += 1
                count_lock.release() # 释放
                print(f""" {count}/{my_car_len}""")

    except Exception as e:
        print(e)
        count_lock.acquire()
        count += 1
        count_lock.release()


if __name__ == '__main__':
    st = time.time()
    my_sql = car_inout_sql()
    my_result, my_car_len = db(my_sql)  # [a,b,c] [{},{}]

    # 创建多个线程 方式一 线程数没有上限
    # threads = []
    #
    # for i in my_result:
    #     thread = threading.Thread(target=download, args=(i["photo"], i["id"]))
    #     threads.append(thread)
    #
    # # 开启所有线程
    # for thread in threads:
    #     thread.start()
    #
    # # 等待所有线程完成
    # for thread in threads:
    #     thread.join()
    # et=time.time()
    # print(et-st)
    
    # 创建多个线程 方式 二 固定线程数
    with ThreadPoolExecutor(max_workers=200) as executor:
        # 提交下载任务
        # futures = [executor.submit(download, i["photo"], i["id"]) for i in my_result]
        futures = [executor.submit(download, i["base_photo2"], i["id"]) for i in my_result]
        # 等待任务完成
        for future in as_completed(futures):
            pass
    et = time.time()
    print(et - st)

• 注意：
  count计数器需要被锁住，不然会造成count值重复，加锁的位置可能有待商榷；i["base_photo2"]是图片的url(可能会没有图片前缀),
  下载6400张图片时间从2000多秒提升700多秒