python multiprocessing加锁

from multiprocessing import Pool,Manager,Process
from functools import partial
from tqdm import tqdm

def test(count,parentindex,d,lock):
    with lock:
        # 计算总和,加锁
        eachindex = d[str(parentindex)]
        eachindex['total_count'] = eachindex['total_count'] + count
        d[str(parentindex)] = eachindex

def main(index,d,lock):
    d[str(index)] = {
        "parentindex":index,
        "total_count":0
    }
    
    with Pool(processes=4) as p, tqdm(total=10, leave=True) as pbar:
        # 预先给参数
        func = partial(test, parentindex=index,d=d,lock = lock)
        # 注意这个参数是batch_query_ck_and_export_csv的第一个参数,即count参数
        for _ in p.imap(func, range(0,10)):
            pbar.update()
            pbar.refresh()
    

if __name__ =='__main__':
    processes = []
    with Manager() as manager:
        d = manager.dict()
        lock = manager.Lock()
        with tqdm(total=8,desc='main process') as bar:
            for i in range(8):
                p = Process(target=main,args=(i,d,lock))
                p.start()
                processes.append(p)
                if len(processes)>=4:
                    for p in processes:
                        p.join()  # 等待当前批次的进程完成
                bar.update()
                bar.refresh()
            for p in processes:
                p.join() 
        
        for key,value in d.items():
            print(f"key {key},value {value}")
    
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值