from multiprocessing import Pool,Manager,Process
from functools import partial
from tqdm import tqdm
def test(count,parentindex,d,lock):
with lock:
# 计算总和,加锁
eachindex = d[str(parentindex)]
eachindex['total_count'] = eachindex['total_count'] + count
d[str(parentindex)] = eachindex
def main(index,d,lock):
d[str(index)] = {
"parentindex":index,
"total_count":0
}
with Pool(processes=4) as p, tqdm(total=10, leave=True) as pbar:
# 预先给参数
func = partial(test, parentindex=index,d=d,lock = lock)
# 注意这个参数是batch_query_ck_and_export_csv的第一个参数,即count参数
for _ in p.imap(func, range(0,10)):
pbar.update()
pbar.refresh()
if __name__ =='__main__':
processes = []
with Manager() as manager:
d = manager.dict()
lock = manager.Lock()
with tqdm(total=8,desc='main process') as bar:
for i in range(8):
p = Process(target=main,args=(i,d,lock))
p.start()
processes.append(p)
if len(processes)>=4:
for p in processes:
p.join() # 等待当前批次的进程完成
bar.update()
bar.refresh()
for p in processes:
p.join()
for key,value in d.items():
print(f"key {key},value {value}")
07-31
1481
