【Python】多进程#181101-优快云博客

本文深入探讨了Python中的进程和线程概念，通过实例对比了它们的效率，并介绍了使用进程池进行并行处理的方法。同时，文章讲解了如何在多进程中实现共享内存，以及解决多进程间同步问题的策略。

2019独角兽企业重金招聘Python工程师标准>>>

进程

线程和进程的比较

import multiprocessing as mp
import threading as td

def job(a,d):
	print('')
t1 = td.Thread(target = job,args = (1,2))
p1 = mp.Process(target = job,args = (1,2))
t1.start()
p1.start()
t1.join()
p1.join()

创建进程

import multiprocessing as mp

def job(a,b):
	print("multiprocessing")

if __name__ == '__main__':
	p1 = mp.Process(target = job,args = (1,2))
	p1.start()
	p1.join()

D:\PythonTest>python test.py
multiprocessing

进程输出queue

import multiprocessing as mp

def job(q):
	res = 0
	for i in range(1000):
		res += i+i**2+i**3
	q.put(res)

if __name__ == '__main__':
	q = mp.Queue()
	p1 = mp.Process(target = job,args = (q,))
	p2 = mp.Process(target = job,args = (q,))
	p1.start()
	p2.start()
	p1.join()
	p2.join()
	res1 = q.get()
	res2 = q.get()
	print(res1 + res2)

D:\PythonTest>python test.py
499667166000

效率比较multithreading和multiprocessing

import multiprocessing as mp
import threading as td
import time

def job(q):
	res = 0
	for i in range(100000):
		res += i+i**2+i**3
	q.put(res)

def multicore():
	q = mp.Queue()
	p1 = mp.Process(target = job,args = (q,))
	p2 = mp.Process(target = job,args = (q,))
	p1.start()
	p2.start()
	p1.join()
	p2.join()
	res1 = q.get()
	res2 = q.get()
	print('multcore:',res1 + res2)

def normal():
	res = 0
	for _ in range(2):
		for i in range(100000):
			res += i+i**2+i**3
	print('normal',res)

def multithread():
	q = mp.Queue()
	t1 = td.Thread(target = job,args = (q,))
	t2 = td.Thread(target = job,args = (q,))
	t1.start()
	t2.start()
	t1.join()
	t2.join()
	res1 = q.get()
	res2 = q.get()
	print('multthread:',res1 + res2)

if __name__ == '__main__':
	st = time.time()
	normal()
	st1 = time.time()
	print('normal time:',st1 - st)
	multithread()
	st2 = time.time()
	print('multithread time:',st2 - st1)
	multithread()
	print('multithread time:',time.time() - st2)

D:\PythonTest>python test.py
normal 49999666671666600000
normal time: 0.16877007484436035
multthread: 49999666671666600000
multithread time: 0.37706947326660156
multthread: 49999666671666600000
multithread time: 0.33821940422058105

进程池pool

import multiprocessing as mp

def job(x):
	return x * x

def multicore():
	pool = mp.Pool()
	res = pool.map(job,range(10))
	print(res)

if __name__ == '__main__':
	multicore()

D:\PythonTest>python test.py
[0, 1, 4, 9, 16, 25, 36, 49, 64, 81]

import multiprocessing as mp

def job(x):
	return x * x

def multicore():
	pool = mp.Pool(processes = 3)
	res = pool.map(job,range(10))
	print(res)
	res = pool.apply_async(job,(2,))
	print(res.get())

if __name__ == '__main__':
	multicore()

D:\PythonTest>python test.py
[0, 1, 4, 9, 16, 25, 36, 49, 64, 81]
4

迭代
import multiprocessing as mp

def job(x):
	return x * x

def multicore():
	pool = mp.Pool(processes = 3)
	res = pool.map(job,range(10))
	print(res)
	res = pool.apply_async(job,(2,))
	print(res.get())
	multi_res = [pool.apply_async(job,(i,)) for i in range(10)]
	print([res.get() for res in multi_res])

if __name__ == '__main__':
	multicore()

D:\PythonTest>python test.py
[0, 1, 4, 9, 16, 25, 36, 49, 64, 81]
4
[0, 1, 4, 9, 16, 25, 36, 49, 64, 81]

共享内存shared memory

multiprocessing.Value()
multiprocessing.Array()

锁loop

import multiprocessing as mp
import time

def job(v,num):
	for _ in range(10):
		time.sleep(0.1)
		v.value += num
		print(v.value)

def multicore():
	v = mp.Value('i',0)
	p1 = mp.Process(target = job,args = (v,1))
	p2 = mp.Process(target = job,args = (v,3))
	p1.start()
	p2.start()
	p1.join()
	p2.join()

if __name__ == '__main__':
	multicore()

D:\PythonTest>python test.py
1
1
4
5
6
9
12
12
13
16
17
20
21
24
25
28
29
32
33
33

import multiprocessing as mp
import time

def job(v,num,l):
	l.acquire()
	for _ in range(10):
		time.sleep(0.1)
		v.value += num
		print(v.value)
	l.release()

def multicore():
	l = mp.Lock()
	v = mp.Value('i',0)
	p1 = mp.Process(target = job,args = (v,1,l))
	p2 = mp.Process(target = job,args = (v,3,l))
	p1.start()
	p2.start()
	p1.join()
	p2.join()

if __name__ == '__main__':
	multicore()

	
D:\PythonTest>python test.py
1
2
3
4
5
6
7
8
9
10
13
16
19
22
25
28
31
34
37
40

转载于:https://my.oschina.net/hellopasswd/blog/2396115