Python 多线程和多进程高级应用指南
1. 多线程基础
多线程是 Python 中一种并发编程方式,它允许我们在同一进程中运行多个线程。
import threading
import time
def worker():
print(f"Worker thread started: {threading.current_thread().name}")
time.sleep(1)
print(f"Worker thread finished: {threading.current_thread().name}")
# 创建线程
threads = []
for i in range(3):
t = threading.Thread(target=worker, name=f"Thread-{i}")
threads.append(t)
t.start()
# 等待所有线程完成
for t in threads:
t.join()
print("All threads finished")
2. 多进程基础
多进程是 Python 中另一种并发编程方式,它允许我们在不同的进程中运行代码,充分利用多核 CPU。
import multiprocessing
import time
def worker():
print(f"Worker process started: {multiprocessing.current_process().name}")
time.sleep(1)
print(f"Worker process finished: {multiprocessing.current_process().name}")
# 创建进程
processes = []
for i in range(3):
p = multiprocessing.Process(target=worker, name=f"Process-{i}")
processes.append(p)
p.start()
# 等待所有进程完成
for p in processes:
p.join()
print("All processes finished")
3. 线程安全
3.1 锁机制
import threading
import time
counter = 0
lock = threading.Lock()
def increment():
global counter
with lock:
current = counter
time.sleep(0.1) # 模拟计算时间
counter = current + 1
# 创建线程
threads = []
for i in range(10):
t = threading.Thread(target=increment)
threads.append(t)
t.start()
# 等待所有线程完成
for t in threads:
t.join()
print(f"Final counter value: {counter}")
3.2 信号量
import threading
import time
# 限制同时运行的线程数
semaphore = threading.Semaphore(2)
def worker(name):
with semaphore:
print(f"Worker {name} started")
time.sleep(1)
print(f"Worker {name} finished")
# 创建线程
threads = []
for i in range(5):
t = threading.Thread(target=worker, args=(i,))
threads.append(t)
t.start()
# 等待所有线程完成
for t in threads:
t.join()
print("All workers finished")
4. 进程间通信
4.1 队列
import multiprocessing
import time
def producer(queue):
for i in range(5):
item = f"Item {i}"
queue.put(item)
print(f"Produced: {item}")
time.sleep(0.5)
def consumer(queue):
while True:
item = queue.get()
if item is None:
break
print(f"Consumed: {item}")
time.sleep(1)
# 创建队列
queue = multiprocessing.Queue()
# 创建进程
producer_process = multiprocessing.Process(target=producer, args=(queue,))
consumer_process = multiprocessing.Process(target=consumer, args=(queue,))
# 启动进程
producer_process.start()
consumer_process.start()
# 等待生产者完成
producer_process.join()
# 发送结束信号
queue.put(None)
# 等待消费者完成
consumer_process.join()
print("All processes finished")
4.2 管道
import multiprocessing
import time
def sender(conn):
for i in range(5):
item = f"Message {i}"
conn.send(item)
print(f"Sent: {item}")
time.sleep(0.5)
conn.close()
def receiver(conn):
while True:
try:
item = conn.recv()
print(f"Received: {item}")
except EOFError:
break
# 创建管道
parent_conn, child_conn = multiprocessing.Pipe()
# 创建进程
sender_process = multiprocessing.Process(target=sender, args=(child_conn,))
receiver_process = multiprocessing.Process(target=receiver, args=(parent_conn,))
# 启动进程
sender_process.start()
receiver_process.start()
# 等待进程完成
sender_process.join()
receiver_process.join()
print("All processes finished")
5. 线程池和进程池
5.1 线程池
import concurrent.futures
import time
def worker(num):
print(f"Worker {num} started")
time.sleep(1)
print(f"Worker {num} finished")
return num * 2
# 创建线程池
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor:
# 提交任务
futures = [executor.submit(worker, i) for i in range(5)]
# 获取结果
for future in concurrent.futures.as_completed(futures):
result = future.result()
print(f"Result: {result}")
print("All tasks completed")
5.2 进程池
import concurrent.futures
import time
def worker(num):
print(f"Worker {num} started")
time.sleep(1)
print(f"Worker {num} finished")
return num * 2
# 创建进程池
with concurrent.futures.ProcessPoolExecutor(max_workers=3) as executor:
# 提交任务
futures = [executor.submit(worker, i) for i in range(5)]
# 获取结果
for future in concurrent.futures.as_completed(futures):
result = future.result()
print(f"Result: {result}")
print("All tasks completed")
6. 实际应用场景
6.1 I/O 密集型任务
import concurrent.futures
import requests
import time
urls = [
"https://www.google.com",
"https://www.baidu.com",
"https://www.github.com",
"https://www.stackoverflow.com",
"https://www.python.org"
]
def fetch_url(url):
print(f"Fetching {url}")
response = requests.get(url)
return f"{url}: {response.status_code}"
# 使用线程池处理 I/O 密集型任务
start_time = time.time()
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
results = list(executor.map(fetch_url, urls))
end_time = time.time()
print(f"Time taken: {end_time - start_time:.2f} seconds")
print("Results:")
for result in results:
print(result)
6.2 CPU 密集型任务
import concurrent.futures
import time
def compute_intensive_task(n):
print(f"Computing for {n}")
result = 0
for i in range(n):
result += i * i
return result
# 使用进程池处理 CPU 密集型任务
start_time = time.time()
with concurrent.futures.ProcessPoolExecutor(max_workers=4) as executor:
results = list(executor.map(compute_intensive_task, [10000000] * 4))
end_time = time.time()
print(f"Time taken: {end_time - start_time:.2f} seconds")
print("Results:")
for result in results:
print(result)
7. 最佳实践
- 选择合适的并发模型:对于 I/O 密集型任务,使用线程池;对于 CPU 密集型任务,使用进程池。
- 避免共享状态:尽量避免在多线程或多进程之间共享状态,使用队列或管道进行通信。
- 使用锁机制:当需要共享状态时,使用锁机制确保线程安全。
- 设置合理的池大小:根据系统资源和任务类型设置合理的线程池或进程池大小。
- 处理异常:在并发任务中正确处理异常,避免进程或线程崩溃。
- 使用
with语句:使用with语句管理线程池和进程池,确保资源被正确释放。 - 测试并发性能:测试不同并发策略的性能,选择最优方案。
8. 总结
多线程和多进程是 Python 中强大的并发编程工具,它们允许我们充分利用系统资源,提高程序的运行效率。通过掌握多线程和多进程的高级应用,我们可以编写更加高效、响应迅速的代码。
在实际应用中,我们可以根据任务类型选择合适的并发模型:对于 I/O 密集型任务,使用线程池;对于 CPU 密集型任务,使用进程池。同时,我们需要注意线程安全和进程间通信等问题,确保程序的正确性和稳定性。
希望本文对你理解和应用 Python 多线程和多进程有所帮助!
334



被折叠的 条评论
为什么被折叠?



