python example

最新推荐文章于 2022-12-05 00:16:03 发布

mydear_11000

最新推荐文章于 2022-12-05 00:16:03 发布

阅读量856

点赞数

CC 4.0 BY-SA版权

本文链接：https://blog.youkuaiyun.com/mydear_11000/article/details/52882978

本文介绍了一个使用Python多进程和队列实现的数据生产和消费模型示例。通过具体代码展示了如何利用多进程来提高文件读取效率，并确保数据正确处理。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

1 try multiprocess:

from threading import Thread
import time
import random
from Queue import Queue

queue = Queue(10)

class ProducerThread(Thread):
    def run(self):
        nums = range(5)
        global queue
        while True:
            num = random.choice(nums)
            queue.put(num)
            print "Produced", num
            time.sleep(random.random())

class ConsumerThread(Thread):
    def run(self):
        global queue
        while True:
            num = queue.get()
            queue.task_done()
            print "Consumed", num
            time.sleep(random.random())

ProducerThread().start()
ConsumerThread().start()

2 try queue

from multiprocessing import Queue, Process, Event
import os
import sys
import time
import Queue as _queue

def producer(file, queue, quitevent):
    flen = os.path.getsize(file)
    with open(file) as f:
        while 1:
            if f.tell() >= flen:
                f.seek(0, 0)
            line = f.readline()
            while 1:
                try:
                    queue.put_nowait(line)
                    break
                except _queue.Full:
                    if quitevent.is_set():
                        print("Loader will quit ...")
                        return
                    time.sleep(0.001)
                    continue

if __name__ == "__main__":
    filenm = sys.argv[1]
    startT = time.time()
    queue = Queue(5)
    quitevent = Event()

    if not os.path.exists(filenm):
        raise Exception("Cant find file {}!".format(filenm))

    prod = Process(name = "Loader"
                   , target = producer
                   , args = (filenm, queue, quitevent))

    prod.start()
    while 1:
        endT = time.time()
        if endT - startT > 10:
            print("Set quitevent and break...")
            quitevent.set()
            break
        line = queue.get()
        print(line)
        time.sleep(1)

    prod.join()