【代码】多队列处理器

import time
from concurrent.futures.thread import ThreadPoolExecutor
from queue import Queue, Empty
from threading import Event, Lock, Condition


class QueueSet:
    """
    unique tasks =>q1=> process =>q2=> process... =>qn=> join()
    """

    def __init__(self, size, queue_size=0):
        self.queue_set = [Queue(queue_size) for _ in range(size)]
        self.size = size
        self.dirty = set()
        self.processing = 0
        self.mutex = Lock()
        self.finished = Condition(Lock())

    def get(self, index, block=True, timeout=None):
        item = self.queue_set[index].get(block, timeout)
        if index == 0:
            with self.mutex:
                self.dirty.remove(item)
        if index == self.size - 1:
            with self.finished:
                self.processing += 1
        return item

    def put(self, index, item):
        if index == 0:
            with self.mutex:
                if item in self.dirty:
                    return
                self.dirty.add(item)
        self.queue_set[index].put(item)

    def done(self):
        with self.finished:
            unfinished = self.processing - 1
            if unfinished <= 0:
                if unfinished < 0:
                    raise ValueError('done() called too many times')
                self.finished.notify_all()
            self.processing = unfinished

    def interrupt(self):
        for i, q in enumerate(self.queue_set):
            while not q.empty():
                try:
                    self.get(index=i, block=False, timeout=None)
                    if i == self.size - 1:
                        self.done()
                except Empty:
                    break

    def join(self):
        with self.finished:
            while self.processing or sum(q.qsize() for q in self.queue_set):
                self.finished.wait()
        self.dirty = set()


class Processor(object):
    """
    pop: pop tasks from delta fifo to work queue set
    start: start worker threads
    stop: stop worker threads
    """

    def __init__(self, work_queue: QueueSet):
        self.work_queue = work_queue
        self.threads = ThreadPoolExecutor(1000)
        self.stopped = Event()

    def pop(self):
        for item in range(1000):
            self.work_queue.put(index=0, item=item)
        print("pop from delta fifo done")

    def _first_worker(self):
        while not self.stopped.is_set():
            try:
                item = self.work_queue.get(index=0, timeout=1)
            except Empty:
                continue
            # print(f"filter {item}")
            [None for i in range(1000000)]
            if item % 3 == 0:
                self.work_queue.put(1, item)

        # print("filter done")

    def _second_worker(self):
        while not self.stopped.is_set():
            try:
                item = self.work_queue.get(index=1, timeout=1)
            except Empty:
                continue
            # print(f"work {item}")
            [None for i in range(1000000)]
            self.work_queue.done()

    def start(self):
        [self.threads.submit(self._first_worker) for _ in range(200)]
        [self.threads.submit(self._second_worker) for _ in range(200)]

    def stop(self):
        print("try end")
        self.stopped.set()
        self.threads.shutdown(wait=True)
        print("end")


if __name__ == '__main__':
    queue_set = QueueSet(size=2)
    print("=" * 100)
    processor = Processor(queue_set)
    processor.start()
    for i in range(1):
        processor.pop()
        time.sleep(0.5)
        queue_set.interrupt()
        print("interrupted")
        queue_set.join()
    processor.stop()
    time.sleep(1)

    print("=" * 100)
    processor = Processor(queue_set)
    processor.start()
    for i in range(1):
        processor.pop()
        queue_set.join()
        time.sleep(1)
    processor.stop()