线程池

1. 任务-Task

笔记顺序-4

Task将callback函数包装成一个class，并带有任务id信息。

import uuid
class Task(object):
    def __init__(self, callback, *args, **kwargs):
        self.id = uuid.uuid4()
        self.callback = callback
        self.args = args
        self.kwargs = kwargs
    def __str__(self):
        return 'Task ID: {0}'.format(self.id)

2. 继承threading.Thread，重写run()-ProcessThread

笔记顺序-2

threading.Event()默认为False，is_set()方法，检测self.dismissFlag是否为True，这个只是用来停止线程。run()中调用队列的pop()，进程只是从任务队列中取任务，来调用callback函数。在上一步中，未向队列中添加任务。可以理解成，生成了数量为size的消费者。

import uuid
import threading
import time
from task import Task
class ProcessThread(threading.Thread):
    def __init__(self, taskQueue, *args, **kwargs):
        threading.Thread.__init__(self,  *args, **kwargs)
        # 通过threading.Event()可以创建一个事件管理标志
        # 该标志（event）默认为False
        self.dismissFlag = threading.Event()
        self.taskQueue = taskQueue
        self.args = args
        self.kwargs = kwargs
    def run(self):
        while True:
            if self.dismissFlag.is_set():
                break
            task = self.taskQueue.pop()
            if isinstance (task, Task):
                result = task.callback(*task.args, **task.kwargs)

3. 带有互斥锁和条件变量的队列-ThreadSafeQueue

import threading
class ThreadSafeQueueException(Exception):
    pass
class ThreadSafeQueue(object):
    def __init__(self, max_size=0):
        self.queue = []
        self.max_size = max_size
        self.lock = threading.Lock()
        self.cond = threading.Condition()
    def size(self):
        self.lock.acquire()
        size = len(self.queue)
        self.lock.release()
        return size
    def put(self, item):
        if self.max_size != 0 and self.size() > self.max_size:
            return ThreadSafeQueueException()
        self.lock.acquire()
        self.queue.append(item)
        self.lock.release()
        self.cond.acquire()
        self.cond.notify()
        self.cond.release()
    def pop(self):
        if self.size() == 0:
                self.cond.acquire()
                print('consumer: 即将被挂起, 直到被notify')
                self.cond.wait()
                print('consumer: 被notify')
                self.cond.release()
        self.lock.acquire()
        item = self.queue.pop()
        self.lock.release()
        return item
    def get(self, index):
        self.lock.acquire()
        item = self.queue[index]
        self.lock.release()
        return item

4. 线程池-ThreadPool

笔记顺序-1

pool = ThreadPool()，会创建两个队列，一个是线程池，一个是任务队列。for循环为线程池添加两个线程，可以看到这两个线程共用一个队列。此时只是创建两个线程，还需调用线程的start()方法。pool.start()从队列中索引进程，逐个调用thread.start()。线程会自动调用重写的ProcessThread.run()方法。

笔记顺序-3

我们只调用pool = ThreadPool()和pool.start()。

class ThreadSafeQueue(object):
    def pop(self):
        if self.size() == 0:
                self.cond.acquire()
                print('consumer: 即将被挂起, 直到被notify')
                self.cond.wait()
                print('consumer: 被notify')
                self.cond.release()

可以看到，当前两个线程均被阻塞，因为任务队列为空，正在等待生产者notify()。
threading.Condition().wait()==>threading.Condition().wait(timeout=None)，当前进程阻塞，直到在另外一个线程中调用同一个条件变量的 notify()或 notify_all()唤醒它。或者直到可选的超时发生。
现在我们往任务队列里添加两个任务。

pool = ThreadPool()
pool.start()
simpleTask = SimpleTask(processA)
pool.put(simpleTask)
simpleTask = SimpleTask(processB)
pool.put(simpleTask)

可以看到，每当生产者向任务队列里添加一个任务，就会notify()一次消费者。

from task_queue import ThreadSafeQueue
from thread import ProcessThread
from task import Task
import time

class ThreadPool(object):
    def __init__(self, size=2):
        # 线程池
        # size 控制 self.queue 的大小
        self.pool = ThreadSafeQueue(size)
        # 任务队列
        self.taskQueue = ThreadSafeQueue()
        for i in range(size):
            self.pool.put(ProcessThread(self.taskQueue))
    def start(self):
        for i in range(self.pool.size()):
            thread = self.pool.get(i)
            thread.start()
    def put(self, task):
        if isinstance(task, Task):
            self.taskQueue.put(task)

# test
class SimpleTask(Task):
    def __init__(self, callback):
        super(SimpleTask, self).__init__(callback)
def processA():
    time.sleep(1)
    print('it is a processAAAAA function.')
def processB():
    time.sleep(1)
    print('it is a processBBBBB function.')
def test():
    # 1. 初始化一个线程池
    pool = ThreadPool()
    # 2. 当一个线程 start 之后, 会调用 run()
    pool.start()
    # 3. 生成一系列任务
    while True:
        simpleTask = SimpleTask(processA)
    # 4. 往线程池提交任务执行
        pool.put(simpleTask)
        simpleTask = SimpleTask(processB)
        pool.put(simpleTask)
if __name__ == "__main__":
    test()

5. All Code

5.1 task

import uuid
class Task(object):
    def __init__(self, callback, *args, **kwargs):
        self.id = uuid.uuid4()
        self.callback = callback
        self.args = args
        self.kwargs = kwargs
    def __str__(self):
        return 'Task ID: {0}'.format(self.id)

5.2 thread

import uuid
import threading
import time
from task import Task
class ProcessThread(threading.Thread):
    def __init__(self, taskQueue, *args, **kwargs):
        threading.Thread.__init__(self,  *args, **kwargs)
        # 通过threading.Event()可以创建一个事件管理标志
        # 该标志（event）默认为False
        self.dismissFlag = threading.Event()
        self.taskQueue = taskQueue
        self.args = args
        self.kwargs = kwargs
    def run(self):
        while True:
            # self.dismissFlag.is_set() and break
            # is_set: 判断 dismissFlag 是否为 True
            if self.dismissFlag.is_set():
                break
            task = self.taskQueue.pop()
            if isinstance (task, Task):
                result = task.callback(*task.args, **task.kwargs)
    def stop(self):
        self.dismissFlag.set()

5.3 task_queue

import threading
class ThreadSafeQueueException(Exception):
    pass
class ThreadSafeQueue(object):
    def __init__(self, max_size=0):
        self.queue = []
        self.max_size = max_size
        self.lock = threading.Lock()
        self.cond = threading.Condition()
    def size(self):
        self.lock.acquire()
        size = len(self.queue)
        self.lock.release()
        return size
    def put(self, item):
        if self.max_size != 0 and self.size() > self.max_size:
            return ThreadSafeQueueException()
        self.lock.acquire()
        self.queue.append(item)
        self.lock.release()
        self.cond.acquire()
        self.cond.notify()
        self.cond.release()
    def batch_list(self, item_list):
        if not isinstance (item_list, list):
            item_list = list(item_list)
        self.lock.acquire()
        for item in item_list:
            self.queue.append(item)
        self.lock.release()
        self.cond.acquire()
        self.cond.notify()
        self.cond.release()
    def pop(self):
        if self.size() == 0:
                self.cond.acquire()
                print('consumer: 即将被挂起, 直到被notify')
                self.cond.wait()
                print('consumer: 被notify')
                self.cond.release()
        self.lock.acquire()
        item = self.queue.pop()
        self.lock.release()
        return item
    def get(self, index):
        self.lock.acquire()
        item = self.queue[index]
        self.lock.release()
        return item

5.4 pool

from task_queue import ThreadSafeQueue
from thread import ProcessThread
from task import Task
import time
class ThreadPool(object):
    def __init__(self, size=2):
        # 线程池
        # 初始化一个带有 条件变量和互斥锁 的队列
        # 条件变量和互斥锁, 保证不同进程安全运行
        # size 控制 self.queue 的大小
        self.pool = ThreadSafeQueue(size)
        # 任务队列
        self.taskQueue = ThreadSafeQueue()
        for i in range(size):
            self.pool.put(ProcessThread(self.taskQueue))
    def start(self):
        for i in range(self.pool.size()):
            thread = self.pool.get(i)
            thread.start()
    def stop(self):
        for i in range(self.pool.size()):
            thread = self.pool.get(i)
            thread.stop()
        while self.pool.size():
            thread = self.pool.get(i)
            thread.join()
    def put(self, task):
        if isinstance(task, Task):
            self.taskQueue.put(task)
    def batch_put(self, taskList):
        if isinstance(taskList, list):
            taskList = list(taskList)
        for task in taskList:
            self.put(task)
    def size(self):
        return self.pool.size()
        
# test
class SimpleTask(Task):
    def __init__(self, callback):
        super(SimpleTask, self).__init__(callback)
def processA():
    time.sleep(1)
    print('it is a processAAAAA function.')
def processB():
    time.sleep(1)
    print('it is a processBBBBB function.')
def test():
    # 1. 初始化一个线程池
    pool = ThreadPool()
    # 2. 当一个线程 start 之后, 会调用 run()
    pool.start()
    # 3. 生成一系列任务
    while True:
        simpleTask = SimpleTask(processA)
    # 4. 往线程池提交任务执行
        pool.put(simpleTask)
        simpleTask = SimpleTask(processB)
        pool.put(simpleTask)
        break
if __name__ == "__main__":
    test()