python多进程(类成员函数、log在多进程中使用、自己维护进程池)

写这篇的原因参考之前写的一篇博客Python 多进程池中子进程挂了导致进程不能结束的问题小结_文锦渡的博客-优快云博客_python 子进程不能退出问题：Python multiprocessing pool not shutting down while child processes is oom开篇点题：在使用python的multiprocessing中的pool时，会出现子进程因为各种原因挂了之后主进程任务停不下来的情况。这是因为主进程没能发现子进程挂了而判定结束，从而卡在了这个地方，是python的一个bug。解决办法有两个，且看下文分解～https://blog.youkuaiyun.com/mxdsdo09/article/details/119728373?spm=1001.2014.3001.5501

 主要是为了记录自己实现的

1、python进程池维护

2、logging在多进程中的使用

3、调用类的成员函数实现多进程

一、调用类的成员函数实现多进程

下面这部分代码是调用python自带的multiprocessing.Pool的进程池实现的对类的成员函数的调用，优点是方便快捷，但缺点在开头的另一篇文章里提到了 无法判断OOM和子进程GG 主进程会一直运行 不能中断

import os
import sys
import time
import multiprocessing


class PrintNumber():

    def __init__(self, multi_num):
        self.multi_num = multi_num

    def run_task(self):
        sum_num = 1
        for i in range(5):
            sum_num += i
            time.sleep(1)
            print("multi_num = {}, output = {}".format(self.multi_num, sum_num))


if __name__ == "__main__":
    print("multiprocess test")

    process_pool = multiprocessing.Pool(3)
    for i in range(3):
        test = PrintNumber(i)
        process_pool.apply_async(test.run_task, args=())

    print("main thread")
    test_main = PrintNumber(10)
    test_main.run_task()

    process_pool.close()
    process_pool.join()

    print("thread join success!")

注意！process_pool.apply_async(test.run_task, args=())不要在test.run_task后加括号（test.run_task（）），否则不会报错且输出相当于一个进程。。。原因懂得得大佬可以解释一下～

输出

#python3 test.py 
multiprocess test
main thread
multi_num = 2, output = 1
multi_num = 1, output = 1
multi_num = 10, output = 1
multi_num = 0, output = 1
multi_num = 2, output = 2
multi_num = 0, output = 2
multi_num = 1, output = 2
multi_num = 10, output = 2
multi_num = 2, output = 4
multi_num = 1, output = 4
multi_num = 0, output = 4
multi_num = 10, output = 4
multi_num = 1, output = 7
multi_num = 2, output = 7
multi_num = 0, output = 7
multi_num = 10, output = 7
multi_num = 1, output = 11
multi_num = 2, output = 11
multi_num = 0, output = 11
multi_num = 10, output = 11
thread join success!

这里做了个验证，得for循环给PrintNumber实例化同一个名称test不会导致进程运行异常（原因暂时不清楚，感觉后面给test赋值应该会覆盖前面先实例化的呀？）

二、自己构建进程池实现

加入对进程数量和结果的判断，防止子进程挂了没能发现

import os
import sys
import time
import multiprocessing
import logging

# 全局变量在多进程中使用
global g_log
g_log = None


def init_log(log_file):
    log = logging.getLogger()  # 不加名称设置root logger
    log.setLevel(logging.DEBUG)
    formatter = logging.Formatter("%(asctime)s<%(levelname)s>: %(message)s",
                                  datefmt="%Y-%m-%d %H:%M:%S")

    # 使用FileHandler输出到文件
    fh = logging.FileHandler(log_file)
    fh.setLevel(logging.DEBUG)
    fh.setFormatter(formatter)

    # 使用StreamHandler输出到屏幕
    ch = logging.StreamHandler()
    ch.setLevel(logging.DEBUG)
    ch.setFormatter(formatter)

    # 添加两个Handler
    log.addHandler(ch)
    log.addHandler(fh)
    log.info("init log success!")

    return log


class PrintNumber():

    def __init__(self, multi_num):
        self.multi_num = multi_num

    def run_task(self, thread_q):
        thread_id = os.getpid()
        result = []

        # debug 验证失败的情况
        # if not self.multi_num > 0:
        #     success = False
        #     thread_q.put((result, success))
        #     return

        sum_num = 1
        for i in range(5):
            sum_num += i
            time.sleep(1)
            g_log.info("thread_id<{}> multi_num = {}, output = {}".format(
                thread_id, self.multi_num, sum_num))

        success = True
        result.append(sum_num)
        thread_q.put((result, success))


if __name__ == "__main__":
    log_file = os.path.join(os.path.abspath(os.path.dirname(__file__)),
                            "test.log")
    log = init_log(log_file)

    log.info("multiprocess test")
    g_log = log

    thread_list = []
    thread_q = multiprocessing.Queue()

    for i in range(3):
        test = PrintNumber(i)
        # 重点注意！
        p = multiprocessing.Process(target=test.run_task, args=(thread_q,))
        p.start()
        thread_list.append(p)

    log.info("main thread")
    test_main = PrintNumber(10)
    test_main.run_task(thread_q)

    for p in thread_list:
        p.join()
    log.info("thread join success!")

    if not len(thread_list) + 1 == thread_q.qsize():
        log.error(
            "thread process false! input thread num:{}, output thread num:{}".
            format(len(thread_list) + 1, thread_q.qsize()))
        sys.exit(-1)

    result_list = []
    sucess_list = []
    for i in range(thread_q.qsize()):
        thread_q_temp = thread_q.get()
        result_list.extend(thread_q_temp[0])
        sucess_list.append(thread_q_temp[1])

    for sucess in sucess_list:
        if not sucess:
            log.error("thread process false! sucess is false!")
            sys.exit(-1)

    log.info("all thread process success!")

参考：

Python logging同时输出到屏幕和文件 - 孤鸿的天空

Python logging浅尝（将log同时输出到Console和日志文件）_小桔帽的博客-优快云博客_python日志输出到文件