python 多线程使用

多线程处理一批机器信息

    
class HandlerHostsInfoMulti:
    # 初始化host_obj字典，用于存储所有主机信息
    # 创建一个锁
    lock = threading.Lock()
    thread_num = 20
    res_map = {}
    res_list = []
    res_dict = {}

    @classmethod
    def create_multi_res_list(cls):
        for i in range(cls.thread_num):
            cls.res_map[i] = {}

    @classmethod
    def add_to_res_map(cls, i, host_name, value):
        cls.res_map[i][host_name] = value

    @classmethod
    def merge_res_map(cls):
        for value in cls.res_map.values():
            cls.res_dict.update(value)

    @classmethod
    def chunk_list(cls, lst, n):
        """Yield successive n-sized chunks from lst.√"""
        # 将host列表分成五部分
        step = len(lst) // n
        print("步长：", step)
        for i in range(0, len(lst), step):
            print("i=={},range:{}".format(i, lst[i : i + step]))
            yield lst[i : i + step]

    @classmethod
    def update_host_info(cls, hosts, thread_index):
        for host in hosts:
            host_obj = {}
            idc = host.get("idc")
            tag_info = host.get("tag_info", [])
            tags_name = ",".join([item.get("name") for item in tag_info if tag_info])
            hostname = host.get("hostname")
            host_obj.update(hostInfoObj(hostname).get_host_info_json(cls.get_runtime_mgs))
            host_obj.update({"idc": idc})
            host_obj.update({"tags": tags_name})
            # cls.append_to_list(host_obj)
            cls.add_to_res_map(thread_index, host_obj)

 # 创建线程函数
    @classmethod
    def worker(cls, q, thread_index):
        while not q.empty():
            hosts = q.get()
            # print("线程【{}】的hosts{}".format(thread_index,hosts))
            cls.update_host_info_dict(hosts, thread_index)
            q.task_done()


    @classmethod
    def multi_handler(cls, hosts):
        cls.create_multi_res_list()
        # 创建队列
        q = Queue()
        # 划分host列表并放入队列
        if len(hosts) <= 10:
            cls.thread_num = len(hosts)
        elif len(hosts) < 100 and len(hosts) > 10:
            cls.thread_num = 4
        else:
            cls.thread_num = 20
        for chunk in cls.chunk_list(hosts, cls.thread_num):
            q.put(chunk)

        # 创建多个线程
        threads = []
        for thread_index in range(cls.thread_num):
            t = threading.Thread(target=cls.worker, args=(q, thread_index))
            t.start()
            threads.append(t)

            # 等待所有任务完成
        q.join()
        # 返回host_obj列表
        cls.merge_res_map()
        return cls.res_dict
    #
    @classmethod
    def update_host_info_dict(cls, hosts, thread_index):
        for host in hosts:
            host_obj = {}
            host_name = host.get("host_name", "")
            if host_name:
                host_obj.update(hostInfoObj(host_name).get_host_info_json())
                cls.add_to_res_map(thread_index, host_name, host_obj)

    @classmethod
    def runtime_info_main_handler(cls, hosts):
        return cls.multi_handler(hosts)

根据线程数量，先使用chunk_list（）将hosts数组分成几份，每个线程单独使用一份hosts切片对象。

create_multi_res_list()生成保存数据，不同线程使用map的不同key对应的value空间保存值，注意：dict是虽然不是线程安全的。但是可以使用不同的线程index，指定线程使用的空间。

list也不是线程安全的，若是需要使用的话，需要threading lock

import threading
  
  lock = threading.Lock()

@classmethod
    def append_to_list(cls,item):
        # 获取锁
        cls.lock.acquire()
        try:
            # 在锁的保护下向列表添加元素
            cls.host_resp.append(item)
        finally:
            # 释放锁
            cls.lock.release()