多线程+队列

关注

原创 2023-01-03 10:29:21 255 阅读 AI 写同款· GEO 优化 ›

多线程无锁队列的实现一、什么是多线程无锁队列？ 多线程无锁队列还是有锁的，只不过是用了cpu层面的CAS原子操作，用到这个操作，只需要在取队列元素和添加队列元素的时候利用CAS原子操作，就可以保证多个线程对队列元素的有序存取；二、什么是CAS操作？ CAS = Compare & Set，或是 Compare & Swap；在windows和Linux平台均有系统api函数，同时在c++11中阅读详情

import sys
from sqltool.mysql_client import MySqlClient
from queue import Queue, Empty
import os
import threading

base_dir = os.path.dirname(os.path.abspath('__file__'))
# 系统分隔符
os_sep = os.sep
sys.path.append(base_dir)


MYSQL_HOST = os.getenv("MYSQL_HOST", "localhost")
MYSQL_PORT = int(os.getenv("MYSQL_PORT", 3306))
MYSQL_USER = os.getenv("MYSQL_USER", "root")
MYSQL_PASSWORD = os.getenv("MYSQL_PASSWORD", "123")

MYSQL_SPIDER_DB = os.getenv("MYSQL_SPIDER_DB", "fosseye")
MYSQL_LJ_ID_DB = os.getenv("MYSQL_LJ_ID_DB", "lj_id")
MYSQL_OPEN_SOURCE_DB = os.getenv("MYSQL_OPEN_SOURCE_DB", "open_source_projects")


class MavenJar:

    def __init__(self):
        self.maven_cli = MySqlClient(
            host=MYSQL_HOST,
            port=MYSQL_PORT,
            user=MYSQL_USER,
            password=MYSQL_PASSWORD,
            db=MYSQL_SPIDER_DB
        )

    def get_maven_mysql(self):
        sql = "SELECT * FROM `表名` WHERE jar_md5 IS NULL AND jar_name IS NOT NULL"
        return self.maven_cli.query(sql)

    def maven_jar(self, res):
        pass

    def run(self, que):
        while True:
            try:
                k = que.get_nowait()
            except Empty:
                return
            self.maven_jar(k)


if __name__ == '__main__':
    maven = MavenJar()
    que = Queue()
    datas = maven.get_maven_mysql()
    for data in datas:
        que.put(data)
    
    ths = [threading.Thread(target=maven.run, args=(que,)) for i in range(20)]
    for th in ths:
        th.start()
    for th in ths:
        th.join()

收录于