Python - 基于nameko任务管理

Python - 基于nameko任务管理

Max.Bai

2019-05

 

功能

1. 任务管理：开始任务，停止任务（排队中，在执行），查看队列（排队中，执行中，slave）

2. 执行端：分不同类型，心跳

需要的库：

amqp==2.2.2
amqplib==1.0.2
Jinja2==2.10
nameko==2.12.0
peewee==3.9.4
six==1.11.0
threadpool==1.3.2
click

Master端代码

# _*_ coding:utf-8 _*_

import queue
import threading
import time
import traceback

from nameko.dependency_providers import Config
from nameko.rpc import rpc
from nameko.standalone.rpc import ServiceRpcProxy
from nameko.timer import timer


'''
Task manager

1. start task 
2. stop task when running (send signal to slave)
3. start task with env

author: Max.Bai
date: 2019-04
'''


# Slave status
SLAVE_STATUS_IDLE = "slave_idle"
SLAVE_STATUS_RUNNING = "slave_running"

# Task signal
TASK_SIGNAL_STOP = "stop_signal"

# Task execute enviroment
ENV_ONLINE = "ONLINE"
ENV_TEST = "TEST"
ENV_PRE = "PRE-RELEASE"

# CONFIG = {'AMQP_URI': "amqp://guest:guest@localhost"}
QUEUE_LOCK = threading.Lock()


class TaskManager:
    name = "task_master"
    task_runing_queue = []
    task_ready_queue = []
    runner_list = []
    _priority = 0

    CONFIG = Config()


    @property
    def priority(self):
        TaskManager._priority += 1
        if TaskManager._priority > 1000:
            TaskManager._priority = 0
        return TaskManager._priority

    @rpc
    def get_runner_list(self):
        return [{"name":r.name, "status":r.status} for r in self.runner_list]

    @rpc
    def get_queue_list(self):
        return [{"id":j.id, "name":j.name} for j in TaskManager.task_ready_queue]
    
    @rpc
    def get_running_list(self):
        return [{"id":j.id, "name":j.name} for j in TaskManager.task_runing_queue]

    @rpc
    def start_task(self, task):
        result = {
            "code": 0,
            "msg": ""
        }
        try:
            self.check_task_data(task)
            job = Job(task["id"], task["name"], task["env"], TaskManager.priority)
            # self.task_ready_queue.put(job)
            TaskManager.task_ready_queue.append(job)
        except Exception as e:
            result["code"] = 1
            result["msg"] = "Start task failed. ERROR:{}".format(str(e))
        return result

    @rpc
    def stop_task(self, task):
        stoped = False
        result = {
            "code": 0,
            "msg": ""
        }
        try:
            global QUEUE_LOCK
            if QUEUE_LOCK.acquire():
                try:
                    for j in TaskManager.task_ready_queue:
                        if j.id == task["id"]:
                            TaskManager.task_ready_queue.remove(j)
                            stoped = True
                            print("Stop task [{}] in queue success.".format(j.name))
                            break
                except Exception as e:
                    print("ERROR:Stop task [{}] in queue Failed!!.".format(task["id"]))
                QUEUE_LOCK.release()
            

            # task in running
            if not stoped:
                payload = {
                    "signal": TASK_SIGNAL_STOP,
                    "task": task
                }