本文介绍了Python中用于加速多线程任务的Pipeline模式的两种实现方式，重点在于并发执行效率。通过示例代码展示了如何使用Pipeline、Pipe、ThreadPoolPipeDecorator等类来构建并发Pipeline，用于处理数据的并发处理。同时，提到了用于流程控制的Pipeline，强调任务的逻辑性和前向依赖性。文章还包含了有向图数据结构DiGraph的使用，用于拓扑排序确定执行顺序。
最近实现复杂问句解析时，需要用到将问句转化成由一系列的小的动作组成的有向图结构。该结构在执行的过程中 会动态生成一个自顶向下执行的逻辑图。 这就用到了Pipeline 模式了。
对于Pipeline 根据侧重点的不同，有两种实现方式用于加速多线程任务的pipeline
用于加速多线程任务的 Pipeline 主要强调 任务的顺序执行， 转移之间不涉及过于复杂的逻辑。所以 每个pipe 通过自身调用 next pipe。整体上强调 后向连续性。
2. 用于控制流程的pipeline
用于流程控制的piepline， 强调任务的 逻辑性， 由外部 manager 来控制 pipeline 的执行方向。整体上强调前向的依赖性， 使用拓扑排序确定执行顺序。
下面首先来看 用于加速多线程任务的pipeline的实现方法:
# !/usr/bin/python
# -*- coding: utf-8 -*-
# ------------------------------------------------------------------------
# @Time : 2020/9/28
# @Author : xiaoshan.zhang
# @Emial : zxssdu@yeah.net
# @File : pipeline.py
# @Software : PyCharm
# @desc : 并发型pipeline ， 强调pipe 的并发执行效率
# java 实现的参考网址:
# https://blog.youkuaiyun.com/tjy451592402/article/details/79459013
# ------------------------------------------------------------------------
"""Pipe: 处理阶段的抽象， 负责对输入输出进行处理， 并将输出作为下一个阶段的输入。pipe 可以理解为 (输入、处理、输出) 三元组init: 初始化当前处理阶段对外提供的服务。shutdown: 关闭当前处理阶段，对外提供的服务。setNextPipe: 设置当前处理阶段的下一个处理阶段。ThreadPoolPipeDecorator: 基于线程池的Pipe 实现类， 主要作用是实现线程池去执行对各个输入元素的处理。AbstractPipe: Pipe 的抽象实现类。process: 接收前一阶段的处理结果，作为输入， 并调用子类的doProcess 方法对元素进行处理，相应的处理结果会提交给下一个阶段进行处理do_process: 留给子类实现的抽象方法PipeContext: 对各个处理阶段的计算环境的抽象， 主要用于异常处理Pipeline: 对 复合pipe的抽象， 一个Pipeline 实例可以包含多个pipe 实例。addPipe: 向该Pipeline 实例中添加一个Pipe实例SimplePipeline: 基于AbstractPipe 的 Pipeline 接口实现的一个简单类PipelineBuilder : pipeline 构造器， 用于从配置文件中加载构建 pieplinePipelineMananger: 管理多个Pipeline 的构建、销毁、执行"""
import time
import random
import Queue
import threading
from abc import ABCMeta, abstractmethod
from threading import Condition, Thread
from concurrent import futures
from concurrent.futures import ThreadPoolExecutor
class CountDownLatch:
"""任务同步，用于同步异步任务，当注册了该同步锁的异步任务都执行完成后才释放锁。"""
def __init__(self, count):
self.count = count
self.condition = Condition()
def await(self):
try:
self.condition.acquire()
while self.count > 0:
self.condition.wait()
finally:
self.condition.release()
def countDown(self):
try:
self.condition.acquire()
self.count -= 1
self.condition.notifyAll()
finally:
self.condition.release()
class AbstractPipe(object):
def __init__(self, pipe_name=None, pipe_context=None):
self.pipe_name = pipe_name
self.next_pipe = None
self.pipe_context = pipe_context
def set_next(self, next_pipe):
self.next_pipe = next_pipe
def init(self, pipe_context):
self.pipe_context = pipe_context
def shut_down(self, timeout, time_unit):
"""关闭 pipe 执行的任务:param timeout::param time_unit::return:"""
def process(self, input):
# try:
out = self.do_process(input)
if 'results' in self.pipe_context:
self.pipe_context['results'][self.pipe_name] = out
# 如果正确输出，并且当前正确定义了下一个pipe,调用下一个pipeline
if out and self.next_pipe:
print("当前 结果不为空， 下一个 pipe 不为 None: {}, 主动调用 下一个 pipe: {}".format(self.next_pipe, out))
self.next_pipe.process(out)
def do_process(self, input):
raise NotImplementedError("Please implement do_process in inherit pipe class!")
class Function():
__metaclass__ = ABCMeta
def __init__(self, params={}, result={}, nlu_template=None, nlg_template=None ):
self.params = {}
self.result = {}
self.nlu_template = nlu_template
self.nlg_tempalte = nlg_template
def process(self, input):
raise NotImplementedError("Please implement Function`s process logical")
def gen_nlu_pattern(self):
return self.nlu_template
def gen_nlg_pattern(self):
return self.nlg_tempalte
def __call__(self, input):
self.process(input)
class FunctionPipe(AbstractPipe):
"""Pipe 函数修饰类。调用内部封装的 函数类执行具体的逻辑"""
__metaclass__ = ABCMeta
def __init__(self, pipe_name, function):
super(FunctionPipe, self).__init__(pipe_name=pipe_name)
self.function = function
@abstractmethod
def do_process(self, inputs):
""":param inputs::return:"""
# 根据函数定义的参数列表，从context 中取出参数对应的值，
kwargs = dict([(param_name, self.pipe_context[param_name]) \
for param_name in self.function.params])
# 传入 exec函数中
result = self.function.execute(**kwargs)
# 根据函数定义的返回参数列表，将处理结果放在 context 中
for res_name in self.function.res_names:
self.pipe_context[res_name] = result[res_name]
# 返回 std_nlu 和 nlg语句
std_nlu = None
nlg = None
return std_nlu , nlg
class Constraint(Function):
"""约束基类, 也是函数的一种直接解析往往比较困难，而且会不可避免地造成程序和语言的分歧。数据流的存在给了我们另一种选择：根据字面意思把引用解释成某种约束(Constraint)，再调用「解析」函数把符合约束的程序从数据流中找出来。"""
__metaclass__ = ABCMeta
def __init__(self,type_):
self.type_ = type_
def do_process(self, input):
self.fit(input)
@abstractmethod
def fit(self,input):
raise NotImplementedError("Please implement in inherit class!")
class ThreadPipeDecorator(AbstractPipe):
"""Pipe 的线程修饰类， 它不会维持一直存在的worker，而是任务到来时启动一个thread，这样， 内存压力会比较少，是标准的做法， 但是有线程切换开销。"""
def __init__(self, delegate_pipe, pool_executor):
""":param delegate_pipe::param pool_executor:"""
self.delegate_pipe = delegate_pipe
self.thread_pool = pool_executor
def init(self, pipe_context):
"""为业务对象 pipe 设置上下文:param pipe_context::return:"""
self.delegate_pipe.init(pipe_context)
def process(self, input):
"""注意 线程装饰器 的 process 函数不需要 调用 下一个 pipe， 由业务对象 pipe自己去调用:param input::return:"""
print("当前 pipe thread recive input: {}".format(input))
task = lambda input: self.delegate_pipe.process(input)
self.thread_pool.submit(task, input)
# 使用单线程 提交任务
# thread = threading.Thread(target=task, args=[input,])
# thread.setDaemon(True)
# thread.start()
def set_next(self, next_pipe):
"""为业务对象设置上下文:param next_pipe::return:"""
self.delegate_pipe.set_next(next_pipe)
class WorkerPipeDecorator(AbstractPipe):
"""pipe 的线程池装饰类, 内部会维持一个一直运行的 worker， 无线程切换开销，但是在pipe个数多时，内存压力比较大说明: 使用线程池的时机:被 threadpool pipe 装饰器装饰的 pipe 不应该放入线程池中处理，因为线程池的同时运行的线程是有限的，但是 装饰器包装的 pipe 的线程的任务是放在 while 循环中的，不会主动结束。所以应该放在自由线程中。而 parallel pipe 的 sub pipe 应该放在线程池中，因为 sub pipe 只会执行一次，然后主动结束线程。"""
def __init__(self, delegate_pipe, pool_executor):
""" """
super(WorkerPipeDecorator, self).__init__()
self.delegate_pipe = delegate_pipe
self.thread_pool = pool_executor
self.queue = Queue.Queue() # 内部队列
self.__active = False
# 启动pipe worker
self.start()
def process(self, input):
""":param input::return:"""
# 这里 process 的作用是将 input 和 context 添加到 自己的queue中
event = {"type": "pipe", "data": {
"context": self.pipe_context,
"input": input
}}
print("将输入 {} 封装成event: {}".format(input, event))
self.queue.put(event)
def start(self,):
"""启动 thread pipe:return:"""
self.__active = True
print("将异步pipe 的运行状态设置为: {}".format(self.__active))
def task():
"""当前线程:return:"""
print("启动 异步 pipe")
while self.__active:
event = self.queue.get(block=True, timeout=100)
print("当前 parallel pipe 收到 event: {}".format(event))
pipe_context = event['data']["context"]
input = event['data']['input']
self.delegate_pipe.init(pipe_context)
result = self.delegate_pipe.do_process(input)
event["data"]['input'] = result
# print("当前 pipe 的next pipe 为: {}, 封装的事件为: {}".format(self.next_pipe, event))
# 将当前 pipe 处理完成的 result 传递给下一个 pipe
self.next_pipe.queue.put(event)
self.thread_pool.submit(fn=task)
def shut_down(self):
# 将当前的 thread pipe 设置为不活跃，关闭 pipe 的线程
self.__active = False
class ParallelPipe(AbstractPipe):
"""说明使用线程池"""
def __init__(self, pipe_name=None, pool=None):
super(ParallelPipe, self).__init__(pipe_name=pipe_name)
self.pipes = []
self.count_down = None
self.pool = pool if pool else ThreadPoolExecutor(max_workers=3)
def add_pipe(self, pipe):
self.pipes.append(pipe)
def init(self, pipe_context):
for pipe in self.pipes:
pipe.init(pipe_context)
self.count_down = CountDownLatch(len(self.pipes))
def do_process(self, input):
"""并行执行 内部保存的各个子 pipe ， 所有pipe 执行完成才执行下游 pipe:param input::return:"""
def task(pipe, input, count_down, callback=None):
"""将 pipe 的执行与 同步锁 count_down 包装在一起"""
# count_down.wait()
result = pipe.do_process(input)
# print("当前 parallel pipe 输出结果为: {}".format(result))
if callback:
callback(result)
count_down.countDown()
return result
# results = []
futures = []
for pipe in self.pipes:
# 每个pipe的输入使用 输入的副本
input_cp = {"data": input['data']}
future = self.pool.submit(task, pipe, input_cp, self.count_down)
futures.append(future)
# 下面是使用单线程的方法，注意这里使用了异步回调函数，收集任务结果。
# thread = threading.Thread(target=task, args=(pipe, input_cp, self.count_down, lambda result:results.append(result)))
# thread.setDaemon(True)
# thread.start()
# 进行同步，等待所有的子pipe的任务结束后收集结果
self.count_down.await()
# 从future 中取出结果
results = [future.result() for future in futures]
return results
class SimplePipeline(AbstractPipe):
"""简单 pipeline，"""
def __init__(self, pool_executor):
self.thread_pool = pool_executor
self.pipes = []
def process(self, input):
first_pipe = self.pipes[0]
first_pipe.process(input)
def init(self, pipe_context):
"""完成 pipe 链的链接， 以 pipe 执行上下文的注入:param pipe_context::return:"""
prev_pipe = self
self.pipe_context = pipe_context
for pipe in self.pipes:
prev_pipe.set_next(pipe)
prev_pipe = pipe
pipe.init(pipe_context)
def add_pipe(self, pipe):
self.pipes.append(pipe)
def addAsThreadPoolBasedPipe(self, pipe):
"""将 pipe 按照多线程执行的方式添加到 pipeline 中:param pipe::return:"""
self.add_pipe(ThreadPipeDecorator(pipe, self.thread_pool))
def addAsWokerBasedPipe(self, pipe):
"""将 pipe 作为 woker 的任务添加到 pipeline中:param pipe::return:"""
self.add_pipe(WorkerPipeDecorator(pipe, self.thread_pool))
class DependencyPipeline(AbstractPipe):
"""具有依赖关系的 Pipeline, 每个 pipe 执行前需要， 判断其依赖关系是否满足"""
def __init__(self, ):
self.pipes = []
def init(self, pipe_context):
self.pipe_context = pipe_context
for pipe in self.pipes:
pipe.init(pipe_context)
def add_pipe(self, pipe):
self.pipes.append(pipe)
def dependency_check(self, pipe):
"""检查依赖关系从 context 查看 依赖的函数的 results 是否都输出了。:return:"""
is_check = True
# 首先去除 pipe 依赖的任务
dependencies = self.pipe_context['dependence'][pipe.pipe_name]
if dependencies and len(dependencies) > 0:
# 判断 依赖的是否返回结果
for dep_pipename in dependencies:
if self.pipe_context["results"][dep_pipename] is None:
print("pipe {} result is None".format(dep_pipename))
is_check = is_check and False
return is_check
def process(self, inputs):
for pipe in self.pipes:
is_check = self.dependency_check(pipe)
if not is_check:
print("Pipe {} dependency is not statisified, Please check it!")
pipe.process(inputs)
def reset(self):
"""重置当前pipeLine:return:"""
# 将当前 pipeline 的上下文中存储的result 置为 None
for pipe_name in self.pipe_context['results']:
self.pipe_context['results'][pipe_name] = None
# --------------- pipe ----------------------------------------------------------
class DataTransformPipe(AbstractPipe):
def __init__(self, indicator):
# 先初始化父类
super(DataTransformPipe, self).__init__()
self.indicator = indicator
def do_process(self, input):
result = self.indicator + input['data']
time.sleep(random.randint(1, 3))
print("Data transform entit indicator: {}".format(self.indicator + input['data']))
input["data"] = result
return input
class MapPipe(AbstractPipe):
"""实现map功能的 pipe"""
def __init__(self, add_unit):
super(MapPipe, self).__init__()
self.add_unit = add_unit
def do_process(self, input):
input['data'] = input['data'] + self.add_unit
print("Map pipe add unit: {}, result: {}".format(self.add_unit, input['data']))
return input
class ReducePipe(AbstractPipe):
"""实现Reduce功能的Pipe"""
def __init__(self):
super(ReducePipe, self).__init__()
def do_process(self, input):
print("Reduce pipe 接收到的内容为: {}".format(input))
if not type(input) is list:
inputs = [input]
else:
inputs = input
sum = 0
for input in inputs:
sum += input['data']
result = {"data": sum}
print("Reduce Pipe result is {}".format(result))
return result
def main():
"""测试 多线程执行方法:return:"""
pool = ThreadPoolExecutor(max_workers=20)
simple_pipeline = SimplePipeline(pool_executor=pool)
# 创建 pipe
pipe_one = DataTransformPipe(indicator=1)
pipe_two = DataTransformPipe(indicator=2)
pipe_three = DataTransformPipe(indicator=3)
pipe_four = DataTransformPipe(indicator=4)
pipe_five = DataTransformPipe(indicator=5)
# 测试 parallel pipe 的执行
paral_pipe = ParallelPipe()
for i in range(10):
paral_pipe.add_pipe(MapPipe(i))
reduce_pipe = ReducePipe()
pipes = [pipe_one, pipe_two, pipe_three, pipe_four, pipe_five, paral_pipe, reduce_pipe]
# for pipe in pipes:
# simple_pipeline.add_pipe(pipe)
#
# # 使用Pipe context 来初始化 simple_pipeline
# print("使用单线程执行")
# simple_pipeline.init(pipe_context={})
# simple_pipeline.process(input={'data':10})
# # 下面来验证 多线程下的执行
for pipe in pipes:
simple_pipeline.addAsThreadPoolBasedPipe(pipe)
# simple_pipeline.addAsWokerBasedPipe(pipe)
simple_pipeline.init(pipe_context={})
print("使用多线程执行")
# simple_pipeline.process(input={'data': 10})
# simple_pipeline.process(input={'data': 20})
# simple_pipeline.process(input={'data': 30})
# simple_pipeline.process(input={'data': 40})
# pool.shutdown(wait=True)
for i in range(10):
simple_pipeline.process(input={'data': 10 * i})
# 注意这里需要保持主线程一致运行，否则 线程池也会 一起终止
while True:
time.sleep(2)
print("主线程执行一次")
if __name__ == "__main__":
main()
在上面的基础上添加一个 Pipeline Manager，可以实现具有依赖关系的 Pipeline
下面是 pipeline 中组件依赖管理器
# !/usr/bin/python
# -*- coding: utf-8 -*-
# ------------------------------------------------------------------------
# @Time : 2020/9/29
# @Author : xiaoshan.zhang
# @Emial : zxssdu@yeah.net
# @File : Manager.py
# @Software : PyCharm
# ------------------------------------------------------------------------
class DepPipeLineManager(object):
def __init__(self):
self.pipeline_dict = {}
def regist(self, pipeline_name, pipeline):
"""
注册 pipeline
:param pipeline_name:
:param pipeline:
:return:
"""
self.pipeline_dict[pipeline_name] = pipeline
def unregist(self, pipeline_name):
if pipeline_name in self.pipeline_dict:
del self.pipeline_dict[pipeline_name]
def process(self, pipeline_name, input):
self.pipeline_dict[pipeline_name].process(input)
它用于管理具有前向依赖的组件关系，这些组件依赖管辖的配置如下:
modules: [./tasks/task.py]
runs: 2
pipelines:
- name: TestPipeline
tasks:
- name: task_A
type: normal
class: DataTransformPipe
params:
- indicator: 1
dependence:
- name: task_B
type: normal
class: DataTransformPipe
params:
- indicator: 2
dependence:
- task_A
- task_C
- name: task_C
type: normal
class: DataTransformPipe
params:
- indicator: 3
dependence:
- task_A
- task_E
- name: task_D
type: normal
class: DataTransformPipe
params:
- indicator: 4
dependence:
- task_B
- task_C
- task_E
- name: task_E
type: normal
class: DataTransformPipe
params:
- indicator: 5
dependence:
- task_A
使用 manager_builder.py 模块读取上面的配置文件， 来构建具有前向依赖关系的pipeline
# !/usr/bin/python
# -*- coding: utf-8 -*-
# ------------------------------------------------------------------------
# @Time : 2020/9/29
# @Author : xiaoshan.zhang
# @Emial : zxssdu@yeah.net
# @File : manager_builder.py
# @Software : PyCharm
# @desc : pipeline 管理器的builder
# 负责从指定的配置源加载 pipeline 以及 manager 的信息，并构建
# ------------------------------------------------------------------------
import os
import sys
import abc
import yaml
import imp
import inspect
from abc import ABCMeta, abstractmethod
from src.parallel_pipeline.structure import DiGraph
from src.parallel_pipeline.pipeline import AbstractPipe, DependencyPipeline
from src.parallel_pipeline.Manager import DepPipeLineManager
def import_file(filename):
"""
使用指定的文件名，加载其包含的module
:param filename:
:return:
"""
# 文件绝对路径
path = os.path.abspath(os.path.dirname(filename))
# 去除后缀的文件名
name = os.path.splitext(os.path.basename(filename))[0]
results = imp.find_module(name, [path])
module = imp.load_module(name, results[0], results[1], results[2])
return module
class TaskImporter(object):
__metaclass__ = ABCMeta
def __init__(self):
self.tasks = {}
def import_tasks(self):
raise NotImplementedError("implement import_tasks function!")
class ModuleTaskImporter(TaskImporter):
def __init__(self):
super(ModuleTaskImporter, self).__init__()
def import_tasks(self, modulefile):
"""
从指定的 模块文件中加载任务
:param modulefile:
:return:
"""
module = import_file(modulefile)
# 获取指定 task 的所有子类
sub_class_list = AbstractPipe.__subclasses__()
for sub_class in sub_class_list:
class_name = sub_class.__name__
# print("当前的子类名称为: {}".format(class_name))
# m_py = getattr(model_module, 'm')
# 根据子类名称从m.py中获取该类
has_subclass = hasattr(module, class_name)
if has_subclass:
task_class = getattr(module, class_name)
# 实例化对象
# obj = obj_class_name()
# 调用print_name方法
# getattr(obj, 'do_process')()
self.tasks[class_name] = task_class
# print("当前加载的子类: {}, 加载的对象为: {}".format(class_name, task_class))
else:
# print("当前子类: {}, 不在module{} 中".format(class_name, modulefile))
continue
return self
class ManagerBuilder(object):
"""
管理器构建基础类
"""
__metaclass__ = ABCMeta
def __init__(self):
self.module_importer = ModuleTaskImporter()
def build_controller(self, pipe_spec, tasks):
pass
def build_pipeline(self, spec):
pipeline_name = spec['name']
tasks_spec = spec['tasks']
# 加载 pipeline 中的 依赖定义
# print("当前定义的 pipeline 名称为: {}, 加载到的任务定义为: {}".format(
# pipeline_name, tasks_spec
# ))
pipe_dic = {}
pipe_context = {"dependence": {},
"results": {}}
for task_spec in tasks_spec:
pipe_obj, pipe_dependence= self.process_task(task_spec)
pipe_dic[pipe_obj.pipe_name] = pipe_obj
# pipes.append(pipe_obj)
# 初始化 依赖关系 和 结果
pipe_context["dependence"][pipe_obj.pipe_name] = pipe_dependence
pipe_context["results"][pipe_obj.pipe_name] = None
# print("当前 pipe context 为: {}".format(pipe_context))
# 0 使用依赖关系进行拓扑排序
graph = DiGraph(len(pipe_dic))
# 首先初始化各个节点的邻接列表为空
for pipe in pipe_dic.values():
graph.addEdge(pipe.pipe_name, None)
# 根据依赖关系构建有向图
for dest_pipename in pipe_context["dependence"]:
# print("任务 {} 依赖的任务列表为: {}".format(dest_pipename, pipe_context["dependece"][dest_pipename]))
if pipe_context["dependence"][dest_pipename] is None:
continue
for src_pipename in pipe_context["dependence"][dest_pipename]:
graph.addEdge(src_pipename, dest_pipename)
print("构建的依赖图结构为: \n {}".format(graph.graph))
sorted_pipes = graph.loop_toposort()
print("经过拓扑排序后的 pipes 顺序为: {}".format(sorted_pipes))
pipes = [pipe_dic[pipe_name] for pipe_name in sorted_pipes]
# print("新的pipes列表为: {}".format([pipe.to_dict() for pipe in pipes]))
# 1. 根据依赖关系 初始化 pipe_context
dep_pipeline = DependencyPipeline()
for pipe in pipes:
# 2. 使用pipe_obj 构建pipeline
dep_pipeline.add_pipe(pipe)
# 将运行时上下文注入到 dep_pipeline 中
dep_pipeline.init(pipe_context)
return pipeline_name, dep_pipeline
def process_task(self, spec):
"""
解析生成 task
:param spec:
:return:
"""
# print("Task parser recieve spec : {}".format(spec))
task_name = spec['name']
task_classname = spec['class']
task_params = {}
for param in spec['params']:
task_params.update(param)
task_params.update({"pipe_name":task_name})
task_depdencies = spec['dependence']
# print("当前加载task的情况: task name: {}, class name: {}, params: {}, dependency：{}".format(
# task_name, task_classname, task_params, task_depdencies))
# 1. 实例化 pipe
pipe_obj = self.importer.tasks[task_classname](**task_params)
# print("当前加载的pipe_obj 为: {}".format(pipe_obj))
# 2. 返回其依赖关系
pipe_dependencies = task_depdencies
return pipe_obj, pipe_dependencies
def build_manager(self, spec):
modules = spec["modules"]
# 首先定义 importer , 将指定的模块先加载到内存中
self.importer = ModuleTaskImporter()
for module in modules:
self.importer.import_tasks(module)
# print(" manager 构建 接收到的 定义为: {}".format(spec))
pipeline_manager = DepPipeLineManager()
# 构建 Pipeline
for pipeline_spec in spec['pipelines']:
pipeline_name, dep_pipeline = self.build_pipeline(pipeline_spec)
pipeline_manager.regist(pipeline_name, dep_pipeline)
# 将构建好的 pipeline 注册到管理器中
# print("当前得到的 pipe_manager 为: \n {}".format(pipeline_manager.pipeline_dict))
return pipeline_manager
class PythonManagerBuilder(ManagerBuilder):
def build_manager(self, filename):
spec_module = import_file(filename)
return super(PythonManagerBuilder, self).build_manager(spec_module)
class YamlManagerBuilder(ManagerBuilder):
def build_manager(self, filename):
with open(filename, 'r') as fin:
spec = yaml.load(fin,Loader=yaml.FullLoader)
# print("当前从yaml中加载的内容: {}".format(spec))
return super(YamlManagerBuilder, self).build_manager(spec)
def main():
"""
:return:
"""
# file_name = "./tasks/task.py"
# module = import_file(file_name)
# print("加载的module 为: {}".format(module))
# # dynamic_loadfromfile(file_name)
#
# # 测试加载指定 抽象类的子类
# module_importer = ModuleTaskImporter()
# module_importer.import_tasks(file_name)
# 测试yaml 加载配置文件的方法
yaml_file = "conf/dependcy_pipeline.yaml"
manager_builder = YamlManagerBuilder()
dep_pipeline_manager = manager_builder.build_manager(yaml_file)
input = {"data":1}
dep_pipeline_manager.process('TestPipeline', input)
if __name__ == "__main__":
main()
其中task.py 定义了具体的业务逻辑组件
# !/usr/bin/python
# -*- coding: utf-8 -*-
# ------------------------------------------------------------------------
# @Time : 2020/9/29
# @Author : xiaoshan.zhang
# @Emial : zxssdu@yeah.net
# @File : task.py
# @Software : PyCharm
# ------------------------------------------------------------------------
import time
import random
from src.parallel_pipeline.pipeline import AbstractPipe
class DataTransformPipe(AbstractPipe):
def __init__(self, indicator, pipe_name):
# 先初始化父类
super(DataTransformPipe, self).__init__(pipe_name=pipe_name)
self.indicator = indicator
def do_process(self, input):
result = self.indicator + input['data']
time.sleep(random.randint(1, 3))
print("Data transform {} entit indicator: {}".format( self.pipe_name, self.indicator + input['data']))
input["data"] = result
return input
def to_dict(self):
dic = {
"pipe_name": self.pipe_name,
"indicator": self.indicator
}
return dic
class MapPipe(AbstractPipe):
"""
实现map功能的 pipe
"""
def __init__(self, add_unit):
super(MapPipe, self).__init__()
self.add_unit = add_unit
def do_process(self, input):
input['data'] = input['data'] + self.add_unit
print("Map pipe add unit: {}, result: {}".format(self.add_unit, input['data']))
return input
def to_dict(self):
dic = {
"pipe_name": self.pipe_name,
"add_unit": self.add_unit
}
return dic
class ReducePipe(AbstractPipe):
"""
实现Reduce功能的Pipe
"""
def __init__(self):
super(ReducePipe, self).__init__()
def do_process(self, input):
print("Reduce pipe 接收到的内容为: {}".format(input))
if not type(input) is list:
inputs = [input]
else:
inputs = input
sum = 0
for input in inputs:
sum += input['data']
result = {"data": sum}
print("Reduce Pipe result is {}".format(result))
return result
def to_dict(self):
dic = {
"pipe_name": self.pipe_name
}
return dic
下面是用到的 有向图数据结构：
# !/usr/bin/python
# -*- coding: utf-8 -*-
# ------------------------------------------------------------------------
# @Time : 2020/9/29
# @Author : xiaoshan.zhang
# @Emial : zxssdu@yeah.net
# @File : structure.py
# @Software : PyCharm
# ------------------------------------------------------------------------
from collections import defaultdict
class DiGraph(object):
"""
拓扑排序:
任何无回路的顶点活动网(AOV网)N都可以做出拓扑序列：
1. 从N中选出一个入度为0的顶点作为序列的下一顶点。
2. 从N网中删除所选顶点及其所有的出边。
3. 反复执行上面两个步骤，知道已经选出了图中的所有顶点，或者再也找不到入度为非0的顶点时算法结束。
4. 如果剩下入度非0的顶点，就说明N中有回路，不存在拓扑排序。
存在回路，意味着某些活动的开始要以其自己的完成作为先决条件，这种现象成为活动之间的死锁。
"""
def __init__(self, vertices):
self.graph = defaultdict(list)
self.V = vertices
def addEdge(self, u, v):
if v is None or v == '':
self.graph[u] = []
else:
self.graph[u].append(v)
def toposort_helper(self, v, visited, stack):
visited[v] = True
for i in self.graph[v]:
if visited[i] == False:
self.toposort_helper(i, visited, stack)
stack.insert(0, v)
def recursion_toposort(self):
visited = [False] * self.V
stack = []
for i in range(self.V):
if visited[i] == False:
self.toposort_helper(i, visited, stack)
return stack
def loop_toposort(self):
"""
非递归版本的拓扑排序
:return:
"""
in_degrees = dict((u, 0) for u in self.graph)
vertext_num = len(in_degrees)
for u in self.graph:
# 计算每个顶点的入度
for v in self.graph[u]:
# print("当前入度字典为: {}".format(in_degrees))
in_degrees[v] += 1
Q = [u for u in in_degrees if in_degrees[u] == 0]
Seq = []
while Q:
# 默认从最后一个删除
u = Q.pop()
Seq.append(u)
for v in self.graph[u]:
# 移除其所有的指向
in_degrees[v] -= 1
# 在次筛选入度为0的定点
if in_degrees[v] == 0:
Q.append(v)
# 如果循环结束后存在非0入度的定点
# 说明图中有环， 不存在拓扑结构
if len(Seq) == vertext_num:
return Seq
else:
print("Graph exists a cricle")
def main():
# 测试拓扑排序效果
# g = DiGraph(6)
# g.addEdge(5, 2)
# g.addEdge(5, 0)
# g.addEdge(4, 0)
# g.addEdge(4, 1)
# g.addEdge(2, 3)
# g.addEdge(3, 1)
"""
G = {
'a': 'bce',
'b': 'd',
'c': 'd',
'd': '',
'e': 'cd'
}
"""
g = DiGraph(5)
g.addEdge('a', 'b')
g.addEdge('a', 'c')
g.addEdge('a', 'e')
g.addEdge('b', 'd')
g.addEdge('c', 'd')
g.addEdge('e', 'c')
g.addEdge('e', 'd')
g.addEdge('d', None)
print("当前图结构为: \n{}".format(g.graph))
print("拓扑排序结果: ")
stack = g.loop_toposort()
print(stack)
# 非递归版本
def toposort(graph):
in_degrees = dict((u, 0) for u in graph)
vertex_num = len(in_degrees)
for u in graph:
# 计算每个定点的入度
for v in graph[u]:
in_degrees[v] += 1
Q = [u for u in in_degrees if in_degrees[u] == 0]
Seq = []
while Q:
# 默认从最后一个删除
u = Q.pop()
Seq.append(u)
for v in graph[u]:
in_degrees[v] -= 1 # 移除其所有的指向
# 再次筛选入度为0的定点
if in_degrees[v] == 0:
Q.append(v)
# 如果循环结束后存在非0入度的定点
# 说明图中有环， 不存在拓扑结构
if len(Seq) == vertex_num:
return Seq
else:
print("exists a circle.")
def loop_toposort():
"""
测试非递归版本的 拓扑排序
:return:
"""
G = {
'a': 'bce',
'b': 'd',
'c': 'd',
'd': '',
'e': 'cd'
}
print(toposort(G))
if __name__ == "__main__":
main()
# loop_toposort()
扫一扫
1856
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
