zeroMQ初体验-10.优雅的使用多线程

ZeroMQ多线程实践
本文探讨了ZeroMQ作为高效多线程运行环境的优势,通过去锁化设计确保数据在同一时刻仅由一个线程持有,避免了传统多线程中常见的同步问题。文章提供了两个Python示例,展示了应答模式和服务流程的实现。
"或许,ZeroMQ是最好的多线程运行环境!"官网如是说。

其实它想要支持的是那种类似erlang信号模式。传统多线程总会伴随各种"锁"出现各种稀奇古怪的问题。而zeroMQ的多线程致力于"去锁化",简单来说,一条数据在同一时刻只允许被一个线程持有(而传统的是:只允许被一个线程操作)。而锁,是因为可能会出现的多线程同时操作一条数据才出现的副产品。从这里就可以很清晰的看出zeromq的切入点了,通过线程间的数据流动来保证同一时刻任何数据都只会被一个线程持有。

这里给出传统的应答模式的例子:
[img]http://github.com/imatix/zguide/raw/master/images/fig21.png[/img]

import time
import threading
import zmq

def worker_routine(worker_url, context):

socket = context.socket(zmq.REP)

socket.connect(worker_url)

while True:

string = socket.recv()
print("Received request: [%s]\n" % (string))
time.sleep(1)

socket.send("World")

def main():
url_worker = "inproc://workers"
url_client = "tcp://*:5555"

context = zmq.Context(1)

clients = context.socket(zmq.XREP)
clients.bind(url_client)

workers = context.socket(zmq.XREQ)
workers.bind(url_worker)

for i in range(5):
thread = threading.Thread(target=worker_routine, args=(url_worker, context, ))
thread.start()

zmq.device(zmq.QUEUE, clients, workers)

clients.close()
workers.close()
context.term()

if __name__ == "__main__":
main()


这样的切分还有一个隐性的好处,万一要从多线程转为多进程,可以非常容易的把代码切割过来再利用。
这里还给了一个用多线程不用多进程的理由:
进程开销太大了(话说,python是鼓励多进程替代线程的)。

上面代码给出的例子似乎没有子线程间的通信啊?既然支持用多线程,自然不会忘了这个:
[img]http://github.com/imatix/zguide/raw/master/images/fig22.png[/img]

import threading
import zmq

def step1(context):
sender = context.socket(zmq.PAIR)
sender.connect("inproc://step2")
sender.send("")

def step2(context):
receiver = context.socket(zmq.PAIR)
receiver.bind("inproc://step2")

thread = threading.Thread(target=step1, args=(context, ))
thread.start()

string = receiver.recv()

sender = context.socket(zmq.PAIR)
sender.connect("inproc://step3")
sender.send("")

return

def main():
context = zmq.Context(1)

receiver = context.socket(zmq.PAIR)
receiver.bind("inproc://step3")

thread = threading.Thread(target=step2, args=(context, ))
thread.start()

string = receiver.recv()

print("Test successful!\n")

receiver.close()
context.term()

return

if __name__ == "__main__":
main()


注意:
这里用到了一个新的端口类型:PAIR。专门为进程间通信准备的(文中还列了下为神马么用之前已经出现过的类型比如应答之类的)。这种类型及时,可靠,安全(进程间其实也是可以用的,与应答相似)。

(未完待续)
import time from typing import Any, Optional import numpy as np import zmq from agent.schema import RobotObsShape from dataclasses import dataclass, field class LowLevelRobotConnectionConfig: host: str = field(default="localhost") port: str = field(default="15558") timeout: int = field(default=1000) max_retries: int = field(default=5) delay: float = field(default=0.5) controller: Optional[Any] = field(default_factory=lambda: None) DOC = ''' controller 需要实现 get_obs 和 act 方法。例如: class Controller: def __init__(self, rgb, right_cam_img, right_state): self.rgb = rgb self.right_cam_img = right_cam_img self.right_state = right_state def get_obs(self) -> dict: """ { "rgb" np.ndarray (480, 640, 3), np.uint8 "right_cam_img" np.ndarray (360, 480, 3), np.uint8 "right_state" np.ndarray (7,), np.uint8 } Returns: _type_: _description_ """ return { "rgb": self.rgb, "right_cam_img": self.right_cam_img, "right_state": self.right_state, } def act(self, actions: np.ndarray) -> None: print(f"Executing...") print(f"Executed action: {actions}") ''' class LowLevelRobotConnection: def __init__(self, config): self.host = config.host self.port = config.port self.addr = f"tcp://{self.host}:{self.port}" # self.timeout = config.timeout self.max_retries = config.max_retries self.delay = config.delay self.is_connected = False self.controller = config.controller def _connect(self): print(f"INFO: Robot service connected to server at port {self.port}") self.context = zmq.Context() # 新建上下文 self.socket = self.context.socket(zmq.REQ) # 新建套接字 self.socket.connect(self.addr) self.is_connected = True def _close(self): # if self.context is not None: # self.context.term() print("INFO: context terminated") if self.socket is not None: self.socket.close() print("INFO: socket closed") self.is_connected = False def send_obs(self, obs: bytes) -> bool: fired = False for _ in range(self.max_retries): try: if not self.is_connected: self._connect() fired = True print("INFO: send observation") self.socket.send(obs) break except zmq.Again: print("ERROR: Timeout") self._close() if not fired: print("ERROR: Failed to fire observation to server") self._close() return False print("INFO: observation fired") return True def get_actions(self) -> Optional[np.ndarray]: for _ in range(self.max_retries): try: if not self.is_connected: self._connect() print("INFO: send action request") message = self.socket.recv(copy=False) print(f"INFO: recerved msg size: {len(message)}") if len(message) != RobotObsShape.TOTAL_ACTIONS_SIZE: print( f"ERROR: Invalid message size as {len(message)}, required {RobotObsShape.TOTAL_ACTIONS_SIZE} bytes" ) continue actions = np.frombuffer(message.buffer, dtype=np.float32).reshape( (RobotObsShape.ACTIONS_SHAPE) ) print("INFO: received action") break except zmq.Again: print("ERROR: Timeout") self._close() if actions is None: print("ERROR: Failed to retrieve action from server") return None return actions def run(self) -> None: while True: user_input = input("Press <Enter> to start, <q> to quit.") obs = self.get_obs() if not self.send_obs(obs): print("ERROR: Failed to send observation") continue actions = self.get_actions() if actions is None: print("ERROR: Failed to retrieve action from server") continue self.act(actions) time.sleep(self.delay) def step(self) -> None: obs = self.get_obs() if obs is None: print("ERROR: Failed to retrieve image") return if not self.send_obs(obs): print("ERROR: Failed to send observation") return actions = self.get_actions() if actions is None: print("ERROR: Failed to retrieve action from server") return self.act(actions) def get_obs(self) -> bytes: """获取观察的内容。头部、腕部图像、关节角度和灵巧手状态。 Returns: bytes: 字节流 """ obs = self.controller.get_obs() # head_image = np.zeros(RobotObsShape.HEAD_IMAGE_SHAPE, dtype=np.uint8) # wrist_image = np.zeros(RobotObsShape.WRIST_IMAGE_SHAPE, dtype=np.uint8) # state = np.zeros(RobotObsShape.STATE_SHAPE, dtype=np.float32) obs = ( obs["rgb"].tobytes() + obs["right_cam_img"].tobytes() + obs["right_state"].tobytes() ) return obs def act(self, actions: np.ndarray) -> None: """执行六个策略预测的动作。 Args: actions (np.ndarray): 形状为 (6, 7) """ assert actions.shape == RobotObsShape.ACTIONS_SHAPE, ( f"Expected actions shape {RobotObsShape.ACTIONS_SHAPE}, got {actions.shape}" ) print(f"INFO: actions: {actions}") if __name__ == "__main__": class Controller: def get_obs(self) -> dict: return { "rgb": np.zeros(RobotObsShape.HEAD_IMAGE_SHAPE, dtype=np.uint8), "right_cam_img": np.zeros( RobotObsShape.WRIST_IMAGE_SHAPE, dtype=np.uint8 ), "right_state": np.zeros(RobotObsShape.STATE_SHAPE, dtype=np.float32), } config = dict( host="wujingdp.xyz", port=15558, timeout=2, max_retries=5, delay=0.5, controller=Controller(), ) r = LowLevelRobotConnection(config) r.run() 全文注释
最新发布
07-18
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值