zeromq 并行流水线Parallel task zmqpp实现例子代码

• 并行任务发生器(taskvent.cpp)

#include <zmqpp/zmqpp.hpp>
#include <iostream>
#include <chrono>
#include <thread>

#define within(num) (int) ((float) num * random () / (RAND_MAX + 1.0))

int main() {
    // 初始化 ZeroMQ 上下文和套接字
    zmqpp::context context;
    zmqpp::socket sender(context, zmqpp::socket_type::push);
    sender.bind("tcp://*:5557");

    std::cout << "Press Enter when the workers are ready..." << std::endl;
    std::cin.get();
    std::cout << "Sending tasks to workers..." << std::endl;

    //  The first message is "0" and signals start of batch
    zmqpp::socket sink(context, zmqpp::socket_type::push);
    sink.connect("tcp://localhost:5558");
    zmqpp::message message(2);

    const char* part1 = "0";
    message.add(part1, strlen(part1) + 1); // 包括字符串的 null 终止符
    sink.send(message);

    //  Initialize random number generator
    srandom ((unsigned) time (NULL));
    int total_msec = 0;     //  Total expected cost in msecs

    // 发送 100 个任务
    for (int task_nbr = 0; task_nbr < 100; ++task_nbr) {

        int workload;
        //  Random workload from 1 to 100msecs
        workload = within (100) + 1;
        total_msec += workload;

        zmqpp::message task;
        task << workload;

        sender.send(task);
    }

    std::cout << "Total expected cost: " << total_msec << " msec" << std::endl;
    std::this_thread::sleep_for(std::chrono::seconds(1));      //  Give 0MQ time to deliver

    std::cout << "All tasks sent!" << std::endl;
    return 0;
}

• 并行任务工人(taskwork.cpp)

//
//  Task worker in C++
//
//  Connects PULL socket to tcp://localhost:5557
//  Collects workloads from ventilator via that socket
//
//  Connects PUSH socket to tcp://localhost:5558
//  Sends results to sink via that socket
//

#include <zmqpp/zmqpp.hpp>
#include <iostream>
#include <chrono>
#include <thread>

int main() {
    // 初始化 ZeroMQ 上下文和套接字
    zmqpp::context context;
    zmqpp::socket receiver(context, zmqpp::socket_type::pull);
    receiver.connect("tcp://localhost:5557");

    zmqpp::socket sender(context, zmqpp::socket_type::push);
    sender.connect("tcp://localhost:5558");

    while (true) {
        // 接收任务
        zmqpp::message task;
        receiver.receive(task);

        int workload;
        task >> workload;
        std::cout << "Received: " << workload << std::endl;

        // 模拟任务处理
        std::this_thread::sleep_for(std::chrono::milliseconds(workload));

        // 发送处理结果
        zmqpp::message result;
        // 重置消息
        result << "";
        sender.send(result);
        std::cout << "." << std::flush;
    }

    return 0;
}

• 并行任务接收器(tasksink.cpp)

#include <zmqpp/zmqpp.hpp>
#include <time.h>
#include <sys/time.h>
#include <iostream>
#include <chrono>

int main() {
    // 初始化 ZeroMQ 上下文和套接字
    zmqpp::context context;
    zmqpp::socket receiver(context, zmqpp::socket_type::pull);
    receiver.bind("tcp://*:5558");

    // 等待任务分发器发送开始信号
    zmqpp::message start_signal;
    receiver.receive(start_signal);

    // 记录开始时间
    auto start_time = std::chrono::steady_clock::now();

    //  Process 100 confirmations
    int task_nbr;
    int total_msec = 0;     //  Total calculated cost in msecs
    // 接收 100 个任务结果
    for (int task_nbr = 0; task_nbr < 100; ++task_nbr) {
        zmqpp::message result;
        receiver.receive(result);

        if (task_nbr % 10 == 0)
            std::cout << ":" << std::flush;
        else
            std::cout << "." << std::flush;
    }

    // 计算总耗时
    auto end_time = std::chrono::steady_clock::now();
    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time).count();

    std::cout << "Total elapsed time: " << duration << " ms" << std::endl;
    return 0;
}

Makefile参考

CPP                     = g++
RM                      = rm -f
CPPFLAGS                = -Wall -I. -O2 -std=c++11 -g
LDFLAGS                 = -lpthread -lzmqpp -lzmq -lsodium -Wl,--no-as-needed
SOURCES                 = $(wildcard *.cpp)
TARGETS                 = $(SOURCES:%.cpp=%)

all:    ${TARGETS}

clean:
        ${RM} *.obj *~* ${TARGETS}

${TARGETS}:
        ${CPP} ${CPPFLAGS} -o $@ ${@:%=%.cpp} ${LDFLAGS}

运行结果

# ./taskwork
Received: 33
.Received: 40
.Received: 35
.Received: 27
.Received: 54
.Received: 94
.Received: 43
.Received: 19
.Received: 90
.Received: 29
.Received: 9
.Received: 83
.Received: 45
.Received: 39
.Received: 60
.Received: 53
.Received: 12
.Received: 18
.Received: 25
.Received: 3
.Received: 12




# ./tasksink
:.........:.........:.........:.........:.........:.........:.........:.........:.........:.........Total elapsed time: 1727 ms
# ./tasksink
:.........:.........:.........:.........:.........:.........:.........:.........:.........:.........Total elapsed time: 2557 ms
# ./tasksink
:.........:.........:.........:.........:.........:.........:.........:.........:.........:.........Total elapsed time: 5265 ms


# ./taskvent 
Press Enter when the workers are ready...

Sending tasks to workers...
Total expected cost: 5188 msec
All tasks sent!