UDP GSO + sendmmsg加速数据发送示例

/*************************************************************************
    > File Name: udp_server.cc
    > Author: hsz
    > Brief:
    > Created Time: Mon 24 Nov 2025 04:00:40 PM CST
 ************************************************************************/

#include <iostream>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <cstring>
#include <vector>
#include <errno.h>

#define MAX_BATCH 32           // recvmmsg每批最多收多少包
#define BUF_SIZE  (16 * 1024)  // 每包最大缓存（GRO模式可设大一点）

int main()
{
    // 创建UDP socket
    int sock = socket(AF_INET, SOCK_DGRAM, 0);
    if (sock < 0) {
        perror("socket error");
        return 1;
    }

    // 设置地址和端口
    sockaddr_in serv_addr{};
    serv_addr.sin_family = AF_INET;
    serv_addr.sin_port = htons(9000);
    serv_addr.sin_addr.s_addr = INADDR_ANY; // 0.0.0.0

    // 绑定
    if (bind(sock, (sockaddr*)&serv_addr, sizeof(serv_addr)) < 0) {
        perror("bind error");
        close(sock);
        return 1;
    }
    std::cout << "UDP服务器已启动, 监听 0.0.0.0:9000\n";

    // 批量收包结构体
    struct mmsghdr msgs[MAX_BATCH];
    struct msghdr hdrs[MAX_BATCH];
    struct iovec iovs[MAX_BATCH];
    std::vector<std::vector<char>> bufs(MAX_BATCH, std::vector<char>(BUF_SIZE));
    sockaddr_in peer_addrs[MAX_BATCH];
    memset(msgs, 0, sizeof(msgs));
    memset(hdrs, 0, sizeof(hdrs));
    memset(peer_addrs, 0, sizeof(peer_addrs));

    for (int i = 0; i < MAX_BATCH; ++i) {
        iovs[i].iov_base = bufs[i].data();
        iovs[i].iov_len  = BUF_SIZE;
        hdrs[i].msg_name = &peer_addrs[i];
        hdrs[i].msg_namelen = sizeof(sockaddr_in);
        hdrs[i].msg_iov = &iovs[i];
        hdrs[i].msg_iovlen = 1;
        msgs[i].msg_hdr = hdrs[i];
    }

    // 主循环批量收包
    while (true) {
        // 非阻塞套接字会尽可能的多读取数据包
        // 阻塞套接字则会等到 MAX_BATCH 个才会返回, 故阻塞套接字建议设置合理的超时
        struct timespec timeout{};  
        timeout.tv_sec = 1;
        timeout.tv_nsec = 0;
        int n = recvmmsg(sock, msgs, MAX_BATCH, MSG_DONTWAIT, &timeout);
        if (n < 0) {
            if (errno == EINTR || errno == ETIMEDOUT || errno == EAGAIN) {
                usleep(100000); // 100ms
                continue;
            }
            perror("recvmmsg error");
            break;
        }

        printf("\n本次收到 %d 个数据包\n", n);
        for (int i = 0; i < n; ++i) {
            char addrbuf[64]{};
            inet_ntop(AF_INET, &(peer_addrs[i].sin_addr), addrbuf, sizeof(addrbuf));
            printf("收到数据包，来自 %s:%d, 长度 %u\n",
                   addrbuf, ntohs(peer_addrs[i].sin_port), msgs[i].msg_len);
            // 如需解析内容，可：bufs[i].data(), msgs[i].msg_len
        }
    }

    close(sock);
    return 0;
}

/*************************************************************************
    > File Name: udp_client.cc
    > Author: eular
    > Brief:
    > Created Time: Mon 24 Nov 2025 04:00:40 PM CST
 ************************************************************************/

#include <cstring>
#include <iostream>
#include <vector>
#include <algorithm>

#include <unistd.h>
#include <linux/udp.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>

#ifndef UDP_SEGMENT
#define UDP_SEGMENT 103
#endif

#ifndef SOL_UDP
#define SOL_UDP 17
#endif


int main()
{
    // 配置参数
    const size_t TOTAL_SIZE = 32 * 1024 - 1400; // 128KB
    const int MTU = 1500;
    const int IP_UDP_HDR = 20 + 8 + 8;
    const int GSO_PAYLOAD = MTU - IP_UDP_HDR; // 1472
    const size_t BATCH_SIZE = 32; // 每批最多发多少mmsghdr
    const char* DST_IP = "127.0.0.1";
    const int DST_PORT = 9000;

    // 准备数据缓存，内容全置'A'
    std::vector<char> send_buf(TOTAL_SIZE, 'A');

    // 目标地址
    sockaddr_in dst_addr{};
    dst_addr.sin_family = AF_INET;
    dst_addr.sin_port = htons(DST_PORT);
    inet_pton(AF_INET, DST_IP, &dst_addr.sin_addr);

    // 建立UDP socket
    int sock = socket(AF_INET, SOCK_DGRAM, 0);
    if (sock < 0) {
        perror("socket");
        return 1;
    }

    size_t pos = 0;
    size_t msg_count = 0;
    while (pos < TOTAL_SIZE) {
        size_t remain = TOTAL_SIZE - pos;
        size_t segment_size = std::min(remain, size_t(GSO_PAYLOAD * BATCH_SIZE));
        size_t batch_msgs = (segment_size + GSO_PAYLOAD - 1) / GSO_PAYLOAD;

        // mmsghdr批次
        std::vector<mmsghdr> mms(batch_msgs);
        std::vector<msghdr> hdrs(batch_msgs);
        std::vector<iovec> iovs(batch_msgs);
        std::vector<char> cmsg_bufs(batch_msgs * CMSG_SPACE(sizeof(uint16_t)));

        for (size_t i = 0; i < batch_msgs; ++i) {
            size_t this_size = std::min(remain, size_t(GSO_PAYLOAD));
            iovs[i].iov_base = send_buf.data() + pos;
            iovs[i].iov_len  = this_size;

            hdrs[i].msg_name    = &dst_addr;
            hdrs[i].msg_namelen = sizeof(dst_addr);
            hdrs[i].msg_iov     = &iovs[i];
            hdrs[i].msg_iovlen  = 1;

            // UDP_SEGMENT设置
            memset(&cmsg_bufs[i * CMSG_SPACE(sizeof(uint16_t))], 0, CMSG_SPACE(sizeof(uint16_t)));
            hdrs[i].msg_control = &cmsg_bufs[i * CMSG_SPACE(sizeof(uint16_t))];
            hdrs[i].msg_controllen = CMSG_SPACE(sizeof(uint16_t));
            cmsghdr* cmsg = CMSG_FIRSTHDR(&hdrs[i]);
            cmsg->cmsg_level = SOL_UDP;
            cmsg->cmsg_type  = UDP_SEGMENT;
            cmsg->cmsg_len   = CMSG_LEN(sizeof(uint16_t));
            uint16_t gso_val = GSO_PAYLOAD;
            memcpy(CMSG_DATA(cmsg), &gso_val, sizeof(uint16_t));

            mms[i].msg_hdr = hdrs[i];
            mms[i].msg_len = 0;

            pos += this_size;
            remain -= this_size;
        }

        // 批量发送
        int ret = sendmmsg(sock, mms.data(), batch_msgs, 0);
        if (ret < 0) {
            perror("sendmmsg");
            close(sock);
            return 1;
        }
        msg_count += ret;
        for (int i = 0; i < ret; ++i) {
            printf("Sent message %d, size = %d\n", i, mms[i].msg_len);
        }
        printf("This time sending the fragmented GSO packages: %d\n", ret);
    }
    std::cout << "全部分片包批量发送完成，共计 GSO 包数：" << msg_count << std::endl;

    close(sock);
    return 0;
}

https://calendar.perfplanet.com/2019/accelerating-udp-packet-transmission-for-quic/