探讨 dpdk 中几处原子变量与原子操作用户多线程同步场景

问题描述

最近有同事问我，dpdk-19.11 版本在一个进程中多次调用 rte_eal_init 是否会存在问题。我确信 16.04 中是不能多次调用 rte_eal_init 的，但是 19.11 中能不能倒没有研究过，于是阅读了下代码，发现并不会存在问题，其中着实有一些小技巧，在本文中记录一下。

在一个进程中多次调用 rte_eal_init

dpdk-19.11 的 rte_eal_init 函数起始位置有如下代码：

	static rte_atomic32_t run_once = RTE_ATOMIC32_INIT(0);
	...................................................
    if (!rte_atomic32_test_and_set(&run_once)) {
        rte_eal_init_alert("already called initialization.");
        rte_errno = EALREADY;
        return -1;
    }

run_once 被定义为静态变量具有全局声明周期，当进程调用了一次 rte_eal_init 后，run_once 原子变量将会被设置为 1，此后再次调用 rte_eal_init 函数时，由于 run_once 已经为 1，此时 rte_atomic32_test_set 会返回 0，rte_eal_init 函数立刻返回。同时用户可以通过获取 rte_errno 的值，判断是否为 EALREADY 来确定是否已经初始化过。

这里使用原子变量与原子操作保证了多线程中同时调用 rte_eal_init 也不会存在问题。

问题扩展：巧用原子变量解决多线程同步问题

pthread_barrier_wait 函数的返回值

manual 中对其返回值的描述：

       Upon successful completion, thepthread_barrier_wait() function
       shall return PTHREAD_BARRIER_SERIAL_THREAD for a single
       (arbitrary) thread synchronized at the barrier and zero for each
       of the other threads. Otherwise, an error number shall be
       returned to indicate the error.

只有一个线程会返回 PTHREAD_BARRIER_SERIAL_THREAD，其它使用线程会返回 0。

dpdk 中使用 pthread 屏障处理同步问题

dpdk-19.11 中控制线程的创建使用如下接口：

static void *rte_thread_init(void *arg)
{
	int ret;
	struct rte_thread_ctrl_params *params = arg;
	void *(*start_routine)(void *) = params->start_routine;
	void *routine_arg = params->arg;

	ret = pthread_barrier_wait(&params->configured);
	if (ret == PTHREAD_BARRIER_SERIAL_THREAD) {
		pthread_barrier_destroy(&params->configured);
		free(params);
	}

	return start_routine(routine_arg);
}

int
rte_ctrl_thread_create(pthread_t *thread, const char *name,
		const pthread_attr_t *attr,
		void *(*start_routine)(void *), void *arg)
{
	rte_cpuset_t *cpuset = &internal_config.ctrl_cpuset;
	struct rte_thread_ctrl_params *params;
	int ret;

	params = malloc(sizeof(*params));
	if (!params)
		return -ENOMEM;

	params->start_routine = start_routine;
	params->arg = arg;

	pthread_barrier_init(&params->configured, NULL, 2);

	ret = pthread_create(thread, attr, rte_thread_init, (void *)params);
	if (ret != 0) {
		free(params);
		return -ret;
	}

	if (name != NULL) {
		ret = rte_thread_setname(*thread, name);
		if (ret < 0)
			RTE_LOG(DEBUG, EAL,
				"Cannot set name for ctrl thread\n");
	}

	ret = pthread_setaffinity_np(*thread, sizeof(*cpuset), cpuset);
	if (ret)
		goto fail;

	ret = pthread_barrier_wait(&params->configured);
	if (ret == PTHREAD_BARRIER_SERIAL_THREAD) {
		pthread_barrier_destroy(&params->configured);
		free(params);
	}

	return 0;
................................................................
}

上述代码核心需求是保证在 rte_ctrl_thread_create 函数返回前，创建的线程已经初始化完成。为了实现这一需求，它使用了 pthread 屏障来实现。

屏障初始值设置为 2，则仅当两次调用 pthread_barrier_wait 后线程才会继续向下执行。根据 pthread_barrier_wait 函数的返回值说明，只有一个线程会返 PTHREAD_BARRIER_SERIAL_THREAD，这样只需要判断该返回值，然后释放屏障与动态分配的参数即可，看上去没啥问题。

在将上述代码移植到 dpdk 低版本进行测试时，发现上述代码会偶现段错误。查看 dpdk git log 发现高版本已经修改了此问题，相关的 commit 信息如下：

commit 34cc55cce6b180a6c3ee3fcf70a0fd56927f240d
Author: Luc Pelletier <lucp.at.work@gmail.com>
Date:   Wed Apr 7 16:16:04 2021 -0400

    eal: fix race in control thread creation
    
    The creation of control threads uses a pthread barrier for
    synchronization. This patch fixes a race condition where the pthread
    barrier could get destroyed while one of the threads has not yet
    returned from the pthread_barrier_wait function, which could result in
    undefined behaviour.
    
    Fixes: 3a0d465d4c53 ("eal: fix use-after-free on control thread creation")
    Cc: stable@dpdk.org
    
    Signed-off-by: Luc Pelletier <lucp.at.work@gmail.com>
    Acked-by: Olivier Matz <olivier.matz@6wind.com>
    Reviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>

commit 描述提及，此处代码存在竞争条件。当一个线程还没有从 pthread_barrier_wait 函数中返回时屏障就已经被销毁时会导致未定义的行为。

修复后的代码如下：

static void ctrl_params_free(struct rte_thread_ctrl_params *params)
{
	if (__atomic_sub_fetch(&params->refcnt, 1, __ATOMIC_ACQ_REL) == 0) {
		pthread_barrier_destroy(&params->configured);
		free(params);
	}
}

static void *ctrl_thread_init(void *arg)
{
	struct internal_config *internal_conf =
		eal_get_internal_configuration();
	rte_cpuset_t *cpuset = &internal_conf->ctrl_cpuset;
	struct rte_thread_ctrl_params *params = arg;
	void *(*start_routine)(void *) = params->start_routine;
	void *routine_arg = params->arg;

	__rte_thread_init(rte_lcore_id(), cpuset);

	pthread_barrier_wait(&params->configured);
	ctrl_params_free(params);

	return start_routine(routine_arg);
}

int
rte_ctrl_thread_create(pthread_t *thread, const char *name,
		const pthread_attr_t *attr,
		void *(*start_routine)(void *), void *arg)
{
	struct internal_config *internal_conf =
		eal_get_internal_configuration();
	rte_cpuset_t *cpuset = &internal_conf->ctrl_cpuset;
	struct rte_thread_ctrl_params *params;
	int ret;

	params = malloc(sizeof(*params));
	if (!params)
		return -ENOMEM;

	params->start_routine = start_routine;
	params->arg = arg;
	params->refcnt = 2;

	ret = pthread_barrier_init(&params->configured, NULL, 2);
	if (ret != 0) {
		free(params);
		return -ret;
	}

	ret = pthread_create(thread, attr, ctrl_thread_init, (void *)params);
	if (ret != 0)
		goto fail;

	if (name != NULL) {
		ret = rte_thread_setname(*thread, name);
		if (ret < 0)
			RTE_LOG(DEBUG, EAL,
				"Cannot set name for ctrl thread\n");
	}

	ret = pthread_setaffinity_np(*thread, sizeof(*cpuset), cpuset);
	if (ret != 0)
		goto fail_cancel;

	pthread_barrier_wait(&params->configured);
	ctrl_params_free(params);

	return 0;
...............................................................

修改后的代码继续使用 pthread 屏障确保新创建的线程在函数返回前正确初始化，同时使用原子操作来保证屏障正常销毁。ctrl_params_free 函数调用中使用原子操作，保证了多线程并发执行时数据的一致性，同时 ctrl_params_free 函数调用语句放在 pthread_barrier_wait 函数调用后，确保释放时没有一个线程在使用，不会出现上述竞争条件。

后续版本又对此处代码进行了改进，移除了 pthread 屏障，改为完全使用原子操作保证程序正常运行。修改后的代码如下：

static void *ctrl_thread_init(void *arg)
{
	struct internal_config *internal_conf =
		eal_get_internal_configuration();
	rte_cpuset_t *cpuset = &internal_conf->ctrl_cpuset;
	struct rte_thread_ctrl_params *params = arg;
	void *(*start_routine)(void *) = params->start_routine;
	void *routine_arg = params->arg;

	__rte_thread_init(rte_lcore_id(), cpuset);
	params->ret = pthread_setaffinity_np(pthread_self(), sizeof(*cpuset),
		cpuset);
	if (params->ret != 0) {
		__atomic_store_n(&params->ctrl_thread_status,
			CTRL_THREAD_ERROR, __ATOMIC_RELEASE);
		return NULL;
	}

	__atomic_store_n(&params->ctrl_thread_status,
		CTRL_THREAD_RUNNING, __ATOMIC_RELEASE);

	return start_routine(routine_arg);
}

int
rte_ctrl_thread_create(pthread_t *thread, const char *name,
		const pthread_attr_t *attr,
		void *(*start_routine)(void *), void *arg)
{
	struct rte_thread_ctrl_params *params;
	enum __rte_ctrl_thread_status ctrl_thread_status;
	int ret;

	params = malloc(sizeof(*params));
	if (!params)
		return -ENOMEM;

	params->start_routine = start_routine;
	params->arg = arg;
	params->ret = 0;
	params->ctrl_thread_status = CTRL_THREAD_LAUNCHING;

	ret = pthread_create(thread, attr, ctrl_thread_init, (void *)params);
	if (ret != 0) {
		free(params);
		return -ret;
	}

	if (name != NULL) {
		ret = rte_thread_setname(*thread, name);
		if (ret < 0)
			RTE_LOG(DEBUG, EAL,
				"Cannot set name for ctrl thread\n");
	}

	/* Wait for the control thread to initialize successfully */
	while ((ctrl_thread_status =
			__atomic_load_n(&params->ctrl_thread_status,
			__ATOMIC_ACQUIRE)) == CTRL_THREAD_LAUNCHING) {
		/* Yield the CPU. Using sched_yield call requires maintaining
		 * another implementation for Windows as sched_yield is not
		 * supported on Windows.
		 */
		rte_delay_us_sleep(1);
	}

	/* Check if the control thread encountered an error */
	if (ctrl_thread_status == CTRL_THREAD_ERROR) {
		/* ctrl thread is exiting */
		pthread_join(*thread, NULL);
	}

	ret = params->ret;
	free(params);

	return -ret;
}

主线程中使用 while 循环不断检查原子变量的值，新创建的线程负责修改原子变量的值，当原子变量的值被修改后，主线程检查到后根据值进行处理，释放相应资源，使用原子变量与原子操作保障了此过程的正确性。