/**
* Copyright (C) 2024 TP-Link. All rights reserved.
*/
#include <unistd.h>
#include "aqi_tpaecns.h"
#include "audio_aec.h"
#include "audio_fft_utils.h"
#include "signal_processing_library.h"
#include "webrtc_vad.h"
#ifdef TP_AGC_SUPPORT
#include "audio_agc.h"
#endif
#define AUDIO_PROFILE_NS_PARAM_PATH "/audio_profile/ns_param"
#define AUDIO_PROFILE_AEC3_PARAM_PATH "/audio_profile/aec3_param"
#ifdef TP_AGC_SUPPORT
#define AUDIO_PROFILE_AGC_PARAM_PATH "/audio_profile/tpagc_param"
#endif
extern TP_AUDIO_ATTR_S g_audio_attr;
typedef enum
{
OFF = 0,
ON = 1
} ENBLE_TYPE_E;
typedef enum
{
MOTOR_IDLE = 0,
PAN_MOVING = 1,
TILT_MOVING = 2,
ZOOM_MOVING = 3,
AF_MOVING = 4
} MOTOR_STATUS_TYPE_E;
typedef enum
{
NO_DELAY = ERROR, /* 不应从零开始 */
PAN_DELAY = 1,
TILT_DELAY = 2,
ZOOM_DELAY = 3,
AF_DELAY = 4
} DELAY_TYPE_E;
typedef struct _AECNS_CONTEXT
{
U64 frame_count; /* 帧计数器 */
U64 motor_delay_timer; /* 电机惯性噪声延迟计时器 */
S32 curr_motor_type; /* 当前算法参数对应的电机类型 */
S32 curr_delay_type; /* 当前申请延迟降噪算法的电机类型 */
S32 prev_ptz_type; /* 先前PTZ电机类型 */
PTZ_STATE_TYPE_E curr_ptz_state; /* 当前ptz电机状态 */
AF_STATE_TYPE_E curr_af_state; /* 当前af状态 */
ENBLE_TYPE_E ns_enable; /* 对应于网页端的降噪开关 */
ENBLE_TYPE_E aec_enable;
AUDIO_COMMON_CONTEXT *p_near_common_context;
AEC_CONTEXT *p_aec_context;
short aec_out_buf[D_INPUT_FRAME_MAX_LEN];
#ifdef TP_AGC_SUPPORT
AGC_CONTEXT *p_agc_context;
short agc_out_buf[D_INPUT_FRAME_MAX_LEN];
#endif
S16 *data_tmp;
S16 init_state;
} AECNS_CONTEXT;
LOCAL AECNS_SYS_CONFIG g_aecns_sys_config = { 0 };
LOCAL AEC3_SYS_CONFIG g_aec3_sys_config = { 0 };
#ifdef TP_AGC_SUPPORT
LOCAL AGC_SYS_CONFIG g_agc_sys_config = { 0 };
#endif
LOCAL AECNS_CONTEXT *gp_aecns_context = NULL;
pthread_mutex_t g_aecns_mutex_lock = PTHREAD_MUTEX_INITIALIZER;
/* ===== 新增:VAD 实例与跨回调共享标志(由 process 更新,ptz 回调读取) ===== */
static VadInst *vad_inst = NULL;
static volatile int vad_has_voice = 0; /* 0: 无人声;1: 有人声 */
static int vad_mode_default = 2; /* 0~3,数值越大越激进 */
/* ---- VAD 统计窗口(30 帧) ---- */
static int vad_win_cnt = 0; /* 当前窗口累计的帧数 [0..30) */
static int vad_hit_cnt = 0; /* 当前窗口内“命中帧”数(vad_voice_count > 5 的帧数) */
#define VAD_STAT_WINDOW_FRAMES 30 /* 统计窗口长度:30 帧 */
#define VAD_PERFRAME_HIT_THRESHOLD 5 /* 单帧内命中块阈值:>5 视作该帧有人声 */
#define VAD_REQUIRED_HITS 3 /* 一个 30帧窗口内至少 3 帧命中才认定“有人声” */
/**********************************************************************
Function: aecns_stop_timer
Description:
电机降噪算法延迟定时器停止计时
Input:
@timer: 定时器指针
Return: return OK if success, ERROR if fail.
************************************************************************/
LOCAL S32 aecns_stop_timer(U64 *timer)
{
if (!timer)
{
return ERROR;
}
if (ERROR != (*timer))
{
if (ERROR == inet_del_timer((*timer)))
{
AMS_ERROR("delete motor delay timer wrong");
return ERROR;
}
}
*timer = ERROR;
return OK;
}
#ifndef AEC_ARGS_MOTOR_TYPE_SUPPORT
/* =========================
* 电机状态回调(主进程)
* 与条件门控:仅当 (电机非 idle) && (VAD 判定有人声) 才把“电机动”传给 AEC/NS;
* 否则传/保持 idle。
* ========================= */
LOCAL S32 aecns_ptz_callback(dms_handler_t *handler, U8 *mbuf, U32 mlen, U32 sender_dms_id)
{
PTZ_STATUS_EVENT_MSG *ptz_event_msg = (PTZ_STATUS_EVENT_MSG *)mbuf;
MOTOR_STATUS_TYPE_E motor_type_to_set = MOTOR_IDLE;
if (!ptz_event_msg || mlen != sizeof(PTZ_STATUS_EVENT_MSG)) {
AMS_ERROR("aecns_ptz_callback invalid msg");
return ERROR;
}
/* 电机是否非 idle(原逻辑:ptz_state != 0 视为转动) */
const int ptz_is_moving = (ptz_event_msg->ptz_state != 0);
/* 与条件:电机非 idle 且 VAD 有人声 才让 AEC/NS 进入“电机动”模式;
否则传 idle,避免仅因电机动而切至强降噪 */
if (ptz_is_moving && vad_has_voice) {
motor_type_to_set = PAN_MOVING;
} else {
motor_type_to_set = MOTOR_IDLE;
}
if (NULL == gp_aecns_context || NULL == gp_aecns_context->p_aec_context) {
AMS_ERROR("aecns_set_motor_status failed: context null");
return ERROR;
}
gp_aecns_context->p_aec_context->den->motor_state = motor_type_to_set;
AMS_DEBUG("[PTZ_CB] ptz_state=%d, vad=%d -> motor_state=%d",
(int)ptz_event_msg->ptz_state, vad_has_voice, (int)motor_type_to_set);
}
#endif
#ifdef BUILD_AMS_MINI
LOCAL S32 ams_mini_aecns_ptz_callback(dms_handler_t *handler, U8 *mbuf, U32 mlen, U32 sender_dms_id)
{
TP_SYNC_SERVER_MSG *msg = (TP_SYNC_SERVER_MSG *)mbuf;
MOTOR_STATUS_TYPE_E motor_type_to_set = MOTOR_IDLE;
if (!msg || mlen != sizeof(TP_SYNC_SERVER_MSG)) {
AMS_ERROR("ams_mini_aecns_ptz_callback invalid msg");
return ERROR;
}
PTZ_STATE_TYPE_E *ptz_state = NULL;
/* 仅当消息类型为 PTZ 通知时解析电机状态 */
if (msg->msg_id == TSS_MSG_PTZ_STATE_NOTIFY) {
ptz_state = (PTZ_STATE_TYPE_E *)msg->msg_buf;
} else {
/* 非电机消息:不改 motor_state */
return OK;
}
const int ptz_is_moving = (ptz_state && (*ptz_state != 0));
if (ptz_is_moving && vad_has_voice) {
motor_type_to_set = PAN_MOVING;
} else {
motor_type_to_set = MOTOR_IDLE;
}
if (NULL == gp_aecns_context || NULL == gp_aecns_context->p_aec_context) {
AMS_ERROR("aecns_set_motor_status failed: context null");
return ERROR;
}
gp_aecns_context->p_aec_context->den->motor_state = motor_type_to_set;
AMS_DEBUG("[PTZ_CB_MINI] ptz_state=%d, vad=%d -> motor_state=%d",
(ptz_state ? (int)(*ptz_state) : -1), vad_has_voice, (int)motor_type_to_set);
return OK;
}
#endif
LOCAL void aecns_load_config()
{
/* 获取云台转动的降噪参数 */
if (0 == AMS_READ(AUDIO_PROFILE_NS_PARAM_PATH, &g_aecns_sys_config, sizeof(AECNS_SYS_CONFIG)))
{
AMS_WARN("read AUDIO_PROFILE_NS_PARAM_PATH fail");
}
if (0 == AMS_READ(AUDIO_PROFILE_AEC3_PARAM_PATH, &g_aec3_sys_config, sizeof(AEC3_SYS_CONFIG)))
{
AMS_WARN("[AMS] read AUDIO_PROFILE_AEC3_PARAM_PATH fail");
}
#ifdef TP_AGC_SUPPORT
if (0 == AMS_READ(AUDIO_PROFILE_AGC_PARAM_PATH, &g_agc_sys_config, sizeof(AGC_SYS_CONFIG)))
{
AMS_WARN("[AMS] read AUDIO_PROFILE_AGC_PARAM_PATH fail");
}
#endif
#ifdef AQI_DEBUG
else
{
AMS_DEBUG("read PAN: ns_mode:%d ns_intensity:%d fGain:%f", g_aecns_sys_config.pan_ns_mode, g_aecns_sys_config.pan_ns_intensity, g_aecns_sys_config.pan_ns_gain);
AMS_DEBUG("read TILT: ns_mode:%d ns_intensity:%d fGain:%f", g_aecns_sys_config.tilt_ns_mode, g_aecns_sys_config.tilt_ns_intensity, g_aecns_sys_config.tilt_ns_gain);
AMS_DEBUG("read ZOOM: ns_mode:%d ns_intensity:%d fGain:%f", g_aecns_sys_config.zoom_ns_mode, g_aecns_sys_config.zoom_ns_intensity, g_aecns_sys_config.zoom_ns_gain);
AMS_DEBUG("read FOCUS: ns_mode:%d ns_intensity:%d fGain:%f", g_aecns_sys_config.focus_ns_mode, g_aecns_sys_config.focus_ns_intensity, g_aecns_sys_config.focus_ns_gain);
// NONE MOTOR NS
AMS_DEBUG("read NONE_MOTOR: ns_mode:%d ns_intensity:%d fGain:%f", g_aecns_sys_config.none_motor_mode, g_aecns_sys_config.none_motor_ns_intensity, g_aecns_sys_config.none_motor_ns_gain);
// AEC
AMS_DEBUG("read ECHO_INTENSITY: aec_intensity:%d", g_aecns_sys_config.echo_intensity);
AMS_DEBUG("read AEC_GAIN: aec_gain:%d", g_aecns_sys_config.aec_gain);
}
#endif
/* 配置信息检查:当配置参数未进行配置或读取配置失败时,参数值是0,进行默认值配置 */
/* 水平方向 */
if ((g_aecns_sys_config.pan_ns_mode < 0) || (g_aecns_sys_config.pan_ns_mode > 4))
{
AMS_WARN("Illegal pan_ns_mode! pan_ns_mode set default: 2");
g_aecns_sys_config.pan_ns_mode = 2; /* 噪声概率为1.0的降噪 */
}
if ((g_aecns_sys_config.pan_ns_intensity < 0) || (g_aecns_sys_config.pan_ns_intensity > 4))
{
AMS_WARN("Illegal pan_ns_intensity! pan_ns_intensity set default: 2");
g_aecns_sys_config.pan_ns_intensity = 3;
}
if (g_aecns_sys_config.pan_ns_gain <= 0.0)
{
AMS_WARN("Illegal pan_ns_mode! pan_ns_mode set default: 2.0");
g_aecns_sys_config.pan_ns_gain = 2.0;
}
/* 垂直方向 */
if ((g_aecns_sys_config.tilt_ns_mode < 0) || (g_aecns_sys_config.tilt_ns_mode > 4))
{
AMS_WARN("Illegal tilt_ns_mode! tilt_ns_mode set default: 3");
g_aecns_sys_config.tilt_ns_mode = 3; /* 噪声概率为1.0的降噪,并经过低通滤波 */
}
if ((g_aecns_sys_config.tilt_ns_intensity < 0) || (g_aecns_sys_config.tilt_ns_intensity > 4))
{
AMS_WARN("Illegal tilt_ns_intensity! tilt_ns_intensity set default: 4");
g_aecns_sys_config.tilt_ns_intensity = 4;
}
if (g_aecns_sys_config.tilt_ns_gain <= 0.0)
{
AMS_WARN("Illegal tilt_ns_gain! tilt_ns_gain set default: 2.0");
g_aecns_sys_config.tilt_ns_gain = 2.0;
}
/* ZOOM */
if ((g_aecns_sys_config.zoom_ns_mode < 0) || (g_aecns_sys_config.zoom_ns_mode > 4))
{
AMS_WARN("Illegal zoom_ns_mode! zoom_ns_mode set default: 2");
g_aecns_sys_config.zoom_ns_mode = 2; /* 噪声概率为1.0的降噪 */
}
if ((g_aecns_sys_config.zoom_ns_intensity < 0) || (g_aecns_sys_config.zoom_ns_intensity > 4))
{
AMS_WARN("Illegal zoom_ns_intensity! zoom_ns_intensity set default: 3");
g_aecns_sys_config.zoom_ns_intensity = 3;
}
if (g_aecns_sys_config.zoom_ns_gain <= 0.0)
{
AMS_WARN("Illegal zoom_ns_gain! zoom_ns_gain set default: 2.0");
g_aecns_sys_config.zoom_ns_gain = 2.0;
}
/* AUTO FOCUS */
if ((g_aecns_sys_config.focus_ns_mode < 0) || (g_aecns_sys_config.focus_ns_mode > 4))
{
AMS_WARN("Illegal focus_ns_mode! focus_ns_mode set default: 2");
g_aecns_sys_config.focus_ns_mode = 2; /* 噪声概率为1.0的降噪 */
}
if ((g_aecns_sys_config.focus_ns_intensity < 0) || (g_aecns_sys_config.focus_ns_intensity > 4))
{
AMS_WARN("Illegal focus_ns_intensity! focus_ns_intensity set default: 3");
g_aecns_sys_config.focus_ns_intensity = 3;
}
if (g_aecns_sys_config.focus_ns_gain <= 0.0)
{
AMS_WARN("Illegal focus_ns_gain! focus_ns_gain set default: 2.0");
g_aecns_sys_config.focus_ns_gain = 2.0;
}
/* None Motor */
if ((g_aecns_sys_config.none_motor_mode < 0) || (g_aecns_sys_config.none_motor_mode > 4))
{
AMS_WARN("Illegal none_motor_mode! none_motor_mode set default: 1");
g_aecns_sys_config.none_motor_mode = 1;
}
if ((g_aecns_sys_config.none_motor_ns_intensity < 0) || (g_aecns_sys_config.none_motor_ns_intensity > 4))
{
AMS_WARN("Illegal none_motor_ns_intensity! none_motor_ns_intensity set default: 2");
g_aecns_sys_config.none_motor_ns_intensity = 2;
}
if (g_aecns_sys_config.none_motor_ns_gain <= 0.0)
{
AMS_WARN("Illegal none_motor_gain! none_motor_gain set default: 1.0");
g_aecns_sys_config.none_motor_ns_gain = 1.0;
}
/* AEC */
if ((g_aecns_sys_config.echo_intensity < 0) || (g_aecns_sys_config.echo_intensity > 6))
{
AMS_WARN("Illegal echo_intensity! echo_intensity set default: 3");
g_aecns_sys_config.echo_intensity = 3;
}
if ((g_aecns_sys_config.aec_gain < 0.001) && (g_aecns_sys_config.aec_gain > -0.001))
{
AMS_WARN("Illegal echo_intensity! aec_gain set default: 1.3");
g_aecns_sys_config.aec_gain = 1.3;
}
return;
};
S32 aecns_init(AEC_INIT_ARGS *args)
{
S32 sample_rate = 8000;
S32 init = 1;
if (NULL == args)
{
AMS_ERROR("init aecInst failed: args is NULL");
return ERROR;
}
// add mutext lock
pthread_mutex_lock(&g_aecns_mutex_lock);
init = args->init;
if (init)
{
if (TP_AUDIO_SAMPLING_FREQ_8000HZ == args->sample_rate)
{
sample_rate = 8000;
}
else if (TP_AUDIO_SAMPLING_FREQ_16000HZ == args->sample_rate)
{
sample_rate = 16000; /* 16kHz采样,fft按长度256处理 */
}
else
{
AMS_ERROR("Invalid sample_rate");
pthread_mutex_unlock(&g_aecns_mutex_lock);
return ERROR;
}
AMS_DEBUG("sample_rate:%d", sample_rate);
/* 读取用户配置 */
aecns_load_config();
// create global gp_aecns_context
if (NULL == gp_aecns_context)
{
gp_aecns_context = (AECNS_CONTEXT *)calloc(1, sizeof(AECNS_CONTEXT));
if (NULL == gp_aecns_context)
{
AMS_ERROR("init error: cannot create gp_aecns_context");
goto error_exit;
}
}
if (1 == gp_aecns_context->init_state)
{
pthread_mutex_unlock(&g_aecns_mutex_lock);
return OK;
}
gp_aecns_context->ns_enable = 0; /* 默认不开启ns分支 */
AMS_DEBUG("anr_on: %d", gp_aecns_context->ns_enable);
gp_aecns_context->frame_count = 0;
gp_aecns_context->aec_enable = OFF; /* 回声消除默认关闭,等process时根据远端数据来决定是否开启 */
gp_aecns_context->curr_af_state = AF_STATE_TYPE_IDLE; /* af状态为空闲 */
gp_aecns_context->motor_delay_timer = ERROR; /* 电机降噪算法延迟关闭 */
gp_aecns_context->curr_ptz_state = PTZ_STATE_TYPE_IDLE;
gp_aecns_context->curr_motor_type = MOTOR_IDLE;
gp_aecns_context->curr_delay_type = NO_DELAY;
// create common context
if (NULL == gp_aecns_context->p_near_common_context)
{
gp_aecns_context->p_near_common_context = (AUDIO_COMMON_CONTEXT *)calloc(1, sizeof(AUDIO_COMMON_CONTEXT));
}
// create aec context
if (NULL == gp_aecns_context->p_aec_context)
{
gp_aecns_context->p_aec_context = (AEC_CONTEXT *)calloc(1, sizeof(AEC_CONTEXT));
}
#ifdef TP_AGC_SUPPORT
// create agc context
if (NULL == gp_aecns_context->p_agc_context)
{
gp_aecns_context->p_agc_context = (AGC_CONTEXT *)calloc(1, sizeof(AGC_CONTEXT));
}
#endif
if (NULL == gp_aecns_context->data_tmp)
{
gp_aecns_context->data_tmp = (S16 *)calloc(2048, sizeof(S16));
}
if (NULL == gp_aecns_context->p_near_common_context)
{
AMS_ERROR("init error: cannot create common/aec/ns contexts");
goto error_exit;
}
// init common context
if (ERROR == audio_common_init(gp_aecns_context->p_near_common_context, sample_rate))
{
AMS_ERROR("audio_common_init: gp_aecns_context->p_near_common_context");
goto error_exit;
}
// init aec context
if (ERROR == aec_module_init(gp_aecns_context->p_aec_context, sample_rate, args->use_timestamp, &g_aec3_sys_config))
{
AMS_ERROR("aec_module_init error");
goto error_exit;
}
#ifdef TP_AGC_SUPPORT
// init agc context
if (ERROR == agc_init(gp_aecns_context->p_agc_context, 0, 255, AGC_MODE_FIXED_DIGITAL, sample_rate, &g_agc_sys_config))
{
AMS_ERROR("agc_module_init error");
goto error_exit;
}
#endif
/***
ns_set_attr(gp_aecns_context->p_ns_context, g_aecns_sys_config.none_motor_mode, g_aecns_sys_config.none_motor_ns_intensity, g_aecns_sys_config.none_motor_ns_gain);
***/
#ifndef AEC_ARGS_MOTOR_TYPE_SUPPORT
/* 关联电机事件 rtos和tp_server采用不同的解决方案,rtos由mpp层传递args->motor_type参数,不监听消息, 在主进程内进行 */
if (ERROR == interface_msg_attach_handler(PTZ_STATUS_EVENT, aecns_ptz_callback))
{
AMS_ERROR("msg attach handler for PTZ_STATUS_EVENT failed");
goto error_exit;
}
#endif
#ifdef BUILD_AMS_MINI
/* tp_setver增加一个监听的消息,pir唤醒同时监听小进程和主进程的消息,在小进程里进行, 主进程依然监听原来的消息 */
if (ERROR == interface_msg_attach_handler(TP_SYNC_SERVER_MSG_ID, ams_mini_aecns_ptz_callback))
{
AMS_ERROR("msg attach handler for TP_SYNC_SERVER_MSG_ID failed");
goto error_exit;
}
#endif
/* VAD集成 */
if (vad_inst == NULL) {
vad_inst = WebRtcVad_Create();
if (vad_inst == NULL) {
AMS_ERROR("vad_create error");
goto error_exit;
}
}
if (ERROR == WebRtcVad_Init(vad_inst)) {
AMS_ERROR("vad_init error");
goto error_exit;
}
/* 使用默认 2;可按需修改为 0~3 */
if (ERROR == WebRtcVad_set_mode(vad_inst, vad_mode_default)) {
AMS_WARN("vad_set_mode error, fallback to mode=2");
WebRtcVad_set_mode(vad_inst, 2);
}
vad_has_voice = 0;
vad_win_cnt = 0;
vad_hit_cnt = 0;
/* 关联 auto focus 事件 */
/***
if (ERROR == interface_msg_attach_handler(AF_STATUS_EVENT, aecns_af_callback))
{
AMS_ERROR("msg attach handler for AF_STATUS_EVENT failed");
goto error_exit;
}
***/
gp_aecns_context->init_state = 1;
AMS_LOG(LOG_WARNING, "init ok!");
}
else
{
AMS_DEBUG("deinit");
#ifndef AEC_ARGS_MOTOR_TYPE_SUPPORT
interface_msg_detach_handler(PTZ_STATUS_EVENT, aecns_ptz_callback);
#endif
#ifdef BUILD_AMS_MINI
interface_msg_detach_handler(PTZ_STATUS_EVENT, ams_mini_aecns_ptz_callback);
#endif
if (NULL == gp_aecns_context)
{
/* 同时释放 VAD */
if (vad_inst) { WebRtcVad_Free(vad_inst); vad_inst = NULL; }
vad_has_voice = 0;
pthread_mutex_unlock(&g_aecns_mutex_lock);
return OK;
}
gp_aecns_context->init_state = 0;
/* 算法退出前停止定时 */
aecns_stop_timer(&(gp_aecns_context->motor_delay_timer));
if (gp_aecns_context->p_near_common_context)
{
audio_common_deinit(gp_aecns_context->p_near_common_context);
free(gp_aecns_context->p_near_common_context);
gp_aecns_context->p_near_common_context = NULL;
}
if (gp_aecns_context->p_aec_context)
{
aec_module_deinit(gp_aecns_context->p_aec_context);
free(gp_aecns_context->p_aec_context);
gp_aecns_context->p_aec_context = NULL;
}
#ifdef TP_AGC_SUPPORT
if (gp_aecns_context->p_agc_context)
{
agc_deinit(gp_aecns_context->p_agc_context);
free(gp_aecns_context->p_agc_context);
gp_aecns_context->p_agc_context = NULL;
}
#endif
if (gp_aecns_context->data_tmp)
{
free(gp_aecns_context->data_tmp);
gp_aecns_context->data_tmp = NULL;
}
/* 释放 VAD */
if (vad_inst)
{
WebRtcVad_Free(vad_inst);
vad_inst = NULL;
}
vad_has_voice = 0;
free(gp_aecns_context);
gp_aecns_context = NULL;
}
pthread_mutex_unlock(&g_aecns_mutex_lock);
return OK;
error_exit:
#ifndef AEC_ARGS_MOTOR_TYPE_SUPPORT
interface_msg_detach_handler(PTZ_STATUS_EVENT, aecns_ptz_callback);
#endif
#ifdef BUILD_AMS_MINI
interface_msg_detach_handler(PTZ_STATUS_EVENT, ams_mini_aecns_ptz_callback);
#endif
if (NULL != gp_aecns_context)
{
gp_aecns_context->init_state = 0;
/* 算法退出前停止定时 */
aecns_stop_timer(&(gp_aecns_context->motor_delay_timer));
if (gp_aecns_context->p_near_common_context)
{
audio_common_deinit(gp_aecns_context->p_near_common_context);
free(gp_aecns_context->p_near_common_context);
gp_aecns_context->p_near_common_context = NULL;
}
if (gp_aecns_context->p_aec_context)
{
aec_module_deinit(gp_aecns_context->p_aec_context);
free(gp_aecns_context->p_aec_context);
gp_aecns_context->p_aec_context = NULL;
}
#ifdef TP_AGC_SUPPORT
if (gp_aecns_context->p_agc_context)
{
agc_deinit(gp_aecns_context->p_agc_context);
free(gp_aecns_context->p_agc_context);
gp_aecns_context->p_agc_context = NULL;
}
#endif
if (gp_aecns_context->data_tmp)
{
free(gp_aecns_context->data_tmp);
gp_aecns_context->data_tmp = NULL;
}
free(gp_aecns_context);
gp_aecns_context = NULL;
}
/* 释放 VAD */
if (vad_inst)
{
WebRtcVad_Free(vad_inst);
vad_inst = NULL;
}
vad_has_voice = 0;
pthread_mutex_unlock(&g_aecns_mutex_lock);
AMS_LOG(LOG_ERROR, "TPAECNS init FAILED!");
return ERROR;
}
S32 aecns_process(AEC_PROCESS_ARGS *args)
{
pthread_mutex_lock(&g_aecns_mutex_lock);
if ((NULL == args) || (NULL == args->out) || (args->data_len <= 0) || (NULL == args->mic_buf))
{
AMS_ERROR("aecns_process args is NULL");
goto error_exit;
}
if ((NULL == gp_aecns_context) || (0 == gp_aecns_context->init_state))
{
AMS_ERROR("process failed! aecns should be INIT FIRST");
/* 防止出现因为没有init,没有进行aecns_process造成的输出语音中断 */
memcpy(args->out, args->mic_buf, args->data_len * sizeof(U8));
goto error_exit;
}
/* ---- 30帧统计VAD----
1) 先对当前这一帧做 10ms 分块 webrtc VAD,累加本帧命中块数为 vad_voice_count
2) 若 vad_voice_count > 6,则本帧计为“命中帧”
3) 在 30 帧窗口内累计命中帧数,达到 3 帧则置 vad_has_voice = 1
4) 窗口结束(到第 30 帧)时:
- 命中帧数 < 3 -> 清 vad_has_voice = 0
- 重置窗口计数器
*/
if (vad_inst != NULL)
{
const int16_t *pcm = (const int16_t *)(args->mic_buf);
const U32 samps = (U32)(args->data_len >> 1);
int sr = 16000; /* fallback */
if (gp_aecns_context->p_near_common_context)
sr = gp_aecns_context->p_near_common_context->sample_rate;
/* 10ms 窗长 */
int vad_win = (sr == 8000) ? 80 :
(sr == 16000) ? 160 :
(sr == 32000) ? 320 :
(sr == 48000) ? 480 : 160;
/* 计算“本帧”命中块数 */
int vad_voice_count = 0;
for (U32 off = 0; off + (U32)vad_win <= samps; off += vad_win)
{
int vad_ret = WebRtcVad_Process(vad_inst, sr, pcm + off, (size_t)vad_win);
if (vad_ret == 1) {
vad_voice_count++;
}
}
/* 进入 30 帧统计窗口逻辑 */
if (vad_win_cnt == 0) {
/* 新窗口起始,可按需在这里“软衰减”上一窗口结论;当前我们保持 vad_has_voice 不变 */
}
/* 若本帧命中块数满足阈值,则窗口命中帧 +1;命中帧累计到 3 则立即认为“有人声” */
if (vad_voice_count > VAD_PERFRAME_HIT_THRESHOLD)
{
if (vad_hit_cnt < VAD_REQUIRED_HITS)
{
vad_hit_cnt++;
if (vad_hit_cnt >= VAD_REQUIRED_HITS)
{
vad_has_voice = 1;
}
}
}
/* 窗口推进 */
vad_win_cnt++;
/* 窗口结束:检查是否需要清 vad_has_voice,并清空计数器 */
if (vad_win_cnt >= VAD_STAT_WINDOW_FRAMES)
{
if (vad_hit_cnt < VAD_REQUIRED_HITS)
{
/* 30 帧内命中不足 3 帧 -> 认为“无人声” */
vad_has_voice = 0;
}
/* 重置窗口 */
vad_win_cnt = 0;
vad_hit_cnt = 0;
}
printf("[VAD] sr=%d this_frame_hits=%d | win_cnt=%d hit_cnt=%d -> has_voice=%d",
sr, vad_voice_count, vad_win_cnt, vad_hit_cnt, vad_has_voice);
}
/* 开启aec时spk不能为null */
if ((args->aec) && (args->spk_buf == NULL))
{
AMS_ERROR("process failed! aec param error");
memcpy(args->out, args->mic_buf, args->data_len * sizeof(U8));
goto error_exit;
}
#ifdef AEC_ARGS_MOTOR_TYPE_SUPPORT
/* 电机状态根据软件传入修改 */
U8 motor_type = gp_aecns_context->p_aec_context->den->motor_state;
if ((1 == args->motor_type) || ((args->motor_type) && (0 == motor_type)))
{
motor_type = 1;
}
if ((0 == args->motor_type) && (motor_type))
{
motor_type = 0;
}
gp_aecns_context->p_aec_context->den->motor_state = motor_type;
#endif
U8 aec_enable = args->aec; /* 添加回声消除开关flag */
#ifdef PCDBUG_AEC_STATE_FILE_ENABLE
aec_enable = args->aec_state_buf[0];
#endif
#ifdef PCDBUG_MOTOR_STATE_FILE_ENABLE
gp_aecns_context->p_aec_context->den->motor_state = args->motor_state_buf[0];
#endif
gp_aecns_context->p_aec_context->st->aec_enable = aec_enable;
unsigned int sdata_len = args->data_len >> 1;
gp_aecns_context->p_aec_context->sdata_len = sdata_len;
gp_aecns_context->p_aec_context->mic_buf_ptr = (S16 *)(args->mic_buf);
if (aec_enable) /* aec为0时对于spk_buf_ptr赋值spk_buf_zero */
{
gp_aecns_context->p_aec_context->spk_buf_ptr = (S16 *)(args->spk_buf);
} else {
memset(gp_aecns_context->p_aec_context->spk_buf_zero, 0, 2 * sizeof(short) * D_AEC_FRAME_LEN_WB);
gp_aecns_context->p_aec_context->spk_buf_ptr = gp_aecns_context->p_aec_context->spk_buf_zero;
}
gp_aecns_context->p_aec_context->out_buf_ptr = gp_aecns_context->aec_out_buf;
gp_aecns_context->p_aec_context->mic_timestamp = args->mic_timestamp;
gp_aecns_context->p_aec_context->spk_timestamp = args->spk_timestamp;
gp_aecns_context->frame_count += 1;
if (sdata_len > D_INPUT_FRAME_MAX_LEN)
{
AMS_ERROR("input frame len should smaller than %d", D_INPUT_FRAME_MAX_LEN);
memcpy(args->out, args->mic_buf, args->data_len * sizeof(U8));
goto error_exit;
}
/* (!支持直播 && !aec_enable) || (!支持双讲 && aec_enable)的时候不做处理,下面这个是反条件,进入处理的条件 */
if ((gp_aecns_context->p_aec_context->aec3_live_support_switch || aec_enable) && (gp_aecns_context->p_aec_context->aec3_dt_support_switch || !(aec_enable)))
{
/* 每隔一段时间获取当前speaker增益,根据增益来初判speaker数据是否失真, 从而采取不同的回声消除策略 */
if (gp_aecns_context->frame_count % 20 == 0)
{
AUDIO_CONFIG_SPEAKER speaker_config;
memset(&speaker_config, 0, sizeof(AUDIO_CONFIG_SPEAKER));
if (0 == AMS_READ(AUDIO_CONFIG_SPEAKER_PATH, &speaker_config, sizeof(AUDIO_CONFIG_SPEAKER)))
{
AMS_WARN("read AUDIO_CONFIG_SPEAKER_PATH fail");
speaker_config.volume = 50;
}
/* TODO: 不同机型的增益阈值通过配置文件来设置 */
if (speaker_config.volume > 70)
{
gp_aecns_context->p_aec_context->st->speaker_distort = 1;
}
else
{
gp_aecns_context->p_aec_context->st->speaker_distort = 0;
}
}
aec_module_process(gp_aecns_context->p_aec_context);
#ifdef TP_AGC_SUPPORT
if (gp_aecns_context->p_agc_context->agc_core->agc_enable && ((aec_enable && (DOUBLE_TALK_AGC_MODE_ON & gp_aecns_context->p_agc_context->agc_core->agc_enable)) || ((!aec_enable) && (LIVE_AGC_MODE_ON & gp_aecns_context->p_agc_context->agc_core->agc_enable))))
{
agc_process(gp_aecns_context->p_agc_context, 0, gp_aecns_context->aec_out_buf, gp_aecns_context->agc_out_buf, gp_aecns_context->p_aec_context->vad_for_agc_enable, gp_aecns_context->p_aec_context->vad_list, sdata_len);
memcpy(args->out, gp_aecns_context->agc_out_buf, args->data_len * sizeof(U8));
#ifdef DUMP_RAWDATA_SUPPORT
FILE *agc_file = NULL;
if (access("/tmp/mnt/harddisk_1/audio_stream", 0) == 0)
{
agc_file = fopen("/tmp/mnt/harddisk_1/audio_stream/agc.pcm", "a+");
fwrite(gp_aecns_context->agc_out_buf, sizeof(short), sdata_len, agc_file);
fclose(agc_file);
}
#endif
} else {
memcpy(args->out, gp_aecns_context->aec_out_buf, args->data_len * sizeof(U8));
}
#else
memcpy(args->out, gp_aecns_context->aec_out_buf, args->data_len * sizeof(U8));
#endif
gp_aecns_context->p_aec_context->st->old_aec_enable = aec_enable; /* 记录上一帧aec开关状态 */
pthread_mutex_unlock(&g_aecns_mutex_lock);
return 0;
}
else
{
memcpy(gp_aecns_context->aec_out_buf, gp_aecns_context->p_aec_context->mic_buf_ptr, sizeof(short) * gp_aecns_context->p_aec_context->sdata_len);
memcpy(args->out, gp_aecns_context->aec_out_buf, args->data_len * sizeof(U8));
gp_aecns_context->p_aec_context->st->old_aec_enable = aec_enable;
pthread_mutex_unlock(&g_aecns_mutex_lock);
return 0;
}
if (ERROR == audio_common_buffer_data_in(gp_aecns_context->p_near_common_context->analysis_buffer, gp_aecns_context->aec_out_buf, sdata_len))
{
AMS_ERROR("p_near_common_context->analysis_buffer: audio_common_buffer_data_in failed");
goto error_exit;
}
while (1)
{
/* 近端buffer更新 */
if (ERROR == audio_common_analysis_buffer_update(gp_aecns_context->p_near_common_context))
{
break;
}
/* 近端common process:不管aec/ns开不开都要处理,防止电话模式切换前后*common_context->synthesis_buffer数据有误 */
{
audio_common_process(gp_aecns_context->p_near_common_context);
AUDIO_COMPLEX_DATA *complex_data = (AUDIO_COMPLEX_DATA *)(gp_aecns_context->p_near_common_context->frequency_data);
complex_data[0].real = 0;
complex_data[0].imag = 0;
complex_data[1].real >>= 3;
complex_data[1].imag >>= 3;
complex_data[2].real >>= 2;
complex_data[2].imag >>= 2;
calc_magnitude_frequency_bins(gp_aecns_context->p_near_common_context->frequency_data, gp_aecns_context->p_near_common_context->magnitude, &(gp_aecns_context->p_near_common_context->magnitude_sum), &(gp_aecns_context->p_near_common_context->magnitude_energy), gp_aecns_context->p_near_common_context->frame_length >> 1);
}
if (ERROR == audio_common_synthesis_buffer_update(gp_aecns_context->p_near_common_context))
{
AMS_ERROR("audio_common_synthesis_buffer_update failed");
goto error_exit;
}
} // while(1) over
/* synthesis_buffer -> out */
if (ERROR == audio_common_buffer_data_out(gp_aecns_context->p_near_common_context->synthesis_buffer, (S16 *)(args->out), sdata_len))
{
AMS_WARN("process not enough out data");
}
pthread_mutex_unlock(&g_aecns_mutex_lock);
return OK;
error_exit:
pthread_mutex_unlock(&g_aecns_mutex_lock);
return ERROR;
}
/* 这里可以直接把各个函数注册到audio_attr里面,而不需要在node里面再注册,当然也可以由别的实现 */
LOCAL void aecns_main()
{
AQI *aqi = NULL;
AQI *aec = NULL;
aqi = ams_aqi_find("tpaec3ns3");
aec = ams_aqi_find("aec3");
if ((NULL != aqi) && (NULL != aec))
{
g_audio_attr.tp_aec_init_callback = aecns_init;
g_audio_attr.tp_aec_process_callback = aecns_process;
}
// alg_trigger_add(awf_trigger, ADR_DATA_TYPE_FRAME_16K);
}
AMS_INIT(aecns_main);
这个降噪程序里面,webrtc vad是怎么结合进去的,有什么效果