基于ESP32S3的智能对话“机器人”

项目背景：

        随着近年来的大模型的热度愈来愈热，基于大模型的智能设备也在蓬勃发展，利用手边的Seeed Studio XIAO ESP32S3 Sense 开发板接入阿里云百炼大模型的平台，做了个简单的聊天机器人。

硬件方案：

        1. Seeed Studio XIAO ESP32S3 Sense开发版，可以换成其它的乐鑫的ESP32开发板，需要携带mic用于声音的采集

        2. 扬声器用用的是MAX98357A+4Ω 3W的扬声器

        3. 显示采用的是GC9A01+240*240 TFT 圆形显示屏

项目的流程图：

大模型接入的流程：

        本项目用的是阿里云百炼大模型平台的语音转文字大模型，完成语音的识别，然后调用 通义千问，完成文本的理解，然后再调用文本转语音的大模型完成整体的交互。阿里云百炼的大模型平台大模型的调用相对比较简单，完成注册之后，所有的大模型的调用只需要基于api-key即可完成调用，其它的平台 例如豆包大模型等平台的调用过程基本都是相似的，主要用的就是websocket及https完成与大模型的交互。

基于ESP-IDF框架 websocket的创建过程大体如下：

static void wss_event_handler(void *handler_args, esp_event_base_t base, int32_t event_id, void *event_data)
{
    esp_websocket_event_data_t *data = (esp_websocket_event_data_t *)event_data;
    char *rcv_buf = NULL;
    switch (event_id) {
        case WEBSOCKET_EVENT_BEGIN:
            ESP_LOGI(TAG, "WEBSOCKET_EVENT_BEGIN");
        break;
        case WEBSOCKET_EVENT_CONNECTED:
            ESP_LOGI(TAG, "WEBSOCKET_EVENT_CONNECTED");
            send_start_platform_trans_msg();
        break;
        case WEBSOCKET_EVENT_DISCONNECTED:
            ESP_LOGI(TAG, "WEBSOCKET_EVENT_DISCONNECTED");
        break;
        case WEBSOCKET_EVENT_DATA:
            ESP_LOGI(TAG, "WEBSOCKET_EVENT_DATA");
            ESP_LOGI(TAG, "Received=%.*s", data->data_len, (char *)data->data_ptr);
            if (data->op_code == 0x08 && data->data_len == 2) {
                ESP_LOGW(TAG, "Received closed message with code=%d", 256 * data->data_ptr[0] + data->data_ptr[1]);
            }else {
                rcv_buf = (char *)malloc(data->data_len+1);
                if (NULL != rcv_buf) {
                    memset(rcv_buf,0,data->data_len+1);
                    memcpy(rcv_buf, data->data_ptr, data->data_len);
                    phrase_wss_event_msg(rcv_buf);
                    free(rcv_buf);
                }
            }
        break;
        case WEBSOCKET_EVENT_ERROR:
            ESP_LOGI(TAG, "WEBSOCKET_EVENT_ERROR");
        break;
        case WEBSOCKET_EVENT_FINISH:
            ESP_LOGI(TAG, "WEBSOCKET_EVENT_FINISH");
        break;
        case  WEBSOCKET_EVENT_CLOSED:
            ESP_LOGI(TAG, "WEBSOCKET_EVENT_CLOSED");
        break;
    }
}

void start_aliyun_ws_client()
{
    char api_key[64]={0};
    esp_websocket_client_config_t websocket_cfg = {};
    websocket_cfg.uri = ALIYUN_WSS_SERVER_ADDR;
    websocket_cfg.crt_bundle_attach = esp_crt_bundle_attach;
    websocket_cfg.use_global_ca_store = true;
    s_wss_client = esp_websocket_client_init(&websocket_cfg);
    sniprintf(api_key,sizeof(api_key),"bearer %s", ALIYUN_API_KEY);
    esp_websocket_client_append_header(s_wss_client, "Authorization", api_key);
    esp_websocket_client_append_header(s_wss_client, "X-DashScope-DataInspection", "enable");
    esp_websocket_register_events(s_wss_client, WEBSOCKET_EVENT_ANY, wss_event_handler, (void *)s_wss_client);
    esp_websocket_client_start(s_wss_client);
}

基于ESP-IDF框架 HTTPS的创建过程如下：

esp_err_t http_event_handler(esp_http_client_event_t *evt)
{
    switch (evt->event_id) {
        case HTTP_EVENT_ERROR:
            ESP_LOGI(TAG, "HTTP_EVENT_ERROR");
        break;
        case HTTP_EVENT_ON_CONNECTED:
            ESP_LOGI(TAG, "HTTP_EVENT_ON_CONNECTED");
        break;
        case HTTP_EVENT_HEADER_SENT:
            ESP_LOGI(TAG, "HTTP_EVENT_HEADER_SENT, header sent");
        break;
        case HTTP_EVENT_ON_HEADER:
            ESP_LOGI(TAG, "HTTP_EVENT_HEADER, key=%s, value=%s", evt->header_key, evt->header_value);
        break;
        case HTTP_EVENT_ON_DATA:
            if (evt->data) {
                //ESP_LOGI(TAG, "HTTP_EVENT_ON_DATA, len=%d", evt->data_len);
                //ESP_LOGI(TAG, "Response: %.*s", evt->data_len, (char *)evt->data);
                memset(evt->user_data,0,HTTP_RSP_BUF_LEN);
                memcpy(evt->user_data, evt->data+strlen("data: "), evt->data_len-strlen("data: "));
                phrase_filed_chat_rsp(evt->user_data);
            }
        break;
        case HTTP_EVENT_ON_FINISH:
            ESP_LOGI(TAG, "HTTP_EVENT_FINISHED");
        break;
        case HTTP_EVENT_DISCONNECTED:
            ESP_LOGI(TAG, "HTTP_EVENT_DISCONNECTED");
        break;
        case HTTP_EVENT_REDIRECT:
            ESP_LOGI(TAG, "HTTP_EVENT_REDIRECT");
        break;
    }
    return ESP_OK;
}


void init_file_chat_start(void)
{
    char *send_str = NULL;
    esp_http_client_config_t config ={
        .url = ALIYUN_CHAT_SERVER_ADDR,
        .method = HTTP_METHOD_POST,
        .event_handler = http_event_handler,
        .user_data = s_http_rsp_buf,
        .timeout_ms = 10000,
        .crt_bundle_attach = esp_crt_bundle_attach,
        .use_global_ca_store = true,
    };
    s_http_client = esp_http_client_init(&config);
    esp_http_client_set_header(s_http_client, "Authorization", "Bearer "ALIYUN_API_KEY);
    esp_http_client_set_header(s_http_client, "Content-Type", "application/json");
    cJSON *root = cJSON_CreateObject();
    if (NULL != root) {
        build_http_filed_chat(root,s_voice_to_filed_str);
        send_str = cJSON_PrintUnformatted(root);
        ESP_LOGI(TAG,"http send content: %s",send_str);
        esp_http_client_set_post_field(s_http_client, send_str, (int)strlen(send_str));
    }
    esp_err_t err = esp_http_client_perform(s_http_client);
    if (err == ESP_OK) {
        ESP_LOGI(TAG, "HTTPS Status Code: %d", esp_http_client_get_status_code(s_http_client));
    } else {
        ESP_LOGE(TAG, "HTTP GET request failed: %s", esp_err_to_name(err));
    }
    if (NULL != send_str) {
        free(send_str);
        send_str = NULL;
    }
}

运行效果及接线图：

项目代码仓库：

STM32Project/Smart-Robot

https://github.com/yudeguo/Smart-Robot