php函数(error_log())2012.11.28日学

本文详细介绍了PHP中的error_log()函数,包括如何使用该函数向服务器错误记录、文件或远程目标发送错误信息。文章还解释了函数的参数,如error、type、destination和headers,并给出了不同类型的使用场景。

error_log() 函数

向服务器错误记录、文件或远程目标发送一个错误。

 

error_log(error,type,destination,headers)

error必需。要记录的错误消息。
type

可选。规定错误记录的类型。

可能的记录类型:

  • 0 - 默认。根据在 php.ini 文件中的 error_log 配置,错误被发送到服务器日志系统或文件。
  • 1 - 错误被发送到 destination 参数中的地址。只有该类型使用 headers 参数。
  • 2 - 通过 PHP debugging 连接来发送错误。该选项只在 PHP 3 中可用。
  • 3 - 错误发送到文件目标字符串。
destination可选。规定向何处发送错误消息。该参数的值依赖于 "type" 参数的值。
headers

可选。只在 "type" 为 1 时使用。

规定附加的头部,比如 From, Cc 以及 Bcc。由 CRLF (\r\n) 分隔。

注释:在发送电子邮件时,必须包含 From 头部。可以在 php.ini 文件中或者通过此参数设置。

/*!Copyright(c) 2012-2019 Shenzhen TP-LINK Technologies Co.Ltd. * *\file url_class.c *\brief process for URL query * *\author Wanghao *\version 1.0.0 *\date 27Aug19 * *\history * */ #include <sys/stat.h> #include <unistd.h> #include <stdio.h> #include <stdlib.h> #include <sys/socket.h> #include <sys/types.h> #include <string.h> #include <asm/types.h> #include <linux/netlink.h> #include <linux/socket.h> #include <getopt.h> #include <errno.h> #include <avira/auc/aucApi.h> #include <pthread.h> #include <signal.h> #include <sys/time.h> #include <curl/curl.h> #include "cJSON.h" #include <uci.h> #include <mali_url_check.h> #include <fcntl.h> #include <unistd.h> #define printErr(fmt, args...) printf("\033[1m[ %s ] %03d: "fmt"\033[0m", __FUNCTION__, __LINE__, ##args) #define printWar(fmt, args...) printf("\033[1m[ %s ] %03d: "fmt"\033[0m", __FUNCTION__, __LINE__, ##args) #define CURL_DEBUG 0 #define URL_CLASS_LOG_LEVEL LOG_LEVEL_MAX #define URL_CLASS_LOG(level, fmt, args...) \ do \ { \ if (level >= URL_CLASS_LOG_LEVEL) \ { \ static char cmd[1024] = {0}, final[1024+128] = {0}; \ struct timeval tv; \ struct tm* local; \ gettimeofday(&tv, NULL); \ local = localtime(&tv.tv_sec); \ memset(cmd,0,sizeof(cmd));memset(final,0,sizeof(final));\ snprintf(cmd, sizeof(cmd), fmt, ##args); \ snprintf(final, sizeof(final), "echo \"%04d-%02d-%02d %02d:%02d:%02d.%03d|level:%d|%s:%d|%s| - ",\ local->tm_year + 1900, local->tm_mon + 1, local->tm_mday, \ local->tm_hour, local->tm_min, local->tm_sec, (int)(tv.tv_usec / 1000), \ level, __FILE__, __LINE__, __func__); \ int _i = 0, _j = strlen(final); \ for(_i = 0; _i < 1024 && _j < 1024+128-15; _i++,_j++) \ { \ if(cmd[_i] == '\"' || cmd[_i] == '\\' || cmd[_i] == '`' || cmd[_i] == '$') \ { \ final[_j++] = '\\'; \ } \ final[_j] = cmd[_i]; \ } \ strncpy(final + (final[strlen(final) - 1] == '\n'? strlen(final)-1: strlen(final)), "\"> /dev/console", 16); \ system(final); \ } \ } while (0) #define MIN(a,b) ((a)<(b) ? (a):(b)) #define MALI_MAX_WAIT_TIME 1024 typedef enum { LOG_LEVEL_DEBUG = 0, // log level debug LOG_LEVEL_INFO, // log level information LOG_LEVEL_WARN, // log level warning LOG_LEVEL_ERROR, // log level error LOG_LEVEL_EMRG, // log level emergency LOG_LEVEL_MAX }E_LOG_LEVE; struct Options { bool use_tp_service; }; struct Options opts; #define MAX_PAYLOAD 1024 // maximum payload size #ifndef NETLINK_URL_CLASS #define NETLINK_URL_CLASS 27 #endif //#define MAX_URL_ENTRY_LEN 512//256 #define MAX_URL_LEN 256 #define MAX_URL_CAT_DEFAULT 9999 #define PCTL_WEB_URL_ID_ALL 0xfffe #if SUPPORT_CLOUD_UPDATE_AUC_INFO #define POST_TRANSFER_TIMEOUT 20 #define POST_CONNECT_TIMEOUT 20 #define AUC_UPDATE_TIME_INTERVAL_DEFAULT 3600 //1h #define AUC_UPDATE_TIME_INTERVAL_FAST 10 //10s #define REQUEST_TIMEOUT_MS 8000 #define MAX_TOKEN_LEN 128 #define FILE_CLOUD_TOKEN_HOMECARE "/tmp/cloud/cloud_token_homecare" #define GET_AUC_URL_POSTFIX "/v1/configuration/auc-service-info" #define CA_CRT_PATH "/etc/certificate/2048_newroot.cer" struct auc_update_status_info { int update_time_interval; time_t updata_time_stamp_last; }; static struct auc_update_status_info auc_update_status = { .update_time_interval = AUC_UPDATE_TIME_INTERVAL_DEFAULT, .updata_time_stamp_last = 0, }; typedef struct _st_http_resinfo { long status_code; }st_http_resinfo; #endif enum url_req { URL_REQ_VOID = 0, URL_REQ_HELLO, URL_REQ_CAT, URL_REQ_LOG, URL_REQ_LOG_HISTORY, URL_REQ_BYE, URL_REQ_END }; struct url_carrier { int id; int info_id; unsigned char url_len; unsigned int cat_id; unsigned int cat_map; char url[MAX_URL_LEN]; }; enum url_process_status { URL_PROCESS_READY = 0, //waiting for auc process URL_PROCESSING //auc is processing, changing url is useless }; struct url_entry { int id; int info_id; unsigned int cat_map; unsigned char query; unsigned char url_len; enum url_process_status process_flag; struct url_entry *prev; struct url_entry *next; char url[MAX_URL_LEN]; }; pthread_mutex_t url_lock = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_t mali_url_check_lock = PTHREAD_MUTEX_INITIALIZER; pthread_cond_t cond = PTHREAD_COND_INITIALIZER; //struct url_entry url_array[MAX_URL_ENTRY_LEN]; struct url_entry *url_array; static struct url_entry url_list = { .id = 0, .info_id = 0, .cat_map = 0, .query = 0, .url_len = 0, .prev = NULL, .next = NULL, }; int sock_fd = -1; //cat mapping table int mapping_table[][32] = { {1, 274, 276, 374, 132, -1}, //1100, 1-Games {2, 193, 447, 449, -1}, //1000, 2-Download {3, 222, 223, 224, 357, -1}, //900, 3-Media {4, 107, 388, -1}, //800, 4-Pay-to-surf & Shopping {5, 414, 445, -1}, //700, 5-Social Networking {6, 359, 360, 355, 293, 397, 450, 451, -1}, //600, 6-Oline Communications and Search {7, 443, -1}, //500, 7-Gambling {10, 241, 179, 76, 170, 167, -1}, //200, 10-Sex Education {11, 234, 400, 269, 444, 446, 395, 429, 391, 390, 103, 64, 65, 66, 149, 452, 453, 442, -1}, //100, 11-Adult Content {13, 2, -1}, //000, unsafe, 13-Malware {14, 3, -1}, //000, unsafe, 14-Phishing {15, 197, -1}, // 15-web_search {12, MAX_URL_CAT_DEFAULT, -1}, //2000, 12-Other(Default) {-1} //END }; static inline void strncpy_safe(char *__dest, const char *__src, size_t __n) { strncpy(__dest, __src, __n); __dest[__n] = '\0'; } static void on_exit_handler() { struct sockaddr_nl dest_addr; struct nlmsghdr *nlh = NULL; struct iovec iov; struct msghdr msg; int state_smg = 0; nlh = (struct nlmsghdr *)malloc(NLMSG_SPACE(MAX_PAYLOAD)); if (!nlh) { printErr("malloc nlmsghdr error!\n"); close(sock_fd); return; } //say bye to kernel memset(nlh,0,NLMSG_SPACE(MAX_PAYLOAD)); memset(&dest_addr,0,sizeof(dest_addr)); dest_addr.nl_family = AF_NETLINK; dest_addr.nl_pid = 0; dest_addr.nl_groups = 0; nlh->nlmsg_len = NLMSG_SPACE(MAX_PAYLOAD); nlh->nlmsg_pid = 0; nlh->nlmsg_flags = 0; nlh->nlmsg_type = URL_REQ_BYE; iov.iov_base = (void *)nlh; iov.iov_len = NLMSG_SPACE(MAX_PAYLOAD); memset(&msg, 0, sizeof(msg)); msg.msg_name = (void *)&dest_addr; msg.msg_namelen = sizeof(dest_addr); msg.msg_iov = &iov; msg.msg_iovlen = 1; state_smg = sendmsg(sock_fd, &msg, 0); if (state_smg < 0) { printErr("get error sendmsg = %s\n",strerror(errno)); } if (nlh) { free(nlh); } close(sock_fd); return; } static void signal_exit_handler(int sig) { exit(0); } static int url_send(int id, unsigned int cat_id, char *url, unsigned char url_len, unsigned short type) { struct nlmsghdr *nlh = NULL; struct iovec iov; struct msghdr msg; struct sockaddr_nl dest_addr; struct url_carrier data; int state_smg = 0; int ret = 0; nlh = (struct nlmsghdr *)malloc(NLMSG_SPACE(MAX_PAYLOAD)); if (!nlh) { printErr("malloc nlmsghdr error!\n"); return -1; } memset(&dest_addr,0,sizeof(dest_addr)); dest_addr.nl_family = AF_NETLINK; dest_addr.nl_pid = 0; dest_addr.nl_groups = 0; nlh->nlmsg_len = NLMSG_SPACE(MAX_PAYLOAD); nlh->nlmsg_pid = getpid(); nlh->nlmsg_flags = 0; nlh->nlmsg_type = type; memset(&data, 0, sizeof(struct url_carrier)); data.id = id; data.cat_id = cat_id; data.url_len = url_len; strncpy_safe(data.url, url, url_len); printWar("url_send to kernel, id=%d, cat_id=%x, url=%s, url_len=%d \n", data.id, data.cat_id, data.url, data.url_len); memcpy(NLMSG_DATA(nlh), &data, sizeof(struct url_carrier)); iov.iov_base = (void *)nlh; iov.iov_len = NLMSG_SPACE(MAX_PAYLOAD); memset(&msg, 0, sizeof(msg)); msg.msg_name = (void *)&dest_addr; msg.msg_namelen = sizeof(dest_addr); msg.msg_iov = &iov; msg.msg_iovlen = 1; state_smg = sendmsg(sock_fd, &msg, 0); if (state_smg < 0) { printErr("get error sendmsg = %s\n",strerror(errno)); ret = -1; goto out; } out: if (nlh) { free(nlh); } return ret; } static void __attribute__((unused)) dump_list(struct url_entry *list) { printErr("dump:\n"); while (list) { printWar("id=%d url=%s\n", list->id, list->url); list = list->next; } } static void cond_timedwait(long timeout) { struct timespec abstime; struct timeval now; long nsec = 0; pthread_mutex_lock(&url_lock); gettimeofday(&now, NULL); nsec = now.tv_usec * 1000 + (timeout % 1000) * 1000000; abstime.tv_sec=now.tv_sec + nsec / 1000000000 + timeout / 1000; abstime.tv_nsec=nsec % 1000000000; pthread_cond_timedwait(&cond, &url_lock, &abstime); pthread_mutex_unlock(&url_lock); } #if SUPPORT_CLOUD_UPDATE_AUC_INFO static int OnDebug(CURL * curl, curl_infotype itype, char * pData, size_t size, void *lpVoid) { if (itype == CURLINFO_TEXT) { //URL_CLASS_LOG(LOG_LEVEL_DEBUG,"[TEXT]%s", pData); } else if (itype == CURLINFO_HEADER_IN) { URL_CLASS_LOG(LOG_LEVEL_DEBUG, "[HEADER_IN]%s", pData); } else if (itype == CURLINFO_HEADER_OUT) { URL_CLASS_LOG(LOG_LEVEL_DEBUG, "[HEADER_OUT]%s", pData); } else if (itype == CURLINFO_DATA_IN) { URL_CLASS_LOG(LOG_LEVEL_DEBUG, "[DATA_IN]%s", pData); } else if (itype == CURLINFO_DATA_OUT) { URL_CLASS_LOG(LOG_LEVEL_DEBUG, "[DATA_OUT]%s", pData); } return 0; } static size_t OnWriteData_Post(void* buffer, size_t size, size_t nmemb, void* lpVoid) { if (NULL == buffer) { return -1; } unsigned int len = (unsigned int)size * (unsigned int)nmemb; char *str = (char*)malloc(len + 1); if (NULL == str) { return -1; } char* pData = (char*)buffer; memset(str, 0, len + 1); memcpy(str, pData, len); URL_CLASS_LOG(LOG_LEVEL_DEBUG, "response: %s\n", str); char **response = (char**)lpVoid; *response = str; return len; } static size_t cloud_https_post(const char *pUrl, const char *requestHeader, const char *request, char **response, st_http_resinfo *pHttpResInfo) { CURLcode res; CURL* curl = NULL; struct curl_slist *headers = NULL; #ifdef CURL_DEBUG char errbuf[CURL_ERROR_SIZE]; memset(errbuf, '\0', CURL_ERROR_SIZE); #endif if (NULL == pUrl || NULL == request || NULL == response || NULL == pHttpResInfo) { return CURLE_FAILED_INIT; } URL_CLASS_LOG(LOG_LEVEL_DEBUG, "pUrl:%s\n requestHeader:%s\n request:%s\n", pUrl, requestHeader, request); res = curl_global_init(CURL_GLOBAL_ALL); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl global init fail and ret %d", res); return res; } curl = curl_easy_init(); if (NULL == curl) { res = CURLE_FAILED_INIT; URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl init fail"); goto exit; } //headers = curl_slist_append(headers, "Content-Type: application/json;charset=UTF-8"); headers = curl_slist_append(headers, "Content-Type: application/json"); if (NULL == headers) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl get header list fail"); goto exit; } if ((NULL != requestHeader) && (0 < strlen(requestHeader))) { URL_CLASS_LOG(LOG_LEVEL_DEBUG, "requestHeader is %s", requestHeader); headers = curl_slist_append(headers, requestHeader); if (NULL == headers) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl append requestHeader fail"); goto exit; } } #ifdef CURL_DEBUG //provide a buffer to store errors in res = curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_ERRORBUFFER ret %d", res); goto exit; } #endif if (CURL_DEBUG) { URL_CLASS_LOG(LOG_LEVEL_DEBUG, "post sesstion debug on"); res = curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_VERBOSE ret %d", res); goto exit; } res = curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, OnDebug); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_DEBUGFUNCTION ret %d", res); goto exit; } } //set url res = curl_easy_setopt(curl, CURLOPT_URL, pUrl); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_URL ret %d", res); goto exit; } //set header res = curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_HTTPHEADER ret %d", res); goto exit; } //set post method res = curl_easy_setopt(curl, CURLOPT_POST, 1); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_POST ret %d", res); goto exit; } res = curl_easy_setopt(curl, CURLOPT_POSTFIELDS, request); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_POSTFIELDS ret %d", res); goto exit; } //set read/write params res = curl_easy_setopt(curl, CURLOPT_READFUNCTION, NULL); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_READFUNCTION ret %d", res); goto exit; } res = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, OnWriteData_Post); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_WRITEFUNCTION ret %d", res); goto exit; } res = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)response); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_WRITEDATA ret %d", res); goto exit; } res = curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_NOSIGNAL ret %d", res); goto exit; } //set certificate info res = curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_SSL_VERIFYPEER ret %d", res); goto exit; } res = curl_easy_setopt(curl, CURLOPT_CAINFO, CA_CRT_PATH); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_CAINFO ret %d", res); goto exit; } //set CURLOPT_CONNECTTIMEOUT res = curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, POST_TRANSFER_TIMEOUT); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_CONNECTTIMEOUT ret %d", res); goto exit; } //set CURLOPT_TIMEOUT res = curl_easy_setopt(curl, CURLOPT_TIMEOUT, POST_CONNECT_TIMEOUT); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_TIMEOUT ret %d", res); goto exit; } //CURLOPT needs to be set_ FOLLOWLOCATION is 1, otherwise, the data after redirecting will not be returned res = curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION,1); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_FOLLOWLOCATION ret %d", res); goto exit; } res = curl_easy_perform(curl); #ifdef CURL_DEBUG if(CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_DEBUG, "curl post return error: %s", errbuf); } #endif curl_easy_getinfo(curl, CURLINFO_HTTP_CODE, &(pHttpResInfo->status_code)); URL_CLASS_LOG(LOG_LEVEL_DEBUG, "cloud_https_post done. ret %d, http status code %d", res, pHttpResInfo->status_code); exit: if (headers) { curl_slist_free_all(headers); } if (curl) { curl_easy_cleanup(curl); } curl_global_cleanup(); return res; } static int get_token_url(char *token, char *url) { int ret = -1; int retry_count = 3; FILE *fp = NULL; while (retry_count > 0) { if(fp = fopen(FILE_CLOUD_TOKEN_HOMECARE, "r")) { fscanf(fp, "%s", token); fscanf(fp, "%s", url); fclose(fp); ret = 0; break; } else { system("cloud_getDevToken homecare"); } retry_count--; } if ((*(token + MAX_TOKEN_LEN - 1) != 0) || (*(url + MAX_URL_LEN - 1) != 0)) { URL_CLASS_LOG(LOG_LEVEL_WARN, "token or url pointer out of range"); ret = -1; } return ret; } static char* get_auc_info_from_cloud_request_data() { int ret = -1; cJSON* request_data_json = NULL; char* request_data = NULL; unsigned char country_code[3] = {0}; FILE *fp = NULL; if ((request_data_json = cJSON_CreateObject()) == NULL) { goto out; } if (fp = popen("getfirm COUNTRY | tr 'A-Z' 'a-z'", "r")) { if (fread(country_code, sizeof(char), 2, fp) > 0) { cJSON_AddStringToObject(request_data_json, (const char*)"region", (const char*)country_code); request_data = cJSON_PrintUnformatted(request_data_json); URL_CLASS_LOG(LOG_LEVEL_DEBUG, "request_data:%s", request_data); } pclose(fp); } cJSON_Delete(request_data_json); return request_data; out: return NULL; } static int get_auc_info_from_cloud(struct auc_info *auc_info, char *device_token, char *url_prefix) { int ret = -1; int ret_post = 0; char request_url[MAX_URL_LEN] = {0}; char request_header[MAX_TOKEN_LEN + 32] = {0}; char *request_data = NULL; char *response = NULL; st_http_resinfo httpResInfo; cJSON* response_json = NULL; cJSON* auc_server_json= NULL; cJSON* api_key_json = NULL; cJSON* user_guid_json = NULL; if (auc_info == NULL || device_token == NULL || url_prefix == NULL) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "params error\n"); return ret; } memset(&httpResInfo, 0, sizeof(st_http_resinfo)); snprintf(request_url, MAX_URL_LEN, "%s%s", url_prefix, GET_AUC_URL_POSTFIX); snprintf(request_header, MAX_TOKEN_LEN + 32, "Authorization: %s", device_token); request_data = get_auc_info_from_cloud_request_data(); //get post result ret_post = cloud_https_post(request_url, request_header, request_data, &response, &httpResInfo); URL_CLASS_LOG(LOG_LEVEL_DEBUG, "curl post return %d http retcode: %ld", ret_post, httpResInfo.status_code); if(request_data) { cJSON_free(request_data); } if (response && httpResInfo.status_code == 200) { response_json = cJSON_Parse(response); auc_server_json= cJSON_GetObjectItem(response_json, "aucServerUrl"); api_key_json = cJSON_GetObjectItem(response_json, "apiKey"); user_guid_json = cJSON_GetObjectItem(response_json, "userGuid"); if (auc_server_json->valuestring && api_key_json->valuestring && user_guid_json->valuestring) { snprintf(auc_info->auc_server, MAX_AUC_SERVER_LEN, "%s", auc_server_json->valuestring); snprintf(auc_info->api_key, MAX_API_KEY_LEN, "%s", api_key_json->valuestring); snprintf(auc_info->user_guid, MAX_USER_GUID, "%s", user_guid_json->valuestring); ret = 0; } } out: cJSON_Delete(response_json); return ret; } static void set_retry_time_interval(void) { if (auc_update_status.update_time_interval == AUC_UPDATE_TIME_INTERVAL_DEFAULT) { auc_update_status.update_time_interval = AUC_UPDATE_TIME_INTERVAL_FAST; } else if (auc_update_status.update_time_interval < AUC_UPDATE_TIME_INTERVAL_DEFAULT && (auc_update_status.update_time_interval * 2) < AUC_UPDATE_TIME_INTERVAL_DEFAULT) { auc_update_status.update_time_interval = auc_update_status.update_time_interval * 2; } else { auc_update_status.update_time_interval = AUC_UPDATE_TIME_INTERVAL_DEFAULT; } } static void update_libauc_url(struct auc_info *auc_info, bool use_tp_service) { char device_token[MAX_TOKEN_LEN] = {0}; char cloud_homecare_url[MAX_URL_LEN] = {0}; time_t time_stamp; memset(device_token, 0, MAX_TOKEN_LEN); memset(cloud_homecare_url, 0, MAX_URL_LEN); memset(auc_info->auc_server, 0, MAX_AUC_SERVER_LEN); memset(auc_info->api_key, 0, MAX_API_KEY_LEN); memset(auc_info->user_guid, 0, MAX_USER_GUID); //get time stamp time(&time_stamp); URL_CLASS_LOG(LOG_LEVEL_DEBUG, "start time_stamp:%d, updata_time_stamp_last:%d, update_time_interval:%d\n", time_stamp, auc_update_status.updata_time_stamp_last, auc_update_status.update_time_interval); if ((time_stamp - auc_update_status.updata_time_stamp_last) >= auc_update_status.update_time_interval) { if(use_tp_service) { if (0 == (get_token_url(device_token, cloud_homecare_url))) { memset(auc_info->tp_auc_server, 0, MAX_AUC_SERVER_LEN); snprintf(auc_info->tp_auc_server, MAX_AUC_SERVER_LEN, "%s", cloud_homecare_url); auc_update_status.updata_time_stamp_last = time_stamp; auc_update_status.update_time_interval = AUC_UPDATE_TIME_INTERVAL_DEFAULT; } else { auc_update_status.updata_time_stamp_last = time_stamp; set_retry_time_interval(); } } else { if (0 == (get_token_url(device_token, cloud_homecare_url))) { if (0 == get_auc_info_from_cloud(auc_info, device_token, cloud_homecare_url)) { auc_update_status.updata_time_stamp_last = time_stamp; auc_update_status.update_time_interval = AUC_UPDATE_TIME_INTERVAL_DEFAULT; } else { auc_update_status.updata_time_stamp_last = time_stamp; set_retry_time_interval(); } } else { auc_update_status.updata_time_stamp_last = time_stamp; set_retry_time_interval(); } } } else { } URL_CLASS_LOG(LOG_LEVEL_DEBUG, "end time_stamp:%d, updata_time_stamp_last:%d, update_time_interval:%d\n", time_stamp, auc_update_status.updata_time_stamp_last, auc_update_status.update_time_interval); URL_CLASS_LOG(LOG_LEVEL_DEBUG, "auc_server:%s, tp_auc_server:%s, api_key:%s, user_guid:%s\n", auc_info->auc_server, auc_info->tp_auc_server, auc_info->api_key, auc_info->user_guid); return; } #endif static void *mali_checkQuery(void *unused) { int ret = 0; int ret_mali = 0; int failed_num = 0; unsigned int wait_time = 0; parse_para para = {0, 0, 0}; srand((uint32_t)getpid()); while (1) { ret = parse_config(&para); if(0 == ret) { if ((web_protection)&&(para.web_protection)) { ret_mali = mali_url_hash_list_update(true); if (ret_mali < 0) { if (!failed_num) wait_time = 2 + (rand() % 254); else if (wait_time < MALI_MAX_WAIT_TIME) wait_time <<= 1; failed_num++; sleep(MIN(wait_time, MALI_MAX_WAIT_TIME)); } else { if (!access(CLOUD_HASH_FILE_UPDATE_FLAG, R_OK)) { pthread_mutex_lock(&mali_url_check_lock); mali_url_hash_list_update(false); pthread_mutex_unlock(&mali_url_check_lock); } failed_num = 0; } } else if ((web_protection)&&(!para.web_protection)) { pthread_mutex_lock(&mali_url_check_lock); mali_url_check_free(); web_protection = false; pthread_mutex_unlock(&mali_url_check_lock); failed_num = 0; } else if ((!web_protection)&&(para.web_protection)) { pthread_mutex_lock(&mali_url_check_lock); ret_mali = mali_url_check_init(false); if (0 == ret_mali) { web_protection = true; pthread_mutex_unlock(&mali_url_check_lock); failed_num = 0; } else { pthread_mutex_unlock(&mali_url_check_lock); if (!failed_num) wait_time = 2 + (rand() % 254); else if (wait_time < MALI_MAX_WAIT_TIME) wait_time <<= 1; failed_num++; sleep(MIN(wait_time, MALI_MAX_WAIT_TIME)); } } } sleep(5); } } void write_to_csv(const char *url, int original_cat_id, int mapped_cat_id) { int fd = open("/var/auc_classification_results.csv", O_CREAT | O_WRONLY | O_APPEND, 0644); if (fd!= -1) { char buffer[512]; snprintf(buffer, sizeof(buffer), "%s,%d,%d\n", url, original_cat_id, mapped_cat_id); write(fd, buffer, strlen(buffer)); close(fd); } } static void *aucQuery(void *unused) { int ret = 0; struct url_info url_info; bool is_mali = false; #if SUPPORT_CLOUD_UPDATE_AUC_INFO struct auc_info auc_info; #endif unsigned int cat_id = 0; int tmp_id = 0; int index = 0; int index_cat = 0; int index_subcat = 0; unsigned char matched = 0; struct url_entry *tmp_list = NULL; struct url_entry *clean = NULL; unsigned char tmp_url_len; char tmp_url[MAX_URL_LEN] = {0}; printWar("create AUC query thread\n"); while (1) { pthread_mutex_lock(&url_lock); tmp_list = url_list.next; pthread_mutex_unlock(&url_lock); while (tmp_list) { pthread_mutex_lock(&url_lock); tmp_list->process_flag = URL_PROCESSING; pthread_mutex_unlock(&url_lock); url_info.url = tmp_list->url; url_info.info_len = 0; ret = pthread_mutex_trylock(&mali_url_check_lock); if (!ret) { is_mali = web_protection? is_url_malicious(url_info.url) : false; if (is_mali) { cat_id = URL_CAT_SECURITY; #if DEBUG printWar("cat ret=%d url=%s id=%d info_id=%d\n", cat_id, url_info.url, tmp_list->id, tmp_list->info_id); #endif /* DEBUG */ pthread_mutex_lock(&url_lock); tmp_list->prev->next = tmp_list->next; if (tmp_list->next) { tmp_list->next->prev = tmp_list->prev; } tmp_list->query = 1; tmp_id = tmp_list->id; tmp_list->id = 0; tmp_list->info_id = 0; tmp_list->cat_map = 0; tmp_url_len = tmp_list->url_len; memset(tmp_url, 0, MAX_URL_LEN); strncpy_safe(tmp_url, tmp_list->url, tmp_list->url_len); clean = tmp_list; tmp_list = tmp_list->next; clean->next = NULL; clean->prev = NULL; pthread_mutex_unlock(&url_lock); pthread_mutex_unlock(&mali_url_check_lock); write_to_csv(url_info.url, cat_id, cat_id); goto block_mali; } else { pthread_mutex_unlock(&mali_url_check_lock); } } if((!opts.use_tp_service && tmp_list->cat_map) || (opts.use_tp_service && (tmp_list->info_id != PCTL_WEB_URL_ID_ALL) && (tmp_list->cat_map & (~URL_CAT_SECURITY)))) { printWar("before send to auc\n"); #if SUPPORT_CLOUD_UPDATE_AUC_INFO update_libauc_url(&auc_info, opts.use_tp_service); ret = auc_query(&url_info, &auc_info, opts.use_tp_service); #else ret = auc_query(&url_info, opts.use_tp_service); #endif printWar("after send to auc\n"); for (index = 0; index < url_info.info_len; index++) { printWar("cat ret=%d url=%s id=%d name=%s\n", ret, url_info.url, url_info.info[index].id, url_info.info[index].name); } } //remove from url_list pthread_mutex_lock(&url_lock); tmp_list->prev->next = tmp_list->next; if (tmp_list->next) { tmp_list->next->prev = tmp_list->prev; } tmp_list->query = 1; tmp_id = tmp_list->id; tmp_list->id = 0; tmp_list->info_id = 0; tmp_list->cat_map = 0; tmp_url_len = tmp_list->url_len; memset(tmp_url, 0, MAX_URL_LEN); strncpy_safe(tmp_url, tmp_list->url, tmp_list->url_len); clean = tmp_list; tmp_list = tmp_list->next; clean->next = NULL; clean->prev = NULL; pthread_mutex_unlock(&url_lock); //send to kernel cat_id = 0; int original_cat_id = 0; for (index = 0; index < url_info.info_len; index++) { original_cat_id = url_info.info[index].id; for (index_cat = 0; mapping_table[index_cat][0] > 0; index_cat++) { matched = 0; for (index_subcat = 1; mapping_table[index_cat][index_subcat] >= 0; index_subcat++) { if (url_info.info[index].id == mapping_table[index_cat][index_subcat] || mapping_table[index_cat][index_subcat] == MAX_URL_CAT_DEFAULT) { cat_id |= 0x1 << (mapping_table[index_cat][0] - 1); matched = 1; break; } } if (matched) { break; } } } write_to_csv(url_info.url, original_cat_id, cat_id); block_mali: printWar("send to kernel, url=%s cat_id=%x\n", url_info.url, cat_id); url_send(tmp_id, cat_id, tmp_url, tmp_url_len, URL_REQ_CAT); printWar("after send to kernel\n"); } cond_timedwait(500);//500ms } return 0; } static const char* short_opt = "s:h"; static struct option long_opt[] = { {"server" , required_argument , NULL, 'a'}, {"help" , no_argument , NULL, 'h'}, {0 , 0 , NULL, 0 } }; static void printHelp(char *progName); static int parseOpts(int argc, char* argv[], struct Options* opt); static void printHelpInstruction(char *progName); static unsigned int get_max_url_entry_len(void) { unsigned int *url_entry_len = NULL; int state; struct msghdr msg; struct iovec iov; struct nlmsghdr *nlh = NULL; nlh = (struct nlmsghdr *)malloc(NLMSG_SPACE(MAX_PAYLOAD)); if (!nlh) { printErr("malloc nlmsghdr error!\n"); return 0; } nlh->nlmsg_len = NLMSG_SPACE(MAX_PAYLOAD); nlh->nlmsg_pid = getpid(); nlh->nlmsg_flags = 0; nlh->nlmsg_type = URL_REQ_HELLO; iov.iov_base = (void *)nlh; iov.iov_len = NLMSG_SPACE(MAX_PAYLOAD); memset(&msg, 0, sizeof(msg)); msg.msg_name = NULL; msg.msg_namelen = 0; msg.msg_iov = &iov; msg.msg_iovlen = 1; if (sock_fd < 0) { printErr("error getting socket in recving: %s\n", strerror(errno)); return 0; } state = recvmsg(sock_fd, &msg, 0); if(state < 0) { printWar("state<1\n"); return 0; } //store to array url_entry_len = (unsigned int *)NLMSG_DATA(nlh); URL_CLASS_LOG(LOG_LEVEL_DEBUG, "msg from kernel, max_url_entry_len=%u\n", *url_entry_len); return *url_entry_len; } int main(int argc, char* argv[]) { int state; struct sockaddr_nl src_addr, dest_addr; struct nlmsghdr *nlh = NULL; struct iovec iov; struct msghdr msg; int retval; int state_smg = 0; pthread_t pid, pid_mali; int pret = 0; struct url_carrier *tmp = NULL; int index; pid_t fpid = 0; int ret = 0; int rc = parseOpts(argc, argv, &opts); unsigned int url_entry_len = 0; unsigned int malloc_url_entry_len = 0; if (!rc) { return 1; } fpid = fork(); if (fpid < 0) { printErr("error fork.../n"); exit(1); } else if (fpid > 0) { exit(0); } //setup exit handler atexit(on_exit_handler); signal(SIGTERM, signal_exit_handler); signal(SIGINT, signal_exit_handler); // Create a socket sock_fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_URL_CLASS); if (sock_fd < 0) { printErr("error getting socket: %s\n", strerror(errno)); return -1; } int send_buf = 4096; int recv_buf = 4096; if(setsockopt(sock_fd, SOL_SOCKET, SO_SNDBUF, &send_buf, sizeof(send_buf)) < 0) { printErr("setsockopt: %s", strerror(errno)); } if(setsockopt(sock_fd, SOL_SOCKET, SO_RCVBUF, &recv_buf, sizeof(recv_buf)) < 0) { printErr("setsockopt: %s", strerror(errno)); } // To prepare binding memset(&src_addr, 0, sizeof(src_addr)); src_addr.nl_family = AF_NETLINK; src_addr.nl_pid = getpid(); src_addr.nl_groups = 0; retval = bind(sock_fd, (struct sockaddr*)&src_addr, sizeof(src_addr)); if (retval < 0) { printErr("bind failed: %s\n", strerror(errno)); ret = -1; goto exit; } nlh = (struct nlmsghdr *)malloc(NLMSG_SPACE(MAX_PAYLOAD)); if (!nlh) { printErr("malloc nlmsghdr error!\n"); ret = -1; goto exit; } memset(&dest_addr,0,sizeof(dest_addr)); dest_addr.nl_family = AF_NETLINK; dest_addr.nl_pid = 0; dest_addr.nl_groups = 0; nlh->nlmsg_len = NLMSG_SPACE(MAX_PAYLOAD); nlh->nlmsg_pid = getpid(); nlh->nlmsg_flags = 0; nlh->nlmsg_type = URL_REQ_HELLO; iov.iov_base = (void *)nlh; iov.iov_len = NLMSG_SPACE(MAX_PAYLOAD); memset(&msg, 0, sizeof(msg)); msg.msg_name = (void *)&dest_addr; msg.msg_namelen = sizeof(dest_addr); msg.msg_iov = &iov; msg.msg_iovlen = 1; state_smg = sendmsg(sock_fd, &msg, 0);//Hello to kernel if (state_smg < 0) { printErr("get error sendmsg = %s\n",strerror(errno)); ret = -1; goto exit; } url_entry_len = get_max_url_entry_len(); if(url_entry_len == 0) { printErr("error getting url_entry_len\n"); ret = -1; goto exit; } malloc_url_entry_len = url_entry_len * 2.5;//double for backup, ex. id=1 recv and querying, but id=1 deleted by xt_pctl, then another id=1 recv, place it at url_entry_len+1 //if third/fourth... id=1 recv, they are insert in 2*url_entry_len~2.5*url_entry_len in order url_array = (struct url_entry *)malloc(malloc_url_entry_len * sizeof(struct url_entry)); if (!nlh) { printErr("malloc url_array error!\n"); ret = -1; goto exit; } //mali_checkQuery thread and AUC query thread if (pthread_mutex_init(&mali_url_check_lock, NULL) != 0 ) { printErr("mali_checkQuery pthread_mutex_init failed...%d\n", pret); ret = -1; goto exit; } if (pthread_mutex_init(&url_lock, NULL) != 0 || pthread_cond_init(&cond, NULL) != 0) { printErr("auc pthread_mutex_init failed...%d\n", pret); ret = -1; goto exit; } if ((pret = pthread_create(&pid_mali, NULL, mali_checkQuery, NULL)) != 0){ printErr("mali_checkQuery pthread_create failed...%d\n", pret); ret = -1; goto exit; } if ((pret = pthread_create(&pid, NULL, aucQuery, NULL)) != 0){ printErr("auc pthread_create failed...%d\n", pret); ret = -1; goto exit; } memset(url_array, 0, malloc_url_entry_len * sizeof(struct url_entry)); memset(nlh,0,NLMSG_SPACE(MAX_PAYLOAD));//ready to recv kernel info while (1) { state = recvmsg(sock_fd, &msg, 0); if(state < 0) { printWar("state<1\n"); } //store to array tmp = (struct url_carrier *)NLMSG_DATA(nlh); index = tmp->id; //URL_CLASS_LOG(LOG_LEVEL_DEBUG, "msg from kernel, url=%s len=%d id=%d", tmp->url, tmp->url_len, tmp->id); pthread_mutex_lock(&url_lock); if(index >= url_entry_len) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "Error: xt_pctl url entry size is larger than url_class(%u)!", url_entry_len); pthread_mutex_unlock(&url_lock); continue; } if (url_array[index].prev) {//this node is already in list if(url_array[index].url_len != tmp->url_len || strncmp(url_array[index].url, tmp->url, tmp->url_len) || url_array[index].info_id != tmp->info_id || url_array[index].cat_map != tmp->cat_map) { URL_CLASS_LOG(LOG_LEVEL_DEBUG, "Different node already in list, old_url=%s old_len=%d id=%d, old_process_flag=%d", url_array[index].url, url_array[index].url_len, url_array[index].id, url_array[index].process_flag); if(url_array[index].process_flag == URL_PROCESS_READY) {//Still can modify url_array[index].url_len = tmp->url_len; url_array[index].info_id = tmp->info_id; url_array[index].cat_map = tmp->cat_map; strncpy_safe(url_array[index].url, tmp->url, tmp->url_len); pthread_cond_signal(&cond); pthread_mutex_unlock(&url_lock); URL_CLASS_LOG(LOG_LEVEL_DEBUG, "Replace url=%s at url_array[%d]\n", tmp->url, index); continue; } else { index = url_entry_len + tmp->id; if (url_array[index].prev) { if(url_array[index].url_len == tmp->url_len && !strncmp(url_array[index].url, tmp->url, tmp->url_len) && url_array[index].info_id == tmp->info_id && url_array[index].cat_map == tmp->cat_map) {//Exactly same node is already in list URL_CLASS_LOG(LOG_LEVEL_DEBUG, "url=%s at url_array[%d] already exist!\n", tmp->url, index); pthread_mutex_unlock(&url_lock); continue; } if(url_array[index].process_flag == URL_PROCESS_READY) {//Still can modify url_array[index].url_len = tmp->url_len; url_array[index].info_id = tmp->info_id; url_array[index].cat_map = tmp->cat_map; strncpy_safe(url_array[index].url, tmp->url, tmp->url_len); pthread_cond_signal(&cond); pthread_mutex_unlock(&url_lock); URL_CLASS_LOG(LOG_LEVEL_WARN, "Replace url=%s at url_array[%d]\n", tmp->url, index); continue; } index = 2 * url_entry_len; while(index < malloc_url_entry_len && url_array[index].prev) { index++; } if(index >= malloc_url_entry_len) { //An auc query is ignored here! URL_CLASS_LOG(LOG_LEVEL_ERROR, "url_array is full! url=%s query is ignored!\n", tmp->url_len); pthread_mutex_unlock(&url_lock); continue; } } URL_CLASS_LOG(LOG_LEVEL_DEBUG, "Place url=%s at url_array[%d]\n", tmp->url, index); } } else {//Exactly same node is already in list pthread_mutex_unlock(&url_lock); continue; } } url_array[index].id = tmp->id; url_array[index].query = 0; url_array[index].process_flag = URL_PROCESS_READY; url_array[index].url_len = tmp->url_len; url_array[index].info_id = tmp->info_id; url_array[index].cat_map = tmp->cat_map; strncpy_safe(url_array[index].url, tmp->url, tmp->url_len); printWar("msg from kernel, url=%s len=%d id=%d\n", url_array[index].url, url_array[index].url_len, url_array[index].id); //add to list url_array[index].next = url_list.next; if (url_list.next) { url_list.next->prev = &url_array[index]; } url_array[index].prev = &url_list; url_list.next = &url_array[index]; pthread_cond_signal(&cond); pthread_mutex_unlock(&url_lock); } exit: if (nlh) { free(nlh); } if (url_array) { free(url_array); } on_exit_handler(); return ret; } static size_t _strlcpy(char *dst, const char *src, size_t dstsize) { size_t srclen = (size_t)strlen(src); if (dstsize > 0) { size_t len = (srclen >= dstsize) ? dstsize - 1 : srclen; memset(dst, 0, (len + 1)); memcpy(dst, src, len); } return srclen; } static struct uci_context *_uci_context_init(const char *config_path) { struct uci_context *uci_ctx = NULL; uci_ctx = uci_alloc_context(); if (uci_ctx) { uci_set_confdir(uci_ctx, config_path); } return uci_ctx; } static void _uci_context_free(struct uci_context *uci_ctx) { if (uci_ctx) { uci_free_context(uci_ctx); } } static int _uci_get_value(char * p_uci_str, char* p_value) { struct uci_context *uci_ctx = NULL; struct uci_element *e = NULL; struct uci_ptr p_uci; if (NULL == p_uci_str || NULL == p_value) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "p_uci_str or p_value is null"); goto error; } uci_ctx = _uci_context_init("/etc/config"); if (!uci_ctx) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "fail to init uci context:%s", p_uci_str); goto error; } if (UCI_OK != uci_lookup_ptr(uci_ctx, &p_uci, p_uci_str, true)) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "fail to get ptr %s ", p_uci_str); goto error; } e = p_uci.last; if (UCI_TYPE_OPTION != e->type) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "element type is not option:%d", e->type); goto error; } if (UCI_TYPE_STRING != p_uci.o->type) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "option type is not string:%d", p_uci.o->type); goto error; } _strlcpy(p_value, p_uci.o->v.string, 64); URL_CLASS_LOG(LOG_LEVEL_DEBUG, "Success to get option value %s = %s ", p_uci_str,p_uci.o->v.string); _uci_context_free(uci_ctx); return 0; error: _uci_context_free(uci_ctx); return -1; } int parseOpts(int argc, char* argv[], struct Options* opt) { int c; int rc = 999; int use_tp_service = -1; while((c = getopt_long(argc, argv, short_opt, long_opt, NULL)) != -1) { switch(c) { case -1: /* no more arguments */ case 0: /* long options toggles */ break; case 's': if (optarg) { if(strcmp(optarg, "avira") == 0) { use_tp_service = 0; URL_CLASS_LOG(LOG_LEVEL_DEBUG, "parse Opts receive use avira service!"); } else if(strcmp(optarg, "tplink") == 0) { use_tp_service = 1; URL_CLASS_LOG(LOG_LEVEL_DEBUG, "parse Opts receive use tplink service!"); } else { URL_CLASS_LOG(LOG_LEVEL_ERROR, "parse Opts receive %s, not avaliable! Try %s -s tplink or %s -s avira", optarg, argv[0], argv[0]); } } rc = 's'; break; case 't': rc = 't'; break; case 'h': printHelp(argv[0]); rc = 0; break; case ':': case '?': printHelpInstruction(argv[0]); rc = 0; break; default: printf("%s : invalid option --\n", c); printHelpInstruction(argv[0]); rc = 0; break; } } if(use_tp_service == -1) { char _key[64] = "avira.info.status"; char _value[64] = {0}; if (_uci_get_value(_key, _value)) { URL_CLASS_LOG(LOG_LEVEL_DEBUG, "-s Parameter not found! Config avira.info.status not found! use default avira service!"); opt->use_tp_service = false; } else { if(strncmp(_value, "available", sizeof("available") - 1) == 0 || strncmp(_value, "libauc_available", sizeof("libauc_available") - 1) == 0) { URL_CLASS_LOG(LOG_LEVEL_DEBUG, "-s Parameter not found! Config avira.info.status=%s! use avira service!", _value); opt->use_tp_service = false; } else { URL_CLASS_LOG(LOG_LEVEL_DEBUG, "-s Parameter not found! Config avira.info.status=%s! use tplink service!", _value); opt->use_tp_service = true; } } } else { opt->use_tp_service = use_tp_service ? true : false; } return rc; } void printHelpInstruction(char *progName) { printf("Try: %s --help for more information\n", progName); } void printHelp(char *progName) { printf("Usage: %s\n\t -s <tplink/avira> Use the specified service provider", progName); } 还原出代码格式
最新发布
09-23
# -*- coding: utf-8 -*- import tkinter as tk from tkinter import ttk, messagebox, scrolledtext import socket import threading import time import random import struct from datetime import datetime class DOIPTester: def __init__(self, root): self.root = root self.root.title("DOIP Tester v1.0") self.root.geometry("900x700") self.root.resizable(True, True) # 设置主题 self.style = ttk.Style() self.style.theme_use("clam") # 创建标签页 self.tab_control = ttk.Notebook(root) # 配置标签页 self.config_tab = ttk.Frame(self.tab_control) self.diagnostic_tab = ttk.Frame(self.tab_control) self.log_tab = ttk.Frame(self.tab_control) self.tab_control.add(self.config_tab, text='配置') self.tab_control.add(self.diagnostic_tab, text='诊断') self.tab_control.add(self.log_tab, text='志') self.tab_control.pack(expand=1, fill="both") # 初始化变量 self.protocol_version = tk.IntVar(value=2) self.source_address = tk.StringVar(value="0x0E80") self.target_address = tk.StringVar(value="0x0001") self.vin = tk.StringVar(value="VIN12345678901234") self.eid = tk.StringVar(value="EID12345678901234") self.logical_address = tk.StringVar(value="0x0001") self.diagnostic_data = tk.StringVar(value="10 02") self.connection_status = tk.StringVar(value="未连接") self.activation_status = tk.StringVar(value="未激活") # 初始化志 self.log_messages = [] # 创建配置标签页 self.create_config_tab() # 创建诊断标签页 self.create_diagnostic_tab() # 创建志标签页 self.create_log_tab() # 初始化状态栏 self.status_bar = tk.Label(root, textvariable=self.connection_status, bd=1, relief=tk.SUNKEN, anchor=tk.W) self.status_bar.pack(side=tk.BOTTOM, fill=tk.X) # 模拟连接状态 self.simulate_connection() def create_config_tab(self): # 协议配置框架 config_frame = ttk.LabelFrame(self.config_tab, text="协议配置") config_frame.pack(padx=10, pady=10, fill=tk.X) # 协议版本 ttk.Label(config_frame, text="协议版本:").grid(row=0, column=0, padx=5, pady=5, sticky=tk.W) version_frame = ttk.Frame(config_frame) version_frame.grid(row=0, column=1, padx=5, pady=5, sticky=tk.W) ttk.Radiobutton(version_frame, text="2010 (0x01)", variable=self.protocol_version, value=1).pack(side=tk.LEFT) ttk.Radiobutton(version_frame, text="2012 (0x02)", variable=self.protocol_version, value=2).pack(side=tk.LEFT, padx=(10,0)) # 源地址 ttk.Label(config_frame, text="源地址:").grid(row=1, column=0, padx=5, pady=5, sticky=tk.W) ttk.Entry(config_frame, textvariable=self.source_address, width=10).grid(row=1, column=1, padx=5, pady=5, sticky=tk.W) # 目标地址 ttk.Label(config_frame, text="目标地址:").grid(row=2, column=0, padx=5, pady=5, sticky=tk.W) ttk.Entry(config_frame, textvariable=self.target_address, width=10).grid(row=2, column=1, padx=5, pady=5, sticky=tk.W) # VIN码 ttk.Label(config_frame, text="VIN码:").grid(row=3, column=0, padx=5, pady=5, sticky=tk.W) ttk.Entry(config_frame, textvariable=self.vin, width=25).grid(row=3, column=1, padx=5, pady=5, sticky=tk.W) # EID ttk.Label(config_frame, text="EID:").grid(row=4, column=0, padx=5, pady=5, sticky=tk.W) ttk.Entry(config_frame, textvariable=self.eid, width=25).grid(row=4, column=1, padx=5, pady=5, sticky=tk.W) # 连接按钮 connect_frame = ttk.Frame(self.config_tab) connect_frame.pack(padx=10, pady=10, fill=tk.X) self.connect_btn = ttk.Button(connect_frame, text="连接", command=self.connect) self.connect_btn.pack(side=tk.LEFT, padx=5) ttk.Button(connect_frame, text="断开", command=self.disconnect).pack(side=tk.LEFT, padx=5) ttk.Button(connect_frame, text="车辆识别", command=self.vehicle_identification).pack(side=tk.LEFT, padx=5) ttk.Button(connect_frame, text="路由激活", command=self.routing_activation).pack(side=tk.LEFT, padx=5) # 状态显示 status_frame = ttk.Frame(self.config_tab) status_frame.pack(padx=10, pady=10, fill=tk.X) ttk.Label(status_frame, text="连接状态:").pack(side=tk.LEFT, padx=5) ttk.Label(status_frame, textvariable=self.connection_status, foreground="red").pack(side=tk.LEFT, padx=5) ttk.Label(status_frame, text="路由状态:").pack(side=tk.LEFT, padx=(20,5)) ttk.Label(status_frame, textvariable=self.activation_status, foreground="red").pack(side=tk.LEFT, padx=5) def create_diagnostic_tab(self): # 诊断数据框架 diag_frame = ttk.LabelFrame(self.diagnostic_tab, text="诊断数据") diag_frame.pack(padx=10, pady=10, fill=tk.BOTH, expand=True) # 逻辑地址 ttk.Label(diag_frame, text="逻辑地址:").grid(row=0, column=0, padx=5, pady=5, sticky=tk.W) ttk.Entry(diag_frame, textvariable=self.logical_address, width=10).grid(row=0, column=1, padx=5, pady=5, sticky=tk.W) # 诊断数据 ttk.Label(diag_frame, text="诊断数据:").grid(row=1, column=0, padx=5, pady=5, sticky=tk.W) ttk.Entry(diag_frame, textvariable=self.diagnostic_data, width=50).grid(row=1, column=1, padx=5, pady=5, sticky=tk.W) # 发送按钮 ttk.Button(diag_frame, text="发送诊断数据", command=self.send_diagnostic_data).grid(row=2, column=1, padx=5, pady=10, sticky=tk.W) # 诊断历史 diag_history_frame = ttk.LabelFrame(diag_frame, text="诊断历史") diag_history_frame.grid(row=3, column=0, columnspan=2, padx=5, pady=5, sticky=tk.W+tk.E+tk.N+tk.S) columns = ("timestamp", "direction", "address", "data", "response") self.diag_tree = ttk.Treeview(diag_history_frame, columns=columns, show="headings", height=8) self.diag_tree.heading("timestamp", text="时间") self.diag_tree.heading("direction", text="方向") self.diag_tree.heading("address", text="地址") self.diag_tree.heading("data", text="数据") self.diag_tree.heading("response", text="响应") self.diag_tree.column("timestamp", width=120) self.diag_tree.column("direction", width=60) self.diag_tree.column("address", width=80) self.diag_tree.column("data", width=200) self.diag_tree.column("response", width=200) vsb = ttk.Scrollbar(diag_history_frame, orient="vertical", command=self.diag_tree.yview) self.diag_tree.configure(yscrollcommand=vsb.set) self.diag_tree.pack(side=tk.LEFT, fill=tk.BOTH, expand=True) vsb.pack(side=tk.RIGHT, fill=tk.Y) # 添加示例数据 for i in range(5): self.add_diag_history_entry( datetime.now().strftime("%H:%M:%S.%f")[:-3], "发送" if i % 2 == 0 else "接收", f"0x{1000 + i:04X}", "10 02" if i % 2 == 0 else "50 02", "成功" if i % 2 == 0 else "N/A" ) def create_log_tab(self): # 志框架 log_frame = ttk.Frame(self.log_tab) log_frame.pack(fill=tk.BOTH, expand=True, padx=10, pady=10) self.log_text = scrolledtext.ScrolledText(log_frame, wrap=tk.WORD) self.log_text.pack(fill=tk.BOTH, expand=True) self.log_text.config(state=tk.DISABLED) # 添加初始志 self.log_message("DOIP测试上位机已启动") self.log_message("请配置连接参数并连接到目标设备") # 志控制按钮 btn_frame = ttk.Frame(self.log_tab) btn_frame.pack(fill=tk.X, padx=10, pady=5) ttk.Button(btn_frame, text="清空志", command=self.clear_log).pack(side=tk.LEFT, padx=5) ttk.Button(btn_frame, text="保存志", command=self.save_log).pack(side=tk.LEFT, padx=5) def add_diag_history_entry(self, timestamp, direction, address, data, response): self.diag_tree.insert("", tk.END, values=(timestamp, direction, address, data, response)) # 自动滚动到最后 self.diag_tree.yview_moveto(1) def log_message(self, message): timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3] log_entry = f"[{timestamp}] {message}" self.log_messages.append(log_entry) self.log_text.config(state=tk.NORMAL) self.log_text.insert(tk.END, log_entry + "\n") self.log_text.config(state=tk.DISABLED) self.log_text.yview(tk.END) def clear_log(self): self.log_text.config(state=tk.NORMAL) self.log_text.delete(1.0, tk.END) self.log_text.config(state=tk.DISABLED) self.log_messages = [] self.log_message("志已清空") def save_log(self): self.log_message("志保存功能尚未实现") def simulate_connection(self): """模拟连接状态变化""" def update_status(): if self.connection_status.get() == "未连接": self.connection_status.set("已连接") self.connect_btn.config(text="断开") self.log_message("已连接到目标设备") else: self.connection_status.set("未连接") self.connect_btn.config(text="连接") self.activation_status.set("未激活") self.log_message("已断开连接") # 模拟连接状态变化 threading.Thread(target=lambda: (time.sleep(1), update_status()), daemon=True).start() def connect(self): if self.connection_status.get() == "未连接": self.connection_status.set("连接中...") self.log_message(f"尝试连接到目标 {self.target_address.get()}") # 模拟连接过程 threading.Thread(target=self.simulate_connection, daemon=True).start() else: self.disconnect() def disconnect(self): if self.connection_status.get() != "未连接": self.connection_status.set("断开中...") self.log_message("断开连接...") # 模拟断开过程 threading.Thread(target=lambda: (time.sleep(1), self.connection_status.set("未连接"), self.connect_btn.config(text="连接"), self.log_message("已断开连接")), daemon=True).start() def vehicle_identification(self): if self.connection_status.get() != "已连接": messagebox.showwarning("警告", "请先连接到目标设备") return self.log_message("发送车辆识别请求...") # 模拟车辆识别响应 def simulate_response(): time.sleep(0.5) vin = self.vin.get() eid = self.eid.get() logical_address = self.logical_address.get() self.log_message(f"收到车辆识别响应: VIN={vin}, EID={eid}, 逻辑地址={logical_address}") # 在诊断历史中添加条目 self.add_diag_history_entry( datetime.now().strftime("%H:%M:%S.%f")[:-3], "接收", "0xFFFF", f"VIN: {vin}", "成功" ) threading.Thread(target=simulate_response, daemon=True).start() def routing_activation(self): if self.connection_status.get() != "已连接": messagebox.showwarning("警告", "请先连接到目标设备") return self.log_message("发送路由激活请求...") self.activation_status.set("激活中...") # 模拟路由激活响应 def simulate_response(): time.sleep(0.8) self.activation_status.set("已激活") self.log_message("路由激活成功") # 在诊断历史中添加条目 self.add_diag_history_entry( datetime.now().strftime("%H:%M:%S.%f")[:-3], "接收", self.target_address.get(), "Routing Activation Response", "成功" ) threading.Thread(target=simulate_response, daemon=True).start() def send_diagnostic_data(self): if self.connection_status.get() != "已连接": messagebox.showwarning("警告", "请先连接到目标设备") return if self.activation_status.get() != "已激活": messagebox.showwarning("警告", "请先激活路由") return data = self.diagnostic_data.get() address = self.logical_address.get() if not data: messagebox.showwarning("警告", "请输入诊断数据") return self.log_message(f"发送诊断数据到地址 {address}: {data}") # 在诊断历史中添加发送条目 self.add_diag_history_entry( datetime.now().strftime("%H:%M:%S.%f")[:-3], "发送", address, data, "等待响应..." ) # 模拟诊断响应 def simulate_response(): time.sleep(random.uniform(0.2, 1.0)) # 生成模拟响应数据 if data.startswith("10"): response_data = "50" + data[2:] else: response_data = f"7F{data[:2]} 11" # 否定响应 self.log_message(f"收到诊断响应: {response_data}") # 在诊断历史中更新条目 self.add_diag_history_entry( datetime.now().strftime("%H:%M:%S.%f")[:-3], "接收", address, response_data, "成功" if not response_data.startswith("7F") else "失败" ) threading.Thread(target=simulate_response, daemon=True).start() if __name__ == "__main__": root = tk.Tk() app = DOIPTester(root) root.mainloop()上面代码第159行报错 f"0x{1000 + i:04X}", ^ SyntaxError: invalid syntax
08-23
/*!Copyright(c) 2012-2019 Shenzhen TP-LINK Technologies Co.Ltd. * *\file url_class.c *\brief process for URL query * *\author Wanghao *\version 1.0.0 *\date 27Aug19 * *\history * */ #include <sys/stat.h> #include <unistd.h> #include <stdio.h> #include <stdlib.h> #include <sys/socket.h> #include <sys/types.h> #include <string.h> #include <asm/types.h> #include <linux/netlink.h> #include <linux/socket.h> #include <getopt.h> #include <errno.h> #include <avira/auc/aucApi.h> #include <pthread.h> #include <signal.h> #include <sys/time.h> #include <curl/curl.h> #include "cJSON.h" #include <uci.h> #include <mali_url_check.h> #define printErr(fmt, args...) printf("\033[1m[ %s ] %03d: "fmt"\033[0m", __FUNCTION__, __LINE__, ##args) #define printWar(fmt, args...) printf("\033[1m[ %s ] %03d: "fmt"\033[0m", __FUNCTION__, __LINE__, ##args) #define CURL_DEBUG 0 #define URL_CLASS_LOG_LEVEL LOG_LEVEL_MAX #define URL_CLASS_LOG(level, fmt, args...) \ do \ { \ if (level >= URL_CLASS_LOG_LEVEL) \ { \ static char cmd[1024] = {0}, final[1024+128] = {0}; \ struct timeval tv; \ struct tm* local; \ gettimeofday(&tv, NULL); \ local = localtime(&tv.tv_sec); \ memset(cmd,0,sizeof(cmd));memset(final,0,sizeof(final));\ snprintf(cmd, sizeof(cmd), fmt, ##args); \ snprintf(final, sizeof(final), "echo \"%04d-%02d-%02d %02d:%02d:%02d.%03d|level:%d|%s:%d|%s| - ",\ local->tm_year + 1900, local->tm_mon + 1, local->tm_mday, \ local->tm_hour, local->tm_min, local->tm_sec, (int)(tv.tv_usec / 1000), \ level, __FILE__, __LINE__, __func__); \ int _i = 0, _j = strlen(final); \ for(_i = 0; _i < 1024 && _j < 1024+128-15; _i++,_j++) \ { \ if(cmd[_i] == '\"' || cmd[_i] == '\\' || cmd[_i] == '`' || cmd[_i] == '$') \ { \ final[_j++] = '\\'; \ } \ final[_j] = cmd[_i]; \ } \ strncpy(final + (final[strlen(final) - 1] == '\n'? strlen(final)-1: strlen(final)), "\"> /dev/console", 16); \ system(final); \ } \ } while (0) #define MIN(a,b) ((a)<(b) ? (a):(b)) #define MALI_MAX_WAIT_TIME 1024 typedef enum { LOG_LEVEL_DEBUG = 0, // log level debug LOG_LEVEL_INFO, // log level information LOG_LEVEL_WARN, // log level warning LOG_LEVEL_ERROR, // log level error LOG_LEVEL_EMRG, // log level emergency LOG_LEVEL_MAX }E_LOG_LEVE; struct Options { bool use_tp_service; }; struct Options opts; #define MAX_PAYLOAD 1024 // maximum payload size #ifndef NETLINK_URL_CLASS #define NETLINK_URL_CLASS 27 #endif //#define MAX_URL_ENTRY_LEN 512//256 #define MAX_URL_LEN 256 #define MAX_URL_CAT_DEFAULT 9999 #define PCTL_WEB_URL_ID_ALL 0xfffe #if SUPPORT_CLOUD_UPDATE_AUC_INFO #define POST_TRANSFER_TIMEOUT 20 #define POST_CONNECT_TIMEOUT 20 #define AUC_UPDATE_TIME_INTERVAL_DEFAULT 3600 //1h #define AUC_UPDATE_TIME_INTERVAL_FAST 10 //10s #define REQUEST_TIMEOUT_MS 8000 #define MAX_TOKEN_LEN 128 #define FILE_CLOUD_TOKEN_HOMECARE "/tmp/cloud/cloud_token_homecare" #define GET_AUC_URL_POSTFIX "/v1/configuration/auc-service-info" #define CA_CRT_PATH "/etc/certificate/2048_newroot.cer" struct auc_update_status_info { int update_time_interval; time_t updata_time_stamp_last; }; static struct auc_update_status_info auc_update_status = { .update_time_interval = AUC_UPDATE_TIME_INTERVAL_DEFAULT, .updata_time_stamp_last = 0, }; typedef struct _st_http_resinfo { long status_code; }st_http_resinfo; #endif enum url_req { URL_REQ_VOID = 0, URL_REQ_HELLO, URL_REQ_CAT, URL_REQ_LOG, URL_REQ_LOG_HISTORY, URL_REQ_BYE, URL_REQ_END }; struct url_carrier { int id; int info_id; unsigned char url_len; unsigned int cat_id; unsigned int cat_map; char url[MAX_URL_LEN]; }; enum url_process_status { URL_PROCESS_READY = 0, //waiting for auc process URL_PROCESSING //auc is processing, changing url is useless }; struct url_entry { int id; int info_id; unsigned int cat_map; unsigned char query; unsigned char url_len; enum url_process_status process_flag; struct url_entry *prev; struct url_entry *next; char url[MAX_URL_LEN]; }; pthread_mutex_t url_lock = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_t mali_url_check_lock = PTHREAD_MUTEX_INITIALIZER; pthread_cond_t cond = PTHREAD_COND_INITIALIZER; //struct url_entry url_array[MAX_URL_ENTRY_LEN]; struct url_entry *url_array; static struct url_entry url_list = { .id = 0, .info_id = 0, .cat_map = 0, .query = 0, .url_len = 0, .prev = NULL, .next = NULL, }; int sock_fd = -1; //cat mapping table int mapping_table[][32] = { {1, 274, 276, 374, 132, -1}, //1100, 1-Games {2, 193, 447, 449, -1}, //1000, 2-Download {3, 222, 223, 224, 357, -1}, //900, 3-Media {4, 107, 388, -1}, //800, 4-Pay-to-surf & Shopping {5, 414, 445, -1}, //700, 5-Social Networking {6, 359, 360, 355, 293, 397, 450, 451, -1}, //600, 6-Oline Communications and Search {7, 443, -1}, //500, 7-Gambling {10, 241, 179, 76, 170, 167, -1}, //200, 10-Sex Education {11, 234, 400, 269, 444, 446, 395, 429, 391, 390, 103, 64, 65, 66, 149, 452, 453, 442, -1}, //100, 11-Adult Content {13, 2, -1}, //000, unsafe, 13-Malware {14, 3, -1}, //000, unsafe, 14-Phishing {15, 197, -1}, // 15-web_search {12, MAX_URL_CAT_DEFAULT, -1}, //2000, 12-Other(Default) {-1} //END }; static inline void strncpy_safe(char *__dest, const char *__src, size_t __n) { strncpy(__dest, __src, __n); __dest[__n] = '\0'; } static void on_exit_handler() { struct sockaddr_nl dest_addr; struct nlmsghdr *nlh = NULL; struct iovec iov; struct msghdr msg; int state_smg = 0; nlh = (struct nlmsghdr *)malloc(NLMSG_SPACE(MAX_PAYLOAD)); if (!nlh) { printErr("malloc nlmsghdr error!\n"); close(sock_fd); return; } //say bye to kernel memset(nlh,0,NLMSG_SPACE(MAX_PAYLOAD)); memset(&dest_addr,0,sizeof(dest_addr)); dest_addr.nl_family = AF_NETLINK; dest_addr.nl_pid = 0; dest_addr.nl_groups = 0; nlh->nlmsg_len = NLMSG_SPACE(MAX_PAYLOAD); nlh->nlmsg_pid = 0; nlh->nlmsg_flags = 0; nlh->nlmsg_type = URL_REQ_BYE; iov.iov_base = (void *)nlh; iov.iov_len = NLMSG_SPACE(MAX_PAYLOAD); memset(&msg, 0, sizeof(msg)); msg.msg_name = (void *)&dest_addr; msg.msg_namelen = sizeof(dest_addr); msg.msg_iov = &iov; msg.msg_iovlen = 1; state_smg = sendmsg(sock_fd, &msg, 0); if (state_smg < 0) { printErr("get error sendmsg = %s\n",strerror(errno)); } if (nlh) { free(nlh); } close(sock_fd); return; } static void signal_exit_handler(int sig) { exit(0); } static int url_send(int id, unsigned int cat_id, char *url, unsigned char url_len, unsigned short type) { struct nlmsghdr *nlh = NULL; struct iovec iov; struct msghdr msg; struct sockaddr_nl dest_addr; struct url_carrier data; int state_smg = 0; int ret = 0; nlh = (struct nlmsghdr *)malloc(NLMSG_SPACE(MAX_PAYLOAD)); if (!nlh) { printErr("malloc nlmsghdr error!\n"); return -1; } memset(&dest_addr,0,sizeof(dest_addr)); dest_addr.nl_family = AF_NETLINK; dest_addr.nl_pid = 0; dest_addr.nl_groups = 0; nlh->nlmsg_len = NLMSG_SPACE(MAX_PAYLOAD); nlh->nlmsg_pid = getpid(); nlh->nlmsg_flags = 0; nlh->nlmsg_type = type; memset(&data, 0, sizeof(struct url_carrier)); data.id = id; data.cat_id = cat_id; data.url_len = url_len; strncpy_safe(data.url, url, url_len); printWar("url_send to kernel, id=%d, cat_id=%x, url=%s, url_len=%d \n", data.id, data.cat_id, data.url, data.url_len); memcpy(NLMSG_DATA(nlh), &data, sizeof(struct url_carrier)); iov.iov_base = (void *)nlh; iov.iov_len = NLMSG_SPACE(MAX_PAYLOAD); memset(&msg, 0, sizeof(msg)); msg.msg_name = (void *)&dest_addr; msg.msg_namelen = sizeof(dest_addr); msg.msg_iov = &iov; msg.msg_iovlen = 1; state_smg = sendmsg(sock_fd, &msg, 0); if (state_smg < 0) { printErr("get error sendmsg = %s\n",strerror(errno)); ret = -1; goto out; } out: if (nlh) { free(nlh); } return ret; } static void __attribute__((unused)) dump_list(struct url_entry *list) { printErr("dump:\n"); while (list) { printWar("id=%d url=%s\n", list->id, list->url); list = list->next; } } static void cond_timedwait(long timeout) { struct timespec abstime; struct timeval now; long nsec = 0; pthread_mutex_lock(&url_lock); gettimeofday(&now, NULL); nsec = now.tv_usec * 1000 + (timeout % 1000) * 1000000; abstime.tv_sec=now.tv_sec + nsec / 1000000000 + timeout / 1000; abstime.tv_nsec=nsec % 1000000000; pthread_cond_timedwait(&cond, &url_lock, &abstime); pthread_mutex_unlock(&url_lock); } #if SUPPORT_CLOUD_UPDATE_AUC_INFO static int OnDebug(CURL * curl, curl_infotype itype, char * pData, size_t size, void *lpVoid) { if (itype == CURLINFO_TEXT) { //URL_CLASS_LOG(LOG_LEVEL_DEBUG,"[TEXT]%s", pData); } else if (itype == CURLINFO_HEADER_IN) { URL_CLASS_LOG(LOG_LEVEL_DEBUG, "[HEADER_IN]%s", pData); } else if (itype == CURLINFO_HEADER_OUT) { URL_CLASS_LOG(LOG_LEVEL_DEBUG, "[HEADER_OUT]%s", pData); } else if (itype == CURLINFO_DATA_IN) { URL_CLASS_LOG(LOG_LEVEL_DEBUG, "[DATA_IN]%s", pData); } else if (itype == CURLINFO_DATA_OUT) { URL_CLASS_LOG(LOG_LEVEL_DEBUG, "[DATA_OUT]%s", pData); } return 0; } static size_t OnWriteData_Post(void* buffer, size_t size, size_t nmemb, void* lpVoid) { if (NULL == buffer) { return -1; } unsigned int len = (unsigned int)size * (unsigned int)nmemb; char *str = (char*)malloc(len + 1); if (NULL == str) { return -1; } char* pData = (char*)buffer; memset(str, 0, len + 1); memcpy(str, pData, len); URL_CLASS_LOG(LOG_LEVEL_DEBUG, "response: %s\n", str); char **response = (char**)lpVoid; *response = str; return len; } static size_t cloud_https_post(const char *pUrl, const char *requestHeader, const char *request, char **response, st_http_resinfo *pHttpResInfo) { CURLcode res; CURL* curl = NULL; struct curl_slist *headers = NULL; #ifdef CURL_DEBUG char errbuf[CURL_ERROR_SIZE]; memset(errbuf, '\0', CURL_ERROR_SIZE); #endif if (NULL == pUrl || NULL == request || NULL == response || NULL == pHttpResInfo) { return CURLE_FAILED_INIT; } URL_CLASS_LOG(LOG_LEVEL_DEBUG, "pUrl:%s\n requestHeader:%s\n request:%s\n", pUrl, requestHeader, request); res = curl_global_init(CURL_GLOBAL_ALL); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl global init fail and ret %d", res); return res; } curl = curl_easy_init(); if (NULL == curl) { res = CURLE_FAILED_INIT; URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl init fail"); goto exit; } //headers = curl_slist_append(headers, "Content-Type: application/json;charset=UTF-8"); headers = curl_slist_append(headers, "Content-Type: application/json"); if (NULL == headers) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl get header list fail"); goto exit; } if ((NULL != requestHeader) && (0 < strlen(requestHeader))) { URL_CLASS_LOG(LOG_LEVEL_DEBUG, "requestHeader is %s", requestHeader); headers = curl_slist_append(headers, requestHeader); if (NULL == headers) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl append requestHeader fail"); goto exit; } } #ifdef CURL_DEBUG //provide a buffer to store errors in res = curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_ERRORBUFFER ret %d", res); goto exit; } #endif if (CURL_DEBUG) { URL_CLASS_LOG(LOG_LEVEL_DEBUG, "post sesstion debug on"); res = curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_VERBOSE ret %d", res); goto exit; } res = curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, OnDebug); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_DEBUGFUNCTION ret %d", res); goto exit; } } //set url res = curl_easy_setopt(curl, CURLOPT_URL, pUrl); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_URL ret %d", res); goto exit; } //set header res = curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_HTTPHEADER ret %d", res); goto exit; } //set post method res = curl_easy_setopt(curl, CURLOPT_POST, 1); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_POST ret %d", res); goto exit; } res = curl_easy_setopt(curl, CURLOPT_POSTFIELDS, request); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_POSTFIELDS ret %d", res); goto exit; } //set read/write params res = curl_easy_setopt(curl, CURLOPT_READFUNCTION, NULL); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_READFUNCTION ret %d", res); goto exit; } res = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, OnWriteData_Post); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_WRITEFUNCTION ret %d", res); goto exit; } res = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)response); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_WRITEDATA ret %d", res); goto exit; } res = curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_NOSIGNAL ret %d", res); goto exit; } //set certificate info res = curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_SSL_VERIFYPEER ret %d", res); goto exit; } res = curl_easy_setopt(curl, CURLOPT_CAINFO, CA_CRT_PATH); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_CAINFO ret %d", res); goto exit; } //set CURLOPT_CONNECTTIMEOUT res = curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, POST_TRANSFER_TIMEOUT); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_CONNECTTIMEOUT ret %d", res); goto exit; } //set CURLOPT_TIMEOUT res = curl_easy_setopt(curl, CURLOPT_TIMEOUT, POST_CONNECT_TIMEOUT); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_TIMEOUT ret %d", res); goto exit; } //CURLOPT needs to be set_ FOLLOWLOCATION is 1, otherwise, the data after redirecting will not be returned res = curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION,1); if (CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "curl set option CURLOPT_FOLLOWLOCATION ret %d", res); goto exit; } res = curl_easy_perform(curl); #ifdef CURL_DEBUG if(CURLE_OK != res) { URL_CLASS_LOG(LOG_LEVEL_DEBUG, "curl post return error: %s", errbuf); } #endif curl_easy_getinfo(curl, CURLINFO_HTTP_CODE, &(pHttpResInfo->status_code)); URL_CLASS_LOG(LOG_LEVEL_DEBUG, "cloud_https_post done. ret %d, http status code %d", res, pHttpResInfo->status_code); exit: if (headers) { curl_slist_free_all(headers); } if (curl) { curl_easy_cleanup(curl); } curl_global_cleanup(); return res; } static int get_token_url(char *token, char *url) { int ret = -1; int retry_count = 3; FILE *fp = NULL; while (retry_count > 0) { if(fp = fopen(FILE_CLOUD_TOKEN_HOMECARE, "r")) { fscanf(fp, "%s", token); fscanf(fp, "%s", url); fclose(fp); ret = 0; break; } else { system("cloud_getDevToken homecare"); } retry_count--; } if ((*(token + MAX_TOKEN_LEN - 1) != 0) || (*(url + MAX_URL_LEN - 1) != 0)) { URL_CLASS_LOG(LOG_LEVEL_WARN, "token or url pointer out of range"); ret = -1; } return ret; } static char* get_auc_info_from_cloud_request_data() { int ret = -1; cJSON* request_data_json = NULL; char* request_data = NULL; unsigned char country_code[3] = {0}; FILE *fp = NULL; if ((request_data_json = cJSON_CreateObject()) == NULL) { goto out; } if (fp = popen("getfirm COUNTRY | tr 'A-Z' 'a-z'", "r")) { if (fread(country_code, sizeof(char), 2, fp) > 0) { cJSON_AddStringToObject(request_data_json, (const char*)"region", (const char*)country_code); request_data = cJSON_PrintUnformatted(request_data_json); URL_CLASS_LOG(LOG_LEVEL_DEBUG, "request_data:%s", request_data); } pclose(fp); } cJSON_Delete(request_data_json); return request_data; out: return NULL; } static int get_auc_info_from_cloud(struct auc_info *auc_info, char *device_token, char *url_prefix) { int ret = -1; int ret_post = 0; char request_url[MAX_URL_LEN] = {0}; char request_header[MAX_TOKEN_LEN + 32] = {0}; char *request_data = NULL; char *response = NULL; st_http_resinfo httpResInfo; cJSON* response_json = NULL; cJSON* auc_server_json= NULL; cJSON* api_key_json = NULL; cJSON* user_guid_json = NULL; if (auc_info == NULL || device_token == NULL || url_prefix == NULL) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "params error\n"); return ret; } memset(&httpResInfo, 0, sizeof(st_http_resinfo)); snprintf(request_url, MAX_URL_LEN, "%s%s", url_prefix, GET_AUC_URL_POSTFIX); snprintf(request_header, MAX_TOKEN_LEN + 32, "Authorization: %s", device_token); request_data = get_auc_info_from_cloud_request_data(); //get post result ret_post = cloud_https_post(request_url, request_header, request_data, &response, &httpResInfo); URL_CLASS_LOG(LOG_LEVEL_DEBUG, "curl post return %d http retcode: %ld", ret_post, httpResInfo.status_code); if(request_data) { cJSON_free(request_data); } if (response && httpResInfo.status_code == 200) { response_json = cJSON_Parse(response); auc_server_json= cJSON_GetObjectItem(response_json, "aucServerUrl"); api_key_json = cJSON_GetObjectItem(response_json, "apiKey"); user_guid_json = cJSON_GetObjectItem(response_json, "userGuid"); if (auc_server_json->valuestring && api_key_json->valuestring && user_guid_json->valuestring) { snprintf(auc_info->auc_server, MAX_AUC_SERVER_LEN, "%s", auc_server_json->valuestring); snprintf(auc_info->api_key, MAX_API_KEY_LEN, "%s", api_key_json->valuestring); snprintf(auc_info->user_guid, MAX_USER_GUID, "%s", user_guid_json->valuestring); ret = 0; } } out: cJSON_Delete(response_json); return ret; } static void set_retry_time_interval(void) { if (auc_update_status.update_time_interval == AUC_UPDATE_TIME_INTERVAL_DEFAULT) { auc_update_status.update_time_interval = AUC_UPDATE_TIME_INTERVAL_FAST; } else if (auc_update_status.update_time_interval < AUC_UPDATE_TIME_INTERVAL_DEFAULT && (auc_update_status.update_time_interval * 2) < AUC_UPDATE_TIME_INTERVAL_DEFAULT) { auc_update_status.update_time_interval = auc_update_status.update_time_interval * 2; } else { auc_update_status.update_time_interval = AUC_UPDATE_TIME_INTERVAL_DEFAULT; } } static void update_libauc_url(struct auc_info *auc_info, bool use_tp_service) { char device_token[MAX_TOKEN_LEN] = {0}; char cloud_homecare_url[MAX_URL_LEN] = {0}; time_t time_stamp; memset(device_token, 0, MAX_TOKEN_LEN); memset(cloud_homecare_url, 0, MAX_URL_LEN); memset(auc_info->auc_server, 0, MAX_AUC_SERVER_LEN); memset(auc_info->api_key, 0, MAX_API_KEY_LEN); memset(auc_info->user_guid, 0, MAX_USER_GUID); //get time stamp time(&time_stamp); URL_CLASS_LOG(LOG_LEVEL_DEBUG, "start time_stamp:%d, updata_time_stamp_last:%d, update_time_interval:%d\n", time_stamp, auc_update_status.updata_time_stamp_last, auc_update_status.update_time_interval); if ((time_stamp - auc_update_status.updata_time_stamp_last) >= auc_update_status.update_time_interval) { if(use_tp_service) { if (0 == (get_token_url(device_token, cloud_homecare_url))) { memset(auc_info->tp_auc_server, 0, MAX_AUC_SERVER_LEN); snprintf(auc_info->tp_auc_server, MAX_AUC_SERVER_LEN, "%s", cloud_homecare_url); auc_update_status.updata_time_stamp_last = time_stamp; auc_update_status.update_time_interval = AUC_UPDATE_TIME_INTERVAL_DEFAULT; } else { auc_update_status.updata_time_stamp_last = time_stamp; set_retry_time_interval(); } } else { if (0 == (get_token_url(device_token, cloud_homecare_url))) { if (0 == get_auc_info_from_cloud(auc_info, device_token, cloud_homecare_url)) { auc_update_status.updata_time_stamp_last = time_stamp; auc_update_status.update_time_interval = AUC_UPDATE_TIME_INTERVAL_DEFAULT; } else { auc_update_status.updata_time_stamp_last = time_stamp; set_retry_time_interval(); } } else { auc_update_status.updata_time_stamp_last = time_stamp; set_retry_time_interval(); } } } else { } URL_CLASS_LOG(LOG_LEVEL_DEBUG, "end time_stamp:%d, updata_time_stamp_last:%d, update_time_interval:%d\n", time_stamp, auc_update_status.updata_time_stamp_last, auc_update_status.update_time_interval); URL_CLASS_LOG(LOG_LEVEL_DEBUG, "auc_server:%s, tp_auc_server:%s, api_key:%s, user_guid:%s\n", auc_info->auc_server, auc_info->tp_auc_server, auc_info->api_key, auc_info->user_guid); return; } #endif static void *mali_checkQuery(void *unused) { int ret = 0; int ret_mali = 0; int failed_num = 0; unsigned int wait_time = 0; parse_para para = {0, 0, 0}; srand((uint32_t)getpid()); while (1) { ret = parse_config(&para); if(0 == ret) { if ((web_protection)&&(para.web_protection)) { ret_mali = mali_url_hash_list_update(true); if (ret_mali < 0) { if (!failed_num) wait_time = 2 + (rand() % 254); else if (wait_time < MALI_MAX_WAIT_TIME) wait_time <<= 1; failed_num++; sleep(MIN(wait_time, MALI_MAX_WAIT_TIME)); } else { if (!access(CLOUD_HASH_FILE_UPDATE_FLAG, R_OK)) { pthread_mutex_lock(&mali_url_check_lock); mali_url_hash_list_update(false); pthread_mutex_unlock(&mali_url_check_lock); } failed_num = 0; } } else if ((web_protection)&&(!para.web_protection)) { pthread_mutex_lock(&mali_url_check_lock); mali_url_check_free(); web_protection = false; pthread_mutex_unlock(&mali_url_check_lock); failed_num = 0; } else if ((!web_protection)&&(para.web_protection)) { pthread_mutex_lock(&mali_url_check_lock); ret_mali = mali_url_check_init(false); if (0 == ret_mali) { web_protection = true; pthread_mutex_unlock(&mali_url_check_lock); failed_num = 0; } else { pthread_mutex_unlock(&mali_url_check_lock); if (!failed_num) wait_time = 2 + (rand() % 254); else if (wait_time < MALI_MAX_WAIT_TIME) wait_time <<= 1; failed_num++; sleep(MIN(wait_time, MALI_MAX_WAIT_TIME)); } } } sleep(5); } } static void *aucQuery(void *unused) { int ret = 0; struct url_info url_info; bool is_mali = false; #if SUPPORT_CLOUD_UPDATE_AUC_INFO struct auc_info auc_info; #endif unsigned int cat_id = 0; int tmp_id = 0; int index = 0; int index_cat = 0; int index_subcat = 0; unsigned char matched = 0; struct url_entry *tmp_list = NULL; struct url_entry *clean = NULL; unsigned char tmp_url_len; char tmp_url[MAX_URL_LEN] = {0}; printWar("create AUC query thread\n"); while (1) { pthread_mutex_lock(&url_lock); tmp_list = url_list.next; pthread_mutex_unlock(&url_lock); while (tmp_list) { pthread_mutex_lock(&url_lock); tmp_list->process_flag = URL_PROCESSING; pthread_mutex_unlock(&url_lock); url_info.url = tmp_list->url; url_info.info_len = 0; ret = pthread_mutex_trylock(&mali_url_check_lock); if (!ret) { is_mali = web_protection ? is_url_malicious(url_info.url) : false; if (is_mali) { cat_id = URL_CAT_SECURITY; #if DEBUG printWar("cat ret=%d url=%s id=%d info_id=%d\n", cat_id, url_info.url, tmp_list->id, tmp_list->info_id); #endif /* DEBUG */ pthread_mutex_lock(&url_lock); tmp_list->prev->next = tmp_list->next; if (tmp_list->next) { tmp_list->next->prev = tmp_list->prev; } tmp_list->query = 1; tmp_id = tmp_list->id; tmp_list->id = 0; tmp_list->info_id = 0; tmp_list->cat_map = 0; tmp_url_len = tmp_list->url_len; memset(tmp_url, 0, MAX_URL_LEN); strncpy_safe(tmp_url, tmp_list->url, tmp_list->url_len); clean = tmp_list; tmp_list = tmp_list->next; clean->next = NULL; clean->prev = NULL; pthread_mutex_unlock(&url_lock); pthread_mutex_unlock(&mali_url_check_lock); goto block_mali; } else { pthread_mutex_unlock(&mali_url_check_lock); } } if((!opts.use_tp_service && tmp_list->cat_map) || (opts.use_tp_service && (tmp_list->info_id != PCTL_WEB_URL_ID_ALL) && (tmp_list->cat_map & (~URL_CAT_SECURITY)))) { printWar("before send to auc\n"); #if SUPPORT_CLOUD_UPDATE_AUC_INFO update_libauc_url(&auc_info, opts.use_tp_service); ret = auc_query(&url_info, &auc_info, opts.use_tp_service); #else ret = auc_query(&url_info, opts.use_tp_service); #endif printWar("after send to auc\n"); for (index = 0; index < url_info.info_len; index++) { printWar("cat ret=%d url=%s id=%d name=%s\n", ret, url_info.url, url_info.info[index].id, url_info.info[index].name); } } //remove from url_list pthread_mutex_lock(&url_lock); tmp_list->prev->next = tmp_list->next; if (tmp_list->next) { tmp_list->next->prev = tmp_list->prev; } tmp_list->query = 1; tmp_id = tmp_list->id; tmp_list->id = 0; tmp_list->info_id = 0; tmp_list->cat_map = 0; tmp_url_len = tmp_list->url_len; memset(tmp_url, 0, MAX_URL_LEN); strncpy_safe(tmp_url, tmp_list->url, tmp_list->url_len); clean = tmp_list; tmp_list = tmp_list->next; clean->next = NULL; clean->prev = NULL; pthread_mutex_unlock(&url_lock); //send to kernel cat_id = 0; for (index = 0; index < url_info.info_len; index++) { for (index_cat = 0; mapping_table[index_cat][0] > 0; index_cat++) { matched = 0; for (index_subcat = 1; mapping_table[index_cat][index_subcat] >= 0; index_subcat++) { if (url_info.info[index].id == mapping_table[index_cat][index_subcat] || mapping_table[index_cat][index_subcat] == MAX_URL_CAT_DEFAULT) { cat_id |= 0x1 << (mapping_table[index_cat][0] - 1); matched = 1; break; } } if (matched) { break; } } } block_mali: printWar("send to kernel, url=%s cat_id=%x\n", url_info.url, cat_id); url_send(tmp_id, cat_id, tmp_url, tmp_url_len, URL_REQ_CAT); printWar("after send to kernel\n"); } cond_timedwait(500);//500ms } return 0; } static const char* short_opt = "s:h"; static struct option long_opt[] = { {"server" , required_argument , NULL, 'a'}, {"help" , no_argument , NULL, 'h'}, {0 , 0 , NULL, 0 } }; static void printHelp(char *progName); static int parseOpts(int argc, char* argv[], struct Options* opt); static void printHelpInstruction(char *progName); static unsigned int get_max_url_entry_len(void) { unsigned int *url_entry_len = NULL; int state; struct msghdr msg; struct iovec iov; struct nlmsghdr *nlh = NULL; nlh = (struct nlmsghdr *)malloc(NLMSG_SPACE(MAX_PAYLOAD)); if (!nlh) { printErr("malloc nlmsghdr error!\n"); return 0; } nlh->nlmsg_len = NLMSG_SPACE(MAX_PAYLOAD); nlh->nlmsg_pid = getpid(); nlh->nlmsg_flags = 0; nlh->nlmsg_type = URL_REQ_HELLO; iov.iov_base = (void *)nlh; iov.iov_len = NLMSG_SPACE(MAX_PAYLOAD); memset(&msg, 0, sizeof(msg)); msg.msg_name = NULL; msg.msg_namelen = 0; msg.msg_iov = &iov; msg.msg_iovlen = 1; if (sock_fd < 0) { printErr("error getting socket in recving: %s\n", strerror(errno)); return 0; } state = recvmsg(sock_fd, &msg, 0); if(state < 0) { printWar("state<1\n"); return 0; } //store to array url_entry_len = (unsigned int *)NLMSG_DATA(nlh); URL_CLASS_LOG(LOG_LEVEL_DEBUG, "msg from kernel, max_url_entry_len=%u\n", *url_entry_len); return *url_entry_len; } int main(int argc, char* argv[]) { int state; struct sockaddr_nl src_addr, dest_addr; struct nlmsghdr *nlh = NULL; struct iovec iov; struct msghdr msg; int retval; int state_smg = 0; pthread_t pid, pid_mali; int pret = 0; struct url_carrier *tmp = NULL; int index; pid_t fpid = 0; int ret = 0; int rc = parseOpts(argc, argv, &opts); unsigned int url_entry_len = 0; unsigned int malloc_url_entry_len = 0; if (!rc) { return 1; } fpid = fork(); if (fpid < 0) { printErr("error fork.../n"); exit(1); } else if (fpid > 0) { exit(0); } //setup exit handler atexit(on_exit_handler); signal(SIGTERM, signal_exit_handler); signal(SIGINT, signal_exit_handler); // Create a socket sock_fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_URL_CLASS); if (sock_fd < 0) { printErr("error getting socket: %s\n", strerror(errno)); return -1; } int send_buf = 4096; int recv_buf = 4096; if(setsockopt(sock_fd, SOL_SOCKET, SO_SNDBUF, &send_buf, sizeof(send_buf)) < 0) { printErr("setsockopt: %s", strerror(errno)); } if(setsockopt(sock_fd, SOL_SOCKET, SO_RCVBUF, &recv_buf, sizeof(recv_buf)) < 0) { printErr("setsockopt: %s", strerror(errno)); } // To prepare binding memset(&src_addr, 0, sizeof(src_addr)); src_addr.nl_family = AF_NETLINK; src_addr.nl_pid = getpid(); src_addr.nl_groups = 0; retval = bind(sock_fd, (struct sockaddr*)&src_addr, sizeof(src_addr)); if (retval < 0) { printErr("bind failed: %s\n", strerror(errno)); ret = -1; goto exit; } nlh = (struct nlmsghdr *)malloc(NLMSG_SPACE(MAX_PAYLOAD)); if (!nlh) { printErr("malloc nlmsghdr error!\n"); ret = -1; goto exit; } memset(&dest_addr,0,sizeof(dest_addr)); dest_addr.nl_family = AF_NETLINK; dest_addr.nl_pid = 0; dest_addr.nl_groups = 0; nlh->nlmsg_len = NLMSG_SPACE(MAX_PAYLOAD); nlh->nlmsg_pid = getpid(); nlh->nlmsg_flags = 0; nlh->nlmsg_type = URL_REQ_HELLO; iov.iov_base = (void *)nlh; iov.iov_len = NLMSG_SPACE(MAX_PAYLOAD); memset(&msg, 0, sizeof(msg)); msg.msg_name = (void *)&dest_addr; msg.msg_namelen = sizeof(dest_addr); msg.msg_iov = &iov; msg.msg_iovlen = 1; state_smg = sendmsg(sock_fd, &msg, 0);//Hello to kernel if (state_smg < 0) { printErr("get error sendmsg = %s\n",strerror(errno)); ret = -1; goto exit; } url_entry_len = get_max_url_entry_len(); if(url_entry_len == 0) { printErr("error getting url_entry_len\n"); ret = -1; goto exit; } malloc_url_entry_len = url_entry_len * 2.5;//double for backup, ex. id=1 recv and querying, but id=1 deleted by xt_pctl, then another id=1 recv, place it at url_entry_len+1 //if third/fourth... id=1 recv, they are insert in 2*url_entry_len~2.5*url_entry_len in order url_array = (struct url_entry *)malloc(malloc_url_entry_len * sizeof(struct url_entry)); if (!nlh) { printErr("malloc url_array error!\n"); ret = -1; goto exit; } //mali_checkQuery thread and AUC query thread if (pthread_mutex_init(&mali_url_check_lock, NULL) != 0 ) { printErr("mali_checkQuery pthread_mutex_init failed...%d\n", pret); ret = -1; goto exit; } if (pthread_mutex_init(&url_lock, NULL) != 0 || pthread_cond_init(&cond, NULL) != 0) { printErr("auc pthread_mutex_init failed...%d\n", pret); ret = -1; goto exit; } if ((pret = pthread_create(&pid_mali, NULL, mali_checkQuery, NULL)) != 0){ printErr("mali_checkQuery pthread_create failed...%d\n", pret); ret = -1; goto exit; } if ((pret = pthread_create(&pid, NULL, aucQuery, NULL)) != 0){ printErr("auc pthread_create failed...%d\n", pret); ret = -1; goto exit; } memset(url_array, 0, malloc_url_entry_len * sizeof(struct url_entry)); memset(nlh,0,NLMSG_SPACE(MAX_PAYLOAD));//ready to recv kernel info while (1) { state = recvmsg(sock_fd, &msg, 0); if(state < 0) { printWar("state<1\n"); } //store to array tmp = (struct url_carrier *)NLMSG_DATA(nlh); index = tmp->id; //URL_CLASS_LOG(LOG_LEVEL_DEBUG, "msg from kernel, url=%s len=%d id=%d", tmp->url, tmp->url_len, tmp->id); pthread_mutex_lock(&url_lock); if(index >= url_entry_len) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "Error: xt_pctl url entry size is larger than url_class(%u)!", url_entry_len); pthread_mutex_unlock(&url_lock); continue; } if (url_array[index].prev) {//this node is already in list if(url_array[index].url_len != tmp->url_len || strncmp(url_array[index].url, tmp->url, tmp->url_len) || url_array[index].info_id != tmp->info_id || url_array[index].cat_map != tmp->cat_map) { URL_CLASS_LOG(LOG_LEVEL_DEBUG, "Different node already in list, old_url=%s old_len=%d id=%d, old_process_flag=%d", url_array[index].url, url_array[index].url_len, url_array[index].id, url_array[index].process_flag); if(url_array[index].process_flag == URL_PROCESS_READY) {//Still can modify url_array[index].url_len = tmp->url_len; url_array[index].info_id = tmp->info_id; url_array[index].cat_map = tmp->cat_map; strncpy_safe(url_array[index].url, tmp->url, tmp->url_len); pthread_cond_signal(&cond); pthread_mutex_unlock(&url_lock); URL_CLASS_LOG(LOG_LEVEL_DEBUG, "Replace url=%s at url_array[%d]\n", tmp->url, index); continue; } else { index = url_entry_len + tmp->id; if (url_array[index].prev) { if(url_array[index].url_len == tmp->url_len && !strncmp(url_array[index].url, tmp->url, tmp->url_len) && url_array[index].info_id == tmp->info_id && url_array[index].cat_map == tmp->cat_map) {//Exactly same node is already in list URL_CLASS_LOG(LOG_LEVEL_DEBUG, "url=%s at url_array[%d] already exist!\n", tmp->url, index); pthread_mutex_unlock(&url_lock); continue; } if(url_array[index].process_flag == URL_PROCESS_READY) {//Still can modify url_array[index].url_len = tmp->url_len; url_array[index].info_id = tmp->info_id; url_array[index].cat_map = tmp->cat_map; strncpy_safe(url_array[index].url, tmp->url, tmp->url_len); pthread_cond_signal(&cond); pthread_mutex_unlock(&url_lock); URL_CLASS_LOG(LOG_LEVEL_WARN, "Replace url=%s at url_array[%d]\n", tmp->url, index); continue; } index = 2 * url_entry_len; while(index < malloc_url_entry_len && url_array[index].prev) { index++; } if(index >= malloc_url_entry_len) { //An auc query is ignored here! URL_CLASS_LOG(LOG_LEVEL_ERROR, "url_array is full! url=%s query is ignored!\n", tmp->url_len); pthread_mutex_unlock(&url_lock); continue; } } URL_CLASS_LOG(LOG_LEVEL_DEBUG, "Place url=%s at url_array[%d]\n", tmp->url, index); } } else {//Exactly same node is already in list pthread_mutex_unlock(&url_lock); continue; } } url_array[index].id = tmp->id; url_array[index].query = 0; url_array[index].process_flag = URL_PROCESS_READY; url_array[index].url_len = tmp->url_len; url_array[index].info_id = tmp->info_id; url_array[index].cat_map = tmp->cat_map; strncpy_safe(url_array[index].url, tmp->url, tmp->url_len); printWar("msg from kernel, url=%s len=%d id=%d\n", url_array[index].url, url_array[index].url_len, url_array[index].id); //add to list url_array[index].next = url_list.next; if (url_list.next) { url_list.next->prev = &url_array[index]; } url_array[index].prev = &url_list; url_list.next = &url_array[index]; pthread_cond_signal(&cond); pthread_mutex_unlock(&url_lock); } exit: if (nlh) { free(nlh); } if (url_array) { free(url_array); } on_exit_handler(); return ret; } static size_t _strlcpy(char *dst, const char *src, size_t dstsize) { size_t srclen = (size_t)strlen(src); if (dstsize > 0) { size_t len = (srclen >= dstsize) ? dstsize - 1 : srclen; memset(dst, 0, (len + 1)); memcpy(dst, src, len); } return srclen; } static struct uci_context *_uci_context_init(const char *config_path) { struct uci_context *uci_ctx = NULL; uci_ctx = uci_alloc_context(); if (uci_ctx) { uci_set_confdir(uci_ctx, config_path); } return uci_ctx; } static void _uci_context_free(struct uci_context *uci_ctx) { if (uci_ctx) { uci_free_context(uci_ctx); } } static int _uci_get_value(char * p_uci_str, char* p_value) { struct uci_context *uci_ctx = NULL; struct uci_element *e = NULL; struct uci_ptr p_uci; if (NULL == p_uci_str || NULL == p_value) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "p_uci_str or p_value is null"); goto error; } uci_ctx = _uci_context_init("/etc/config"); if (!uci_ctx) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "fail to init uci context:%s", p_uci_str); goto error; } if (UCI_OK != uci_lookup_ptr(uci_ctx, &p_uci, p_uci_str, true)) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "fail to get ptr %s ", p_uci_str); goto error; } e = p_uci.last; if (UCI_TYPE_OPTION != e->type) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "element type is not option:%d", e->type); goto error; } if (UCI_TYPE_STRING != p_uci.o->type) { URL_CLASS_LOG(LOG_LEVEL_ERROR, "option type is not string:%d", p_uci.o->type); goto error; } _strlcpy(p_value, p_uci.o->v.string, 64); URL_CLASS_LOG(LOG_LEVEL_DEBUG, "Success to get option value %s = %s ", p_uci_str,p_uci.o->v.string); _uci_context_free(uci_ctx); return 0; error: _uci_context_free(uci_ctx); return -1; } int parseOpts(int argc, char* argv[], struct Options* opt) { int c; int rc = 999; int use_tp_service = -1; while((c = getopt_long(argc, argv, short_opt, long_opt, NULL)) != -1) { switch(c) { case -1: /* no more arguments */ case 0: /* long options toggles */ break; case 's': if (optarg) { if(strcmp(optarg, "avira") == 0) { use_tp_service = 0; URL_CLASS_LOG(LOG_LEVEL_DEBUG, "parse Opts receive use avira service!"); } else if(strcmp(optarg, "tplink") == 0) { use_tp_service = 1; URL_CLASS_LOG(LOG_LEVEL_DEBUG, "parse Opts receive use tplink service!"); } else { URL_CLASS_LOG(LOG_LEVEL_ERROR, "parse Opts receive %s, not avaliable! Try %s -s tplink or %s -s avira", optarg, argv[0], argv[0]); } } rc = 's'; break; case 't': rc = 't'; break; case 'h': printHelp(argv[0]); rc = 0; break; case ':': case '?': printHelpInstruction(argv[0]); rc = 0; break; default: printf("%s : invalid option --\n", c); printHelpInstruction(argv[0]); rc = 0; break; } } if(use_tp_service == -1) { char _key[64] = "avira.info.status"; char _value[64] = {0}; if (_uci_get_value(_key, _value)) { URL_CLASS_LOG(LOG_LEVEL_DEBUG, "-s Parameter not found! Config avira.info.status not found! use default avira service!"); opt->use_tp_service = false; } else { if(strncmp(_value, "available", sizeof("available") - 1) == 0 || strncmp(_value, "libauc_available", sizeof("libauc_available") - 1) == 0)//avira服务可用 { URL_CLASS_LOG(LOG_LEVEL_DEBUG, "-s Parameter not found! Config avira.info.status=%s! use avira service!", _value); opt->use_tp_service = false; } else { URL_CLASS_LOG(LOG_LEVEL_DEBUG, "-s Parameter not found! Config avira.info.status=%s! use tplink service!", _value); opt->use_tp_service = true; } } } else { opt->use_tp_service = use_tp_service ? true : false; } return rc; } void printHelpInstruction(char *progName) { printf("Try: %s --help for more information\n", progName); } void printHelp(char *progName) { printf("Usage: %s\n\t -s <tplink/avira> Use the specified service provider", progName); } 上面的代码在线程中运行时如何查看状态
09-23
下面这个代码出现了上述报错,帮我修改一下:# -*- coding: utf-8 -*- """ Created on Mon Jul 21 14:13:11 2025 @author: srx20 """ import os import gc import numpy as np import pandas as pd import joblib import talib as ta from tqdm import tqdm import random from sklearn.cluster import MiniBatchKMeans from sklearn.preprocessing import StandardScaler from sklearn.model_selection import RandomizedSearchCV, GroupKFold from sklearn.feature_selection import SelectKBest, f_classif from sklearn.metrics import make_scorer, recall_score, classification_report import lightgbm as lgb import logging import psutil import warnings from scipy import sparse warnings.filterwarnings('ignore') # 设置志记录 logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', handlers=[ logging.FileHandler('stock_prediction_fixed.log'), logging.StreamHandler() ] ) logger = logging.getLogger(__name__) # ========== 配置类 ========== class StockConfig: def __init__(self): # 数据路径 self.SH_PATH = r"D:\股票量化数据库\股票csv数据\上证" self.SZ_PATH = r"D:\股票量化数据库\股票csv数据\深证" # 时间范围 self.START_DATE = "2012-1-1" self.END_DATE = "2025-7-31" self.TEST_START = "2024-5-1" self.TEST_END = "2025-7-31" # 聚类设置 self.CLUSTER_NUM = 8 self.CLUSTER_FEATURES = [ 'price_change', 'volatility', 'volume_change', 'MA5', 'MA20', 'RSI14', 'MACD_hist' ] # 预测特征 (初始列表,实际使用时会动态更新) self.PREDICT_FEATURES = [ 'open', 'high', 'low', 'close', 'volume', 'price_change', 'volatility', 'volume_change', 'MA5', 'MA20', 'RSI14', 'MACD_hist', 'cluster', 'MOM10', 'ATR14', 'VWAP', 'RSI_diff', 'price_vol_ratio', 'MACD_RSI', 'advance_decline', 'day_of_week', 'month' ] # 模型参数优化范围(内存优化版) self.PARAM_GRID = { 'boosting_type': ['gbdt'], # 减少选项 'num_leaves': [31, 63], # 减少选项 'max_depth': [-1, 7], # 减少选项 'learning_rate': [0.01, 0.05], 'n_estimators': [300, 500], # 减少选项 'min_child_samples': [50], # 固定值 'min_split_gain': [0.0, 0.1], 'reg_alpha': [0, 0.1], 'reg_lambda': [0, 0.1], 'feature_fraction': [0.7, 0.9], 'bagging_fraction': [0.7, 0.9], 'bagging_freq': [1] } # 目标条件 self.MIN_GAIN = 0.05 self.MIN_LOW_RATIO = 0.98 # 调试模式 self.DEBUG_MODE = False self.MAX_STOCKS = 50 if self.DEBUG_MODE else None self.SAMPLE_FRACTION = 0.3 if not self.DEBUG_MODE else 1.0 # 采样比例 # ========== 内存管理工具 (修复版) ========== def reduce_mem_usage(df): """优化DataFrame内存使用,只处理数值列""" start_mem = df.memory_usage().sum() / 1024**2 # 只处理数值列 numeric_cols = df.select_dtypes(include=['int', 'float', 'integer']).columns for col in numeric_cols: col_type = df[col].dtype if col_type != object: c_min = df[col].min() c_max = df[col].max() if str(col_type)[:3] == 'int': if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max: df[col] = df[col].astype(np.int8) elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max: df[col] = df[col].astype(np.int16) elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max: df[col] = df[col].astype(np.int32) elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max: df[col] = df[col].astype(np.int64) else: if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max: df[col] = df[col].astype(np.float16) elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max: df[col] = df[col].astype(np.float32) else: df[col] = df[col].astype(np.float64) end_mem = df.memory_usage().sum() / 1024**2 logger.info(f'内存优化: 从 {start_mem:.2f} MB 减少到 {end_mem:.2f} MB ({100*(start_mem-end_mem)/start_mem:.1f}%)') return df def print_memory_usage(): """打印当前内存使用情况""" process = psutil.Process(os.getpid()) mem = process.memory_info().rss / (1024 ** 2) logger.info(f"当前内存使用: {mem:.2f} MB") # ========== 数据加载 (修复版) ========== def load_stock_data(sh_path, sz_path, start_date, end_date, sample_fraction=1.0, debug_mode=False, max_stocks=None): """加载股票数据,并过滤期范围(修复随机抽样问题)""" stock_data = {} # 创建文件列表 all_files = [] for exchange, path in [('SH', sh_path), ('SZ', sz_path)]: if os.path.exists(path): csv_files = [f for f in os.listdir(path) if f.endswith('.csv')] for file in csv_files: all_files.append((exchange, path, file)) if not all_files: logger.warning("没有找到任何CSV文件") return stock_data # 随机抽样(修复一维问题) if sample_fraction < 1.0: sample_size = max(1, int(len(all_files) * sample_fraction)) # 使用random.sample代替np.random.choice all_files = random.sample(all_files, sample_size) logger.info(f"抽样 {len(all_files)} 只股票文件 (比例: {sample_fraction})") total_files = len(all_files) pbar = tqdm(total=total_files, desc='加载股票数据') loaded_count = 0 for exchange, path, file in all_files: if max_stocks is not None and loaded_count >= max_stocks: break if file.endswith('.csv'): stock_code = f"{exchange}_{file.split('.')[0]}" file_path = os.path.join(path, file) try: # 读取数据并验证列名 df = pd.read_csv(file_path) # 验证必要的列是否存在 required_cols = ['date', 'open', 'high', 'low', 'close', 'volume'] if not all(col in df.columns for col in required_cols): logger.warning(f"股票 {stock_code} 缺少必要列,跳过") pbar.update(1) continue # 转换期并过滤 df['date'] = pd.to_datetime(df['date']) df = df[(df['date'] >= start_date) & (df['date'] <= end_date)] if len(df) < 50: # 至少50个交易 logger.info(f"股票 {stock_code} 数据不足({len(df)}条),跳过") pbar.update(1) continue # 转换数据类型 for col in ['open', 'high', 'low', 'close']: df[col] = pd.to_numeric(df[col], errors='coerce').astype(np.float32) df['volume'] = pd.to_numeric(df['volume'], errors='coerce').astype(np.uint32) # 删除包含NaN的行 df = df.dropna(subset=required_cols) if len(df) > 0: stock_data[stock_code] = df loaded_count += 1 logger.debug(f"成功加载股票 {stock_code},数据条数: {len(df)}") else: logger.warning(f"股票 {stock_code} 过滤后无数据") except Exception as e: logger.error(f"加载股票 {stock_code} 失败: {str(e)}", exc_info=True) pbar.update(1) # 调试模式只处理少量股票 if debug_mode and loaded_count >= 10: logger.info("调试模式: 已加载10只股票,提前结束") break pbar.close() logger.info(f"成功加载 {len(stock_data)} 只股票数据") return stock_data # ========== 特征工程 (修复版) ========== class FeatureEngineer: def __init__(self, config): self.config = config def safe_fillna(self, series, default=0): """安全填充NaN值""" if isinstance(series, pd.Series): return series.fillna(default) elif isinstance(series, np.ndarray): return np.nan_to_num(series, nan=default) return series def transform(self, df): """添加技术指标特征(修复NumPy数组问题)""" try: # 创建临时副本用于TA-Lib计算 df_temp = df.copy() # 将价格列转换为float64以满足TA-Lib要求 for col in ['open', 'high', 'low', 'close']: df_temp[col] = df_temp[col].astype(np.float64) # 基础特征 df['price_change'] = df['close'].pct_change().fillna(0) df['volatility'] = df['close'].rolling(5).std().fillna(0) df['volume_change'] = df['volume'].pct_change().fillna(0) df['MA5'] = df['close'].rolling(5).mean().fillna(0) df['MA20'] = df['close'].rolling(20).mean().fillna(0) # 技术指标 - 修复NumPy数组问题 rsi = ta.RSI(df_temp['close'].values, timeperiod=14) df['RSI14'] = self.safe_fillna(rsi, 50) macd, macd_signal, macd_hist = ta.MACD( df_temp['close'].values, fastperiod=12, slowperiod=26, signalperiod=9 ) df['MACD_hist'] = self.safe_fillna(macd_hist, 0) # 新增特征 mom = ta.MOM(df_temp['close'].values, timeperiod=10) df['MOM10'] = self.safe_fillna(mom, 0) atr = ta.ATR( df_temp['high'].values, df_temp['low'].values, df_temp['close'].values, timeperiod=14 ) df['ATR14'] = self.safe_fillna(atr, 0) # 成交量加权平均价 vwap = (df['volume'] * (df['high'] + df['low'] + df['close']) / 3).cumsum() / df['volume'].cumsum() df['VWAP'] = self.safe_fillna(vwap, 0) # 相对强弱指数差值 df['RSI_diff'] = df['RSI14'] - df['RSI14'].rolling(5).mean().fillna(0) # 价格波动比率 df['price_vol_ratio'] = df['price_change'] / (df['volatility'].replace(0, 1e-8) + 1e-8) # 技术指标组合特征 df['MACD_RSI'] = df['MACD_hist'] * df['RSI14'] # 市场情绪指标 df['advance_decline'] = (df['close'] > df['open']).astype(int).rolling(5).sum().fillna(0) # 时间特征 df['day_of_week'] = df['date'].dt.dayofweek df['month'] = df['date'].dt.month # 处理无穷大和NaN df = df.replace([np.inf, -np.inf], np.nan) df = df.fillna(0) # 优化内存(只处理数值列) return reduce_mem_usage(df) except Exception as e: logger.error(f"特征工程失败: {str(e)}", exc_info=True) # 返回基本特征作为回退方案 df['price_change'] = df['close'].pct_change().fillna(0) df['volatility'] = df['close'].rolling(5).std().fillna(0) df['volume_change'] = df['volume'].pct_change().fillna(0) df['MA5'] = df['close'].rolling(5).mean().fillna(0) df['MA20'] = df['close'].rolling(20).mean().fillna(0) # 填充缺失的技术指标 for col in self.config.PREDICT_FEATURES: if col not in df.columns: df[col] = 0 return df # ========== 聚类模型 (添加保存/加载功能) ========== class StockCluster: def __init__(self, config): self.config = config self.scaler = StandardScaler() self.kmeans = MiniBatchKMeans( n_clusters=config.CLUSTER_NUM, random_state=42, batch_size=1000 ) self.cluster_map = {} # 股票代码到聚类ID的映射 self.model_file = "stock_cluster_model.pkl" # 模型保存路径 def save(self): """保存聚类模型到文件""" # 创建包含所有必要组件的字典 model_data = { 'kmeans': self.kmeans, 'scaler': self.scaler, 'cluster_map': self.cluster_map, 'config_cluster_num': self.config.CLUSTER_NUM } # 使用joblib保存模型 joblib.dump(model_data, self.model_file) logger.info(f"聚类模型已保存到: {self.model_file}") def load(self): """从文件加载聚类模型""" if os.path.exists(self.model_file): model_data = joblib.load(self.model_file) self.kmeans = model_data['kmeans'] self.scaler = model_data['scaler'] self.cluster_map = model_data['cluster_map'] logger.info(f"从 {self.model_file} 加载聚类模型") return True else: logger.warning("聚类模型文件不存在,需要重新训练") return False def fit(self, stock_data): """训练聚类模型""" logger.info("开始股票聚类分析...") cluster_features = [] # 提取每只股票的特征 for stock_code, df in tqdm(stock_data.items(), desc="提取聚类特征"): if len(df) < 50: # 至少50个交易 continue features = {} for feat in self.config.CLUSTER_FEATURES: if feat in df.columns: # 使用统计特征 features[f"{feat}_mean"] = df[feat].mean() features[f"{feat}_std"] = df[feat].std() else: # 特征缺失时填充0 features[f"{feat}_mean"] = 0 features[f"{feat}_std"] = 0 cluster_features.append(features) if not cluster_features: logger.warning("没有可用的聚类特征,使用默认聚类") # 创建默认聚类映射 self.cluster_map = {code: 0 for code in stock_data.keys()} return self # 创建特征DataFrame feature_df = pd.DataFrame(cluster_features) feature_df = reduce_mem_usage(feature_df) # 标准化特征 scaled_features = self.scaler.fit_transform(feature_df) # 聚类 self.kmeans.fit(scaled_features) clusters = self.kmeans.predict(scaled_features) feature_df['cluster'] = clusters # 创建股票到聚类的映射 stock_codes = list(stock_data.keys())[:len(clusters)] # 确保长度匹配 for i, stock_code in enumerate(stock_codes): self.cluster_map[stock_code] = clusters[i] logger.info("聚类分布统计:") logger.info(feature_df['cluster'].value_counts().to_string()) logger.info(f"股票聚类完成,共分为 {self.config.CLUSTER_NUM} 个类别") # 训练完成后自动保存模型 self.save() return self def transform(self, df, stock_code): """为数据添加聚类特征""" cluster_id = self.cluster_map.get(stock_code, -1) # 默认为-1表示未知聚类 df['cluster'] = cluster_id return df # ========== 目标创建 ========== class TargetCreator: def __init__(self, config): self.config = config def create_targets(self, df): """创建目标变量 - 增加T+2收盘价不低于T+1收盘价的条件""" # 计算次(T+1)收盘价相对于开盘价的涨幅 df['next_day_open_to_close_gain'] = df['close'].shift(-1) / df['open'].shift(-1) - 1 # 计算次(T+1)最低价与开盘价比例 df['next_day_low_ratio'] = df['low'].shift(-1) / df['open'].shift(-1) # 获取T+1和T+2的收盘价 df['next_day_close'] = df['close'].shift(-1) # T+1收盘价 df['next_next_day_close'] = df['close'].shift(-2) # T+2收盘价 # 创建复合目标: # 1. T+1收盘价比开盘价高5% # 2. T+1最低价≥开盘价98% # 3. T+2收盘价 ≥ T+1收盘价 df['target'] = 0 mask = ( (df['next_day_open_to_close_gain'] > self.config.MIN_GAIN) & (df['next_day_low_ratio'] >= self.config.MIN_LOW_RATIO) & (df['next_next_day_close'] >= df['next_day_close']) ) df.loc[mask, 'target'] = 1 # 删除最后两行(没有完整的T+1和T+2数据) df = df.iloc[:-2] # 检查目标分布 target_counts = df['target'].value_counts() logger.info(f"目标分布: 0={target_counts.get(0, 0)}, 1={target_counts.get(1, 0)}") # 添加调试信息 if self.config.DEBUG_MODE: sample_targets = df[['open', 'close', 'next_day_open_to_close_gain', 'next_day_close', 'next_next_day_close', 'target']].tail(5) logger.debug(f"目标创建示例:\n{sample_targets}") # 清理临时列 df.drop(columns=['next_day_close', 'next_next_day_close'], inplace=True, errors='ignore') return df # ========== 模型训练 (内存优化版) ========== class StockModelTrainer: def __init__(self, config): self.config = config self.model_name = "stock_prediction_model" self.feature_importance = None def prepare_dataset(self, stock_data, cluster_model, feature_engineer): """准备训练数据集(内存优化版)""" logger.info("准备训练数据集...") X_list = [] y_list = [] stock_group_list = [] # 用于分组交叉验证 target_creator = TargetCreator(self.config) # 使用生成器减少内存占用 for stock_code, df in tqdm(stock_data.items(), desc="处理股票数据"): try: # 特征工程 df = feature_engineer.transform(df.copy()) # 添加聚类特征 df = cluster_model.transform(df, stock_code) # 创建目标 df = target_creator.create_targets(df) # 只保留所需特征和目标 features = self.config.PREDICT_FEATURES if 'target' not in df.columns: logger.warning(f"股票 {stock_code} 缺少目标列,跳过") continue X = df[features] y = df['target'] # 确保没有NaN值 if X.isnull().any().any(): logger.warning(f"股票 {stock_code} 特征包含NaN值,跳过") continue # 使用稀疏矩阵存储(减少内存) sparse_X = sparse.csr_matrix(X.values.astype(np.float32)) X_list.append(sparse_X) y_list.append(y.values) stock_group_list.extend([stock_code] * len(X)) # 为每个样本添加股票代码作为组标识 # 定期清理内存 if len(X_list) % 100 == 0: gc.collect() print_memory_usage() except Exception as e: logger.error(f"处理股票 {stock_code} 失败: {str(e)}", exc_info=True) if not X_list: logger.error("没有可用的训练数据") return None, None, None # 合并所有数据 X_full = sparse.vstack(X_list) y_full = np.concatenate(y_list) groups = np.array(stock_group_list) logger.info(f"数据集准备完成,样本数: {X_full.shape[0]}") logger.info(f"目标分布: 0={sum(y_full==0)}, 1={sum(y_full==1)}") return X_full, y_full, groups def feature_selection(self, X, y): """执行特征选择(内存优化版)""" logger.info("执行特征选择...") # 使用基模型评估特征重要性 base_model = lgb.LGBMClassifier( n_estimators=100, random_state=42, n_jobs=-1 ) # 分批训练(减少内存占用) batch_size = 100000 for i in range(0, X.shape[0], batch_size): end_idx = min(i + batch_size, X.shape[0]) X_batch = X[i:end_idx].toarray() if sparse.issparse(X) else X[i:end_idx] y_batch = y[i:end_idx] if i == 0: base_model.fit(X_batch, y_batch) else: base_model.fit(X_batch, y_batch, init_model=base_model) # 获取特征重要性 importance = pd.Series(base_model.feature_importances_, index=self.config.PREDICT_FEATURES) importance = importance.sort_values(ascending=False) logger.info("特征重要性:\n" + importance.to_string()) # 选择前K个重要特征 k = min(15, len(self.config.PREDICT_FEATURES)) selected_features = importance.head(k).index.tolist() logger.info(f"选择前 {k} 个特征: {selected_features}") # 更新配置中的特征列表 self.config.PREDICT_FEATURES = selected_features # 转换特征矩阵 if sparse.issparse(X): # 对于稀疏矩阵,我们需要重新索引 feature_indices = [self.config.PREDICT_FEATURES.index(f) for f in selected_features] X_selected = X[:, feature_indices] else: X_selected = X[selected_features] return X_selected, selected_features def train_model(self, X, y, groups): """训练并优化模型(内存优化版)""" if X is None or len(y) == 0: logger.error("训练数据为空,无法训练模型") return None logger.info("开始训练模型...") # 1. 处理类别不平衡 pos_count = sum(y == 1) neg_count = sum(y == 0) scale_pos_weight = neg_count / pos_count if pos_count > 0 else 1.0 logger.info(f"类别不平衡处理: 正样本权重 = {scale_pos_weight:.2f}") # 2. 特征选择 X_selected, selected_features = self.feature_selection(X, y) # 3. 自定义评分函数 - 关注正类召回率 def positive_recall_score(y_true, y_pred): return recall_score(y_true, y_pred, pos_label=1) custom_scorer = make_scorer(positive_recall_score, greater_is_better=True) # 4. 使用分组时间序列交叉验证(减少折数) group_kfold = GroupKFold(n_splits=2) # 减少折数以节省内存 cv = list(group_kfold.split(X_selected, y, groups=groups)) # 5. 创建模型 model = lgb.LGBMClassifier( objective='binary', random_state=42, n_jobs=-1, scale_pos_weight=scale_pos_weight, verbose=-1 ) # 6. 参数搜索(减少迭代次数) search = RandomizedSearchCV( estimator=model, param_distributions=self.config.PARAM_GRID, n_iter=10, # 减少迭代次数以节省内存 scoring=custom_scorer, cv=cv, verbose=2, n_jobs=1, # 减少并行任务以节省内存 pre_dispatch='2*n_jobs', # 控制任务分发 random_state=42 ) logger.info("开始参数搜索...") # 分批处理数据(减少内存占用) if sparse.issparse(X_selected): X_dense = X_selected.toarray() # 转换为密集矩阵用于搜索 else: X_dense = X_selected search.fit(X_dense, y) # 7. 使用最佳参数训练最终模型 best_params = search.best_params_ logger.info(f"最佳参数: {best_params}") logger.info(f"最佳召回率: {search.best_score_}") final_model = lgb.LGBMClassifier( **best_params, objective='binary', random_state=42, n_jobs=-1, scale_pos_weight=scale_pos_weight ) # 使用早停策略训练最终模型 logger.info("训练最终模型...") final_model.fit( X_dense, y, eval_set=[(X_dense, y)], eval_metric='binary_logloss', callbacks=[ lgb.early_stopping(stopping_rounds=50, verbose=False), lgb.log_evaluation(period=100) ] ) # 保存特征重要性 self.feature_importance = pd.Series( final_model.feature_importances_, index=selected_features ).sort_values(ascending=False) # 8. 保存模型 model_path = f"{self.model_name}.pkl" joblib.dump((final_model, selected_features), model_path) logger.info(f"模型已保存到: {model_path}") return final_model def evaluate_model(self, model, X_test, y_test): """评估模型性能""" if model is None or len(X_test) == 0: logger.warning("无法评估模型,缺少数据或模型") return # 预测测试集 y_pred = model.predict(X_test) # 计算召回率 recall = recall_score(y_test, y_pred, pos_label=1) logger.info(f"测试集召回率: {recall:.4f}") # 计算满足条件的样本比例 condition_ratio = sum(y_test == 1) / len(y_test) logger.info(f"满足条件的样本比例: {condition_ratio:.4f}") # 详细分类报告 report = classification_report(y_test, y_pred) logger.info("分类报告:\n" + report) # 特征重要性 if self.feature_importance is not None: logger.info("特征重要性:\n" + self.feature_importance.to_string()) # ========== 主程序 ========== def main(): # 初始化配置 config = StockConfig() logger.info("===== 股票上涨预测程序 (修复版) =====") # 加载训练数据(添加抽样) logger.info(f"加载训练数据: {config.START_DATE} 至 {config.END_DATE}") train_data = load_stock_data( config.SH_PATH, config.SZ_PATH, config.START_DATE, config.END_DATE, sample_fraction=config.SAMPLE_FRACTION, debug_mode=config.DEBUG_MODE, max_stocks=config.MAX_STOCKS ) if not train_data: logger.error("错误: 没有加载到任何股票数据,请检查数据路径和格式") return # 特征工程 feature_engineer = FeatureEngineer(config) # 聚类分析 - 尝试加载现有模型,否则训练新模型 cluster_model = StockCluster(config) if not cluster_model.load(): # 尝试加载模型 try: cluster_model.fit(train_data) except Exception as e: logger.error(f"聚类分析失败: {str(e)}", exc_info=True) # 创建默认聚类映射 cluster_model.cluster_map = {code: 0 for code in train_data.keys()} logger.info("使用默认聚类(所有股票归为同一类)") cluster_model.save() # 保存默认聚类模型 # 准备训练数据 trainer = StockModelTrainer(config) try: X_train, y_train, groups = trainer.prepare_dataset( train_data, cluster_model, feature_engineer ) except Exception as e: logger.error(f"准备训练数据失败: {str(e)}", exc_info=True) return if X_train is None or len(y_train) == 0: logger.error("错误: 没有可用的训练数据") return # 训练模型 model = trainer.train_model(X_train, y_train, groups) if model is None: logger.error("模型训练失败") return # 加载测试数据(添加抽样) logger.info(f"\n加载测试数据: {config.TEST_START} 至 {config.TEST_END}") test_data = load_stock_data( config.SH_PATH, config.SZ_PATH, config.TEST_START, config.TEST_END, sample_fraction=config.SAMPLE_FRACTION, debug_mode=config.DEBUG_MODE, max_stocks=config.MAX_STOCKS ) if test_data: # 准备测试数据 X_test, y_test, _ = trainer.prepare_dataset( test_data, cluster_model, feature_engineer ) if X_test is not None and len(y_test) > 0: # 评估模型 if sparse.issparse(X_test): X_test = X_test.toarray() trainer.evaluate_model(model, X_test, y_test) else: logger.warning("测试数据准备失败,无法评估模型") else: logger.warning("没有测试数据可用") logger.info("===== 程序执行完成 =====") if __name__ == "__main__": main()
08-05
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值