sphinx 采用c扩展xmlpipe2数据源

由于要采用sphinx 作为全文检索引擎，但是我的很多数据都存在硬盘上，如果这些数据都读入数据库那就太痛苦了，数据量很大，mysql数据库存储这些数据，只能是为了全文检索，那也过于浪费了，所以决定采用xmlpipe2数据源，由于c语言的io操作能力超强为了效率问题也就是用，从不会写的c语言。废话少说，代码如下

 

#include <stdio.h> #include <stdlib.h> #include <string.h> #include <unistd.h> #include <mysql.h> #include <iconv.h> #include "dictionary.h" #include "iniparser.h" //配置文件名称 #define INI_FILE_NAME "sanshi_xmlpipe.ini" //sql多条语句分隔符号 #define SQL_SIGN "|" //设置字段里名字与字段值的分隔符 #define ATTRIBUTE_SIGN ":" //设置多个字段间的分隔符 #define DIELD_SIGN "," //设置读取文件的长度 #define READFILE_MAX_LEN 1024 //设置路径+文件的最大长度 #define FILE_NAME_MAX_LEN 1024 typedef struct tag_sanshi_ini_config { char * db_name; char * db_user; char * db_pwd; char * db_host; char * sql_query_pre; char * sql_query; char * sql_query_post; char * sphinx_schema; char * sphinx_id; char * sphinx_file; char * sphinx_other_field; char * file_dir; }sanshi_ini_config; sanshi_ini_config conf={ "db:db_name", "db:db_user", "db:db_pwd", "db:db_host", "sql:sql_query_pre", "sql:sql_query", "sql:sql_query_post", "xml:schema", "xml:index_id", "xml:file_field", "xml:other_field", "file:base_dir" }; typedef struct tag_sanshi_field { char *field; int id; } sanshi_field; sanshi_field parser_field(char * field_str) { sanshi_field temp; char * buf = strstr( field_str, ATTRIBUTE_SIGN); temp.id = atoi(buf + strlen(ATTRIBUTE_SIGN)); buf[0]='/0'; temp.field = field_str; //temp.id = buf + strlen(ATTRIBUTE_SIGN); //buf = strstr( field_str, ATTRIBUTE_SIGN); //temp.id = atoi(buf + strlen(ATTRIBUTE_SIGN)); //printf("%s %s %d/n",field_str,temp.field,temp.id); /* char * p; char field_tmp[strlen(field_str)+1]; strcpy(field_tmp,field_str); //temp.field = strtok( field_str, ATTRIBUTE_SIGN); //field_str = strtok( NULL, ATTRIBUTE_SIGN); //temp.id = atoi(p); printf("%s /n",field_tmp); field_str =NULL; */ return temp; } void print_file_content(char * file_name) { FILE *fp; char line[READFILE_MAX_LEN]; fp=fopen(file_name,"r"); if(fp!=NULL) { while(fgets(line,READFILE_MAX_LEN,fp)!=NULL) { printf(line); } fclose(fp); } } void exec_mysql_query(MYSQL * mysql_con,char * sql_str) { char* token = strtok( sql_str, SQL_SIGN); while( token != NULL ) { int query_error_no=0; query_error_no = mysql_query(mysql_con,token); if(query_error_no !=0) { printf("ERROR sql=%s /nERROR NO=%d /nERROR msg= %s /n",token,query_error_no,mysql_error(mysql_con)); mysql_close(mysql_con); exit(0); } mysql_free_result(mysql_store_result(mysql_con)); //printf("exec sql : %s /n",token); token = strtok( NULL, SQL_SIGN); } } int main(int argc,char * argv[]) { dictionary * ini; MYSQL mysql,*mysql_con; MYSQL_RES *result; MYSQL_ROW row; int query_error_no,sphinx_id; sanshi_field file_field; ini = iniparser_load(INI_FILE_NAME); //get ini config mysql set conf.db_name = iniparser_getstring(ini,conf.db_name,"test"); conf.db_user = iniparser_getstring(ini,conf.db_user,"root"); conf.db_pwd = iniparser_getstring(ini,conf.db_pwd,""); conf.db_host = iniparser_getstring(ini,conf.db_host,"localhost"); conf.sql_query_pre = iniparser_getstring(ini,conf.sql_query_pre,NULL); conf.sql_query = iniparser_getstring(ini,conf.sql_query,NULL); conf.sql_query_post = iniparser_getstring(ini,conf.sql_query_post,NULL); conf.sphinx_schema = iniparser_getstring(ini,conf.sphinx_schema,NULL); sphinx_id = iniparser_getint(ini,conf.sphinx_id,0); conf.sphinx_file = iniparser_getstring(ini,conf.sphinx_file,NULL); file_field = parser_field(conf.sphinx_file); conf.sphinx_other_field = iniparser_getstring(ini,conf.sphinx_other_field,NULL); conf.file_dir = iniparser_getstring(ini,conf.file_dir,"./"); //printf("db_name=%s /t db_user=%s /t db_pwd=%s /t db_host=%s /n",conf.db_name,conf.db_user,conf.db_pwd,conf.db_host); //mysql connect mysql_init(&mysql); mysql_con =mysql_real_connect(&mysql,conf.db_host,conf.db_user,conf.db_pwd,conf.db_name,0,NULL,0); if(mysql_con == NULL) { printf("ERROR: connect mysql fail! plaese check ini file in set/n %s /n",mysql_error(&mysql)); exit(0); } //printf("mysql connect suc!/n"); //exec sql exec_mysql_query(mysql_con,conf.sql_query_pre); query_error_no = mysql_query(mysql_con,conf.sql_query); if(query_error_no !=0) { printf("ERROR sql=%s /nERROR NO=%d /nERROR msg= %s /n",conf.sql_query,query_error_no,mysql_error(mysql_con)); mysql_close(mysql_con); exit(0); } //printf("exec sql_query : %s /n",conf.sql_query); result = mysql_store_result(mysql_con); //echo xml header printf("<?xml version=/"1.0/" encodeing=/"utf-8/"?>/n<sphinx:docset>/n%s/n",conf.sphinx_schema); //printf("%s/n",conf.sphinx_other_field); while(row = mysql_fetch_row(result)) { printf("<sphinx:document id=/"%d/">/n",(row[sphinx_id]?row[sphinx_id]:0)); char * field_str; char field_tmp[strlen(conf.sphinx_other_field)+1]; char temp_file_name[FILE_NAME_MAX_LEN]; memcpy(field_tmp,conf.sphinx_other_field,strlen(conf.sphinx_other_field)+1); field_str = strtok( field_tmp, DIELD_SIGN); while( field_str != NULL ) { //printf("%s/n",field_str); sanshi_field other_field = parser_field(field_str); printf("

 

本人第一次写c，有些地方优化部够，还望各位指点

 

配置文件如下：

[db] db_host=127.0.0.1 db_name=test db_pwd=123456 db_user=root [sql] sql_query_pre=select * from log|select * from log sql_query = select * from log sql_query_post = [xml] schema=/ <sphinx:schema>/ <sphinx:field name="LogActionType"/>/ <sphinx:field name="LogDataType"/>/ <sphinx:attr name="LogTime" type="timestamp"/>/ <sphinx:attr name="LogIP" type="int" bits="16" default="1"/>/ </sphinx:schema> index_id=0 file_field=LogActionType:1 other_field=LogDataType:2,LogTime:6,LogIP:4 [file] base_dir=

 

备注：该程序的ini解析用到的是iniparser3.0b 的源代码，也就是依赖的2个h文件

#include "dictionary.h"

#include "iniparser.h"

       编译的命令为：

gcc -I /data/app/mysql/include/mysql/ -L /data/app/mysql/lib/mysql/ -l mysqlclient -g -o sanshi sanshi_xmlpipe.c dictionary.c iniparser.c

注意msql的库路径，已经文件名