网络通讯之完成端口

头文件：

 

#ifndef TOMCOMM
#define TOMCOMM
#include <winsock2.h>
#include "dm.h"
#include "sess3.h"
#include "cop.h"
#include "dmlst.h"

#define PORT_THREAD_COUNT 2              /*完成端口工作器起始线程数*/
#define DATA_BUF_SIZE DM4_FIX_MSG_LEN    /*每个连接接收数据缓冲区大小*/

#define RECV_POSTED    0
#define SEND_POSTED    1

#define PORT_THREAD_COUNT_ALLOC 1  /*工作器空闲增长阀值*/
#define PORT_THREAD_COUNT_FREE  5  /*工作器空闲释放阀值*/
#define PORT_THREAD_CHECK_TIME  30 /*检查时间间隔-30秒*/
#define PORT_THREAD_WAIT_TIME  INFINITE//30*1000 /*检查时间间隔-30秒*//*INFINITE*/

/**//*lt-完成端口数据缓冲区-保存接收的数据*/
typedef struct _PER_IO_DATA
...{
    OVERLAPPED Overlapped;
    WSABUF DataBuf;
    byte Buffer[DATA_BUF_SIZE];       /**//*接收数据缓冲区*/
    int OperationType;             /**//*投递类型-此处都为RECV_POSTED，只处理数据接受任务*/
    DWORD   SendBytes;          
    DWORD   RecvBytes;
} PER_IO_DATA,* LPPER_IO_DATA;

/**//*lt-单句柄数据信息结构-保存SOCKET和有用参数信息*/
typedef struct _PER_HANDLE_DATA PER_HANDLE_DATA;
typedef struct _PER_HANDLE_DATA * LPPER_HANDLE_DATA;
struct _PER_HANDLE_DATA
...{
    os_socket_t Socket;       /**//*连接socket*/
    //sess3_id_t sess_id;    /*会话ID*/
    sess3_t* sess;          /**//*会话，避免重复搜索，提高效率*/
    cop_t *cop;            /**//*任务*/
    ulint recvBytes;       /**//*已接收字节数*/
    ulint needBytes;       /**//*实际应该接收字节数*/
    dmbool bGetLen;        /**//*待接收消息长度是否已计算*/
    //time_t startTime;
    //time_t endTime;
    LPPER_IO_DATA lpIoData; /**//*数据接收缓冲区*/
    DM_LIST_NODE_T(PER_HANDLE_DATA) link_node;
    DM_LIST_NODE_T(PER_HANDLE_DATA) free_node;
};

/**//*数据缓冲区空闲队列*/
typedef struct port_free_lst_struct    port_free_lst_t;
struct port_free_lst_struct
...{
    os_mutex_t    mutex;
    DM_LIST_BASE_NODE_T(PER_HANDLE_DATA) free_lst;
};

/**//*数据缓冲区使用队列*/
typedef struct port_link_lst_struct    port_link_lst_t;
struct port_link_lst_struct
...{
    os_mutex_t    mutex;
    DM_LIST_BASE_NODE_T(PER_HANDLE_DATA) link_lst;
};

/**//*工作器线程结构体*/
typedef struct port_thread_struct port_thread_t;
struct port_thread_struct
...{
    dmbool bBusy;
    dmbool bAbort;
    dmbool bExit;
    os_thread_t thread;
    os_thread_id_t thread_id;
    DM_LIST_NODE_T(port_thread_t) link_node;
};

/**//*工作器线程队列*/
typedef struct port_thread_lst_struct    port_thread_lst_t;
struct port_thread_lst_struct
...{
    os_mutex_t    mutex;
    DM_LIST_BASE_NODE_T(port_thread_t) link_lst;
};
//extern port_link_lst_t* port_link_lst;

/**//*创建对列*/
void port_data_lst_create();

/**//*释放对列*/
void port_data_lst_free();

/**//*lt-PerHandleData结构体创建*/
void port_data_create(LPPER_HANDLE_DATA* lpHandle,LPPER_IO_DATA* lpIo);

/**//*lt-PerHandleData结构体释放*/
void port_data_free(LPPER_HANDLE_DATA lpHandle);

/**//*lt-PerHandleData结构体初始化*/
void port_data_init(LPPER_HANDLE_DATA lpHandleData,LPPER_IO_DATA lpIoData,os_socket_t sock,sess3_t* sess,cop_t* cop,dmbool bClear);

/**//*上锁*/
void port_data_enter(dmbool bfree);

/**//*解锁*/
void port_data_exit(dmbool bfree);

/**//*lt-完成端口通讯-错误退出*/
void comm_port_exit(char* error);

/**//*lt-完成端口通讯-监听线程*/
void completion_port_lsnr_thread(void);

/**//*lt-完成端口通讯-工作者线程*/
void completion_port_worker_thread(void* param);

/**//*lt-创建新会话*/
sess3_id_t completion_port_sess_create(cop_t* cop);

/**//*lt-继续接受字节*/
dmbool complete_port_recv_continue(LPPER_HANDLE_DATA lpHandleData,LPPER_IO_DATA lpIoData);

/**//*lt-消息处理*/
dmbool complete_port_process(LPPER_HANDLE_DATA lpHandleData,LPPER_IO_DATA lpIoData);

/**//*lt-断开客户端的消息接受*/
void port_data_close(LPPER_HANDLE_DATA* lpHandle,LPPER_IO_DATA* lpIo,sess3_t* sess,cop_t* cop);

#endif

 源文件：

#include "tomcomm.h"
#include "md5.h"
#include "svr.h"
#include "ini.h"
#include "ckpt.h"
#include "errno.h"
#include "msg.h"
#include "cipher.h"

/**//*需要添加ws2_32.lib*/
//完成端口全局变量
HANDLE CompletionPort;

extern dmbool volatile abort_loop;
extern os_socket_t global_svr_port;
extern ulint   global_comm_mode ;

port_free_lst_t* port_free_lst;
port_link_lst_t* port_link_lst;
port_thread_lst_t* port_thread_link_lst;

/**//*lt-PerHandleData结构体创建*/
void port_data_create(LPPER_HANDLE_DATA* lpHandle,LPPER_IO_DATA* lpIo)
...{    
    LPPER_HANDLE_DATA cport;

    port_data_enter(TRUE);

    /**//**<p>如果cop_free_lst->free_lst表空，则申请一块内存区返回</p>*/
    if (DM_LIST_GET_LEN(port_free_lst->free_lst) == 0)
    ...{        
        cport = mem_malloc(sizeof(PER_HANDLE_DATA) + sizeof(PER_IO_DATA));        
        dm_assert(cport != NULL);        
        cport->lpIoData =(byte*)cport + sizeof(PER_HANDLE_DATA);        
    }
    else
    ...{
        /**//**<p>否则移除该表的首结点，并返回首结点的地址</p>*/
        cport = DM_LIST_GET_FIRST(port_free_lst->free_lst);
        DM_LIST_REMOVE(free_node, port_free_lst->free_lst, cport);        
    }

    port_data_exit(TRUE);

    /**//*返回节点指针*/
    *lpHandle = cport;
    *lpIo     = cport->lpIoData;
    (*lpHandle)->cop == NULL;
    //(*lpHandle)->sess_id = NULL;
    (*lpHandle)->sess = NULL;
    (*lpHandle)->Socket = NULL;
    
    (*lpIo)->DataBuf.buf = (*lpIo)->Buffer;

    /**//*加入使用节点队列*/
    port_data_enter(FALSE);
    DM_LIST_ADD_LAST(link_node, port_link_lst->link_lst, cport);
    port_data_exit(FALSE);        
}

/**//*lt-PerHandleData结构体释放*/
void port_data_free(LPPER_HANDLE_DATA lpHandle)
...{
    LPPER_IO_DATA lpIo = NULL;
    
    port_data_enter(FALSE);
    //查询对应complete_port_t节点 & 从使用队列中删除该节点        
    DM_LIST_REMOVE(link_node,port_link_lst->link_lst,lpHandle);
    port_data_exit(FALSE);

    //释放内存后，添加到空闲队列
    port_data_enter(TRUE);
    lpIo =lpHandle->lpIoData;        
    port_data_init(lpHandle,lpIo,NULL,NULL,NULL,TRUE);    

    /**//**<p>如果cop_free_lst->free_lst表中的结点小于100个，则将该COP节点插入表尾，以供重新利用</p>*/
    if (DM_LIST_GET_LEN(port_free_lst->free_lst) < 100)
    ...{
        DM_LIST_ADD_LAST(free_node, port_free_lst->free_lst, lpHandle);
    }
    /**//**<p>否则直接释放cop指针指向的内存区域</p>*/
    else
    ...{        
        mem_free(lpHandle);
        //os_free(cport);
    }

    port_data_exit(TRUE);
}

/**//*lt-PerHandleData结构体初始化*/
void port_data_init(LPPER_HANDLE_DATA lpHandleData,LPPER_IO_DATA lpIoData,os_socket_t sock,sess3_t* sess,cop_t* cop,dmbool bClear)
...{
    dm_assert(NULL != lpIoData);
    dm_assert(NULL != lpHandleData);    
    
    if(bClear)
    ...{
        if(NULL != lpHandleData->cop)
            cop_free(lpHandleData->cop);        
    }
    lpHandleData->Socket = sock;
    lpHandleData->cop = cop;
    //lpHandleData->sess_id = sess_id;    
    lpHandleData->sess = sess;
    lpHandleData->needBytes = DM4_FIX_MSG_LEN; /**//*初始接收字节数-设置为 DM4_FIX_MSG_LEN*/
    lpHandleData->recvBytes = 0;               /**//*已接收字节数-设置为0*/    
    lpHandleData->bGetLen = FALSE;             /**//*待接收消息长度未计算*/

    /**//*起始时间设置*/
    //lpHandleData->startTime = dm_local_time();
    //lpHandleData->endTime = dm_local_time();
    
    memset(&(lpIoData->Overlapped),0,sizeof(OVERLAPPED));    
    memset(lpIoData->Buffer,0,DATA_BUF_SIZE);
    lpIoData->OperationType = RECV_POSTED;
    lpIoData->RecvBytes = 0;
    //lpIoData->SendBytes = 0;
    lpIoData->DataBuf.buf = lpIoData->Buffer;    /**//*接收缓冲区起始位置*/
    lpIoData->DataBuf.len = lpHandleData->needBytes;     /**//*接收缓冲区长度*/
}

/**//*上锁*/
void port_data_enter(dmbool bfree)
...{
    if(bfree)
        os_mutex_enter(&port_free_lst->mutex);
    else
        os_mutex_enter(&port_link_lst->mutex);    
}

/**//*解锁*/
void port_data_exit(dmbool bfree)
...{
    if(bfree)
        os_mutex_exit(&port_free_lst->mutex);
    else
        os_mutex_exit(&port_link_lst->mutex);
}

/**//*创建对列*/
void port_data_lst_create()
...{
    port_free_lst = (port_free_lst_t *)mem_malloc(sizeof(port_free_lst_t));/**//*mem_malloc*/
    port_link_lst = (port_link_lst_t *)mem_malloc(sizeof(port_link_lst_t));
    dm_assert(port_free_lst != NULL);
    dm_assert(port_link_lst != NULL);

    /**//**<p>创建空闲列表的互斥量</p>*/
    os_mutex_create(&(port_free_lst->mutex));
    os_mutex_create(&(port_link_lst->mutex));
    /**//**<p>设置互斥量的级别</p>*/
    os_mutex_set_level(&(port_free_lst->mutex), RM_LEVEL_NO_ORDER - 51);
    os_mutex_set_level(&(port_link_lst->mutex), RM_LEVEL_NO_ORDER - 51);

    DM_LIST_INIT(port_free_lst->free_lst);
    DM_LIST_INIT(port_link_lst->link_lst);   

    /**//*创建工作器线程队列*/
    port_thread_link_lst = (port_thread_lst_t *)mem_malloc(sizeof(port_thread_lst_t));/**//*mem_malloc*/    
    dm_assert(port_thread_link_lst != NULL);    

    /**//**<p>创建空闲列表的互斥量</p>*/
    os_mutex_create(&(port_thread_link_lst->mutex));    
    /**//**<p>设置互斥量的级别</p>*/
    os_mutex_set_level(&(port_thread_link_lst->mutex), RM_LEVEL_NO_ORDER - 51);    

    DM_LIST_INIT(port_thread_link_lst->link_lst);    
}

/**//*释放对列*/
void port_data_lst_free()
...{
    LPPER_HANDLE_DATA cport = NULL;
    port_thread_t* port_thread = NULL;

    os_mutex_free(&port_free_lst->mutex);
    os_mutex_free(&port_link_lst->mutex);

    while ((cport = DM_LIST_GET_LAST(port_free_lst->free_lst)) != NULL)
    ...{
        DM_LIST_REMOVE(free_node, port_free_lst->free_lst, cport);
        mem_free(cport);        
    }

    mem_free(port_free_lst);    

    while ((cport = DM_LIST_GET_LAST(port_link_lst->link_lst)) != NULL)
    ...{
        DM_LIST_REMOVE(link_node, port_link_lst->link_lst, cport);
        mem_free(cport);        
    }

    mem_free(port_link_lst);    

    /**//*释放工作器线程队列*/
    os_mutex_free(&port_thread_link_lst->mutex);
    while((port_thread = DM_LIST_GET_LAST(port_thread_link_lst->link_lst))!=NULL)
    ...{
        DM_LIST_REMOVE(link_node,port_thread_link_lst->link_lst,port_thread);
        CloseHandle(port_thread->thread);
        mem_free(port_thread);
    }

    mem_free(port_thread_link_lst);
}

/**//*创建新的工作器线程*/
dmbool port_thread_create()
...{
    dmbool bSucc;    
    port_thread_t* port_thread;    

    bSucc = TRUE;
    /**//*分配空间*/
    port_thread = mem_malloc(sizeof(port_thread_t));
    dm_assert(port_thread != NULL);

    port_thread->bAbort = FALSE;
    port_thread->bBusy = FALSE;
    port_thread->bExit = FALSE;
    port_thread->thread = os_thread_create((os_thread_fun_t)completion_port_worker_thread,(CompletionPort),&(port_thread->thread_id));
    if (port_thread->thread == INVALID_OS_THREAD_HANDLE)
    ...{
        fprintf(stderr, "fail to alloc comm thread");
        elog_report(SESS3_INVALID_ID, "fail to alloc comm thread");    
        mem_free(port_thread);
        bSucc = FALSE;
    }
    else
    ...{
        /**//*加入使用节点队列*/        
        DM_LIST_ADD_LAST(link_node, port_thread_link_lst->link_lst, port_thread);
        
    }

    printf("Completion Port Mode Create New Thread:%d/%d ",get_port_thread_free_count(),get_port_thread_count());

    return bSucc;
}
/**//*得到当前工作器线程数*/
ulint get_port_thread_count()
...{
    return port_thread_link_lst->link_lst.count;
}

/**//*得到当前空闲工作器线程数*/
ulint get_port_thread_free_count()
...{
    port_thread_t* port_thread;
    ulint count = 0;    
    port_thread = DM_LIST_GET_FIRST(port_thread_link_lst->link_lst);    
    while(port_thread!=NULL)
    ...{
        if(!port_thread->bBusy/**//* && !port_thread->bAbort*/)
            count++;
        port_thread = port_thread->link_node.next;
    }    

    return count;
}

/**//*得到当前需要释放的工作器线程数*/
ulint get_port_thread_close_count()
...{
    port_thread_t* port_thread;
    lint freeCount = 0;    
    lint abortCount = 0;
    ulint result = 0;
    port_thread = DM_LIST_GET_FIRST(port_thread_link_lst->link_lst);    
    while(port_thread!=NULL)
    ...{
        if(!port_thread->bBusy)
            freeCount++;
        if(port_thread->bAbort)
            abortCount++;

        port_thread = port_thread->link_node.next;
    }    

    result = freeCount - abortCount;
    if(result <= 0)
        return 0;
    else
        return result;    
}

/**//*得到当前工作器*/
port_thread_t* get_cur_port_thread()
...{
    port_thread_t* port_thread = NULL;
    os_thread_id_t thread_id = os_thread_self_id();

    os_mutex_enter(&port_thread_link_lst->mutex);
    port_thread = DM_LIST_GET_FIRST(port_thread_link_lst->link_lst);    
    while(port_thread!=NULL)
    ...{
        if(port_thread->thread_id == thread_id)
            break;
        port_thread = port_thread->link_node.next;
    }    

    os_mutex_exit(&port_thread_link_lst->mutex);    

    return port_thread;
}

/**//*关闭当前工作器线程*/
void port_thread_close(port_thread_t* port_thread)
...{
    dm_assert(port_thread != NULL);
    
    DM_LIST_REMOVE(link_node,port_thread_link_lst->link_lst,port_thread);
    CloseHandle(port_thread->thread);
    mem_free(port_thread);
    
    printf("Completion Port Mode Close Thread:%d/%d ",get_port_thread_free_count(),get_port_thread_count());
}

/**//*检查工作器线程队列*/
void port_thread_check()
...{
    port_thread_t* port_thread;    
    ulint count = 0;

    os_mutex_enter(&port_thread_link_lst->mutex);

    count = get_port_thread_free_count(FALSE);
    /**//*超过上限-开辟新的线程*/
    if(count < PORT_THREAD_COUNT_ALLOC)
    ...{
        port_thread_create();/**//*?是否会出现死锁*/
    }
    /**//*低于下限-关闭多余线程*/
    else
    ...{
        while((count = get_port_thread_close_count())>PORT_THREAD_COUNT_FREE)
        ...{
            port_thread = DM_LIST_GET_FIRST(port_thread_link_lst->link_lst);    
            while(port_thread!=NULL)
            ...{
                if(/**//*!port_thread->bBusy&&*/!port_thread->bAbort)
                ...{
                    port_thread->bAbort = TRUE;
                    break;
                }                
                port_thread = port_thread->link_node.next;
            }            
        }
    }
    /**//*关闭退出线程*/
    port_thread = DM_LIST_GET_FIRST(port_thread_link_lst->link_lst);    
    while(port_thread!=NULL)
    ...{
        if(port_thread->bExit)
        ...{
            port_thread_close(port_thread);
            break;
        }                
        port_thread = port_thread->link_node.next;
    }    

    os_mutex_exit(&port_thread_link_lst->mutex);    
}

/**//*lt-断开客户端的消息接受*/
void port_data_close(LPPER_HANDLE_DATA* lpHandle,LPPER_IO_DATA* lpIo,sess3_t* sess,cop_t* cop)
...{
    cop_free(cop);    
    sess3_enqueue_oob(sess);
    (*lpHandle)->cop = NULL;
    //(*lpHandle)->sess_id = NULL;
    (*lpHandle)->sess = NULL;
    (*lpHandle)->Socket = 0;
    port_data_free(*lpHandle);
    *lpHandle = NULL;
    *lpIo = NULL;
}

/**//*lt-完成端口通讯-错误退出*/
void completion_port_exit(char* error)
...{
    WSACleanup();
    printf("%s",error);
    getchar();
    exit(-1);
}

/**//*lt-完成端口通讯-监听线程*/
void completion_port_lsnr_thread(void)
...{       
    comm_port_t             client_port;
    svr_sockaddrin_t        lsnraddr;
    svr_sockaddrin_t        clntaddr;
    os_socket_t                sock,new_sock;
    vio_t*                    vio;
#ifdef WIN32
    ulint                   addrlen;
#else
    int                        addrlen;
#endif
   
    sess3_id_t sess_id = NULL;
    sess3_t*                sess = NULL;    
    
    fd_set readFDs,clientFDs;
    int one = 1;
    cop_t *cop;

    //1.定义参数    
    WSADATA wsd;
    //SYSTEM_INFO SystemInfo;    
    int remoteLen = 0;
    

    PER_HANDLE_DATA *lpHandleData = NULL;
    PER_IO_DATA     *lpIoData     = NULL;    
    DWORD Flags  = 0;
    DWORD RecvBytes=0;    
    
    int ret,i=0;

    /**//*超时设置*/
    /**//*struct timeval timeout;
    timeout.tv_sec = PORT_THREAD_CHECK_TIME;
    timeout.tv_usec = 0;*/
    
    

#ifdef WIN32
    //2.初始化
    port_data_lst_create();/**//*创建数据对列*/
    if(SOCKET_ERROR == WSAStartup(MAKEWORD(2,2),&wsd))
        completion_port_exit("WSAStartup Error! ");
#endif /* WIN32 */

    //3,创建一个I/O完成端口
    CompletionPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE,NULL,0,0);
    if(CompletionPort == NULL)
        completion_port_exit("CreateIoCompletionPort Error! ");

    //4.创建起始工作器线程
    //GetSystemInfo(&SystemInfo);
    //printf("Completion Port Mode:%d ",PORT_THREAD_COUNT);
    for(i=0;i<PORT_THREAD_COUNT/**//*SystemInfo.dwNumberOfProcessors*/;i++)
    ...{        
        //添加到工作器线程队列
        port_thread_create();

        /**//*ThreadHandle = CreateThread(NULL,0,completion_port_worker_thread,CompletionPort,0,NULL);
        if(NULL == ThreadHandle)
        completion_port_exit("CreateThread error! ");
        else
        CloseHandle(ThreadHandle);*/
    }

    /**//**<p>创建地址结构: lsnraddr</p>*/
    memset(&lsnraddr, 0, sizeof(svr_sockaddrin_t));
    lsnraddr.sin_family         = AF_INET;
    lsnraddr.sin_addr.s_addr    = INADDR_ANY;
    lsnraddr.sin_port           = htons(global_ini_info.port_num);

    
    //5.创建监听套接字
    global_svr_port = WSASocket(AF_INET,SOCK_STREAM,0,NULL,0,WSA_FLAG_OVERLAPPED);//socket(AF_INET,SOCK_STREAM,IPPROTO_TCP);
    if (global_svr_port == -1)
    ...{
        fprintf(stderr, "DM Listener can not create the socket:%d ", WSAGetLastError());
        elog_report(SESS3_INVALID_ID, "DM Listener can not create the socket");
        exit(-1);
    }
    

    /**//**<p>设置socket的选项</p>*/
#ifndef WIN32
    if(0 > setsockopt(global_svr_port, SOL_SOCKET, SO_REUSEADDR, (char *) &one, sizeof(one)))
    ...{
        fprintf(stderr, "DM Listener can not set the options of the socket ");
        elog_report(SESS3_INVALID_ID, "DM Listener can not set the options of the socket");
        exit(-1);
    }
#endif

    /**//**<p>将地址结构和套接字绑定</p>*/
#ifdef WIN32
    if (-1 == bind(global_svr_port, (svr_sockaddr_t *)&lsnraddr, sizeof(svr_sockaddrin_t)))
#else
    if (-1 == bind(global_svr_port, &lsnraddr, sizeof(svr_sockaddrin_t)))
#endif
    ...{
        fprintf(stderr, "DM Listener can not bind the address! ");
        elog_report(SESS3_INVALID_ID, "DM Listener can not bind the address!");
        exit(-1);
    }

    
    /**//**<p>监听用户的连接请求</p>*/
    ret = listen(global_svr_port, SOMAXCONN);
    if (ret == -1)
    ...{
        fprintf(stderr, "DM Listener's unable to listen on socket! ");
        elog_report(SESS3_INVALID_ID, "DM Listener's unable to listen on socket!");
        exit(-1);
    }

    /**//* now everything is ready */
    FD_ZERO(&clientFDs);
    FD_SET(global_svr_port,&clientFDs);

    /**//**<p>下面循环接收客户端的连接信息</p>*/
    while (!abort_loop)
    ...{   
        /**//**<p>死循环，直到系统将要关闭时，才跳出循环</p>*/
        if (SYS_SHUTDOWN == sys_get_status()) 
            break;
        
        readFDs=clientFDs;
        ret = select((int)global_svr_port + 1, &readFDs, 0, 0, 0/**//*&timeout*/);
        if (ret < 0)/**//*错误*/
        ...{
            if (socket_errno != SOCKET_EINTR)
            ...{
                if (!abort_loop)    /**//* purecov: inspected */
                    fprintf(stderr, "Error: Got error %d from select ",socket_errno); /**//* purecov: inspected */
                
            }

            port_thread_check();/**//*检查工作器线程*/
            continue;
        }
        if(ret == 0)/**//*超时*/
        ...{
            port_thread_check();/**//*检查工作器线程*/
            continue;
        }

        if(abort_loop)
            break;

        //port_thread_check();/*检查工作器线程*/
        sock = global_svr_port;
        comm_port_init(&client_port);
        addrlen = sizeof(svr_sockaddrin_t);

        /**//**<p>产生实际进行数据接收的socket</p>*/
#ifdef WIN32
        new_sock = accept(sock, (svr_sockaddr_t *)&clntaddr, &addrlen);
#else
        new_sock = accept(sock, (struct sockaddr *)&clntaddr, &addrlen);
#endif

        if (new_sock == SOCKET_INVALID_SOCKET)
        ...{
            if (socket_errno == ENFILE || socket_errno == EMFILE)
                os_thread_sleep(1);                // Give other threads some time
            
            continue;
        }

        ...{
            int dummyLen;
            struct sockaddr dummy;
            dummyLen = sizeof(struct sockaddr);
            if (getsockname(new_sock, &dummy, &dummyLen) < 0)
            ...{
                (void) closesocket(new_sock);
                continue;
            }
        }

        vio = &client_port.vio;
        vio_setsocket(vio, new_sock);
        
        if (!my_net_init(&client_port))
        ...{
            vio_close(vio);
            continue;
        }

#ifdef WIN32
        ...{
            /**//* Set default wait_timeout */
            ulint wait_timeout = NET_WAIT_TIMEOUT * 1000UL;
            setsockopt(vio->handle, SOL_SOCKET, SO_RCVTIMEO, (char*)&wait_timeout, sizeof(wait_timeout));
        }
#endif
        cop = cop_create(COP_SESS_ALLOC, SESS3_DUMY_ID, (byte*)&client_port, 1024);/**//*COP_SESS_ALLOC表示是第一次连接，处理后设置为其他类型*/
        dm_assert(cop != NULL);
        //tsk_cop_enqueue_first(cop); 

        /**//*直接创建会话,避免搜索阻塞*/
        sess_id = completion_port_sess_create(cop);
        sess = sess3_search(sess_id);
        
        /**//*已经到达sess上限*/
        if(NULL == sess_id)
            continue;

        //7.创建单句柄数据信息结构
        port_data_create(&lpHandleData,&lpIoData);
        dm_assert(NULL != lpHandleData);
        dm_assert(NULL != lpIoData);
        port_data_init(lpHandleData,lpIoData,new_sock,sess,cop,FALSE);        

        //8.关联完成端口
        CreateIoCompletionPort((HANDLE)(lpHandleData->Socket),CompletionPort,(DWORD)lpHandleData,0);

        //9.投递重叠I/O
        ret = WSARecv(lpHandleData->Socket,&(lpIoData->DataBuf),1,&RecvBytes,&Flags,&(lpIoData->Overlapped),NULL);
        
        if (ret == SOCKET_ERROR)
        ...{
            ret = WSAGetLastError();
            
            if (WSA_IO_PENDING != ret)
            ...{
                fprintf(stderr,"WSARecv error! ");
                /**//*断开客户端的消息接受*/            
                port_data_close(&lpHandleData,&lpIoData,sess,cop);            
            }
        }
    }

    port_data_lst_free();/**//*释放对列*/
#ifndef WIN32
    os_thread_exit(0);
#endif
}



/**//*lt-创建新会话*/
sess3_id_t completion_port_sess_create(cop_t* cop)
...{
    sess3_t*        sess = NULL;
    comm_port_t      port;
    vio_t            *vio;
    dmcode_t        code;

    memcpy(&port,  (comm_port_t*)cop->cop_req->data, 1024);

    sess = sess3_alloc(port);
    vio = &port.vio;
    if (sess == NULL)
    ...{
        fprintf(stderr, "Reached the max session limit. ");
        //global_conn_fails ++;
        vio_close(vio);
        return NULL;
    }
    else
    ...{
        //thrd_set_sessid(sess->id);当前线程是监听线程,不需要设置sess

        if(global_comm_mode == COMM_MODE_P2P || vio->type == port_ipc)
        ...{                    
#ifndef TOMCOMM
            /**//*不采用完成端口模式*/
            /**//**<p>创建一新会话线程，监听来自客户端的消息</p>*/
            sess->thread = os_thread_create((os_thread_fun_t)sess3_thread,
                (sess),
                NULL);

            if (sess->thread == INVALID_OS_THREAD_HANDLE)
            ...{
                fprintf(stderr, "fail to alloc comm thread");
                elog_report(SESS3_INVALID_ID, "fail to alloc comm thread");
                sess3_free(sess->id);
            }  
#endif
        }
        else
        ...{
            code = sess3_scan_add_sess(sess);
            if (!DM_SUCCESS(code))
            ...{
                fprintf(stderr, "fail to alloc sess");
                elog_report(SESS3_INVALID_ID, "fail to alloc sess");
                sess3_free(sess->id);
                return NULL;
            }
        }

        //thrd_reset_sessid();当前线程是监听线程,不需要设置sess

    }
    
    return sess->id;
}

/**//*lt-完成端口通讯-工作者线程*/
void completion_port_worker_thread(void* param)
...{
    HANDLE CompletionPortID = (HANDLE)param;
    DWORD BytesTransferred; /**//*当前接受字节数*/
    //LPOVERLAPPED Overlapped;
    LPPER_HANDLE_DATA lpHandleData = NULL;
    LPPER_IO_DATA lpIoData = NULL;    
    DWORD Flags = 0;
    DWORD RecvBytes = 0;
    
    //OVERLAPPED      *lpOverlapped = NULL;
    int ret;

    sess3_t*    sess = NULL;
    cop_t*      cop = NULL;    
    ulint    msg_len;
    byte*    new_buf;

    /**//*超时设置*/
    port_thread_t* port_thread = get_cur_port_thread();
    dm_assert(port_thread != NULL);

    while (!port_thread->bAbort)
    ...{
        port_thread->bBusy = FALSE;    

        //INFINITE可修改为毫秒数，否则会无休止等待
        ret = GetQueuedCompletionStatus(CompletionPortID,&BytesTransferred,(LPDWORD)&lpHandleData,(LPOVERLAPPED *)&lpIoData,PORT_THREAD_WAIT_TIME/**//*INFINITE*/);    

        port_thread->bBusy = TRUE;
        //超时返回
        if (ret == 0)
            continue;
        
        /**//*检查sess和cop*/
        //dm_assert(lpHandleData != NULL);
        sess = lpHandleData->sess;    
        dm_assert(sess != NULL);
        cop = lpHandleData->cop;
        dm_assert(cop != NULL);

         /**//*表示套接字已被通信对方关闭*/
        if (BytesTransferred == 0&&lpIoData->OperationType == RECV_POSTED)
        ...{
            /**//*断开客户端的消息接受*/            
            port_data_close(&lpHandleData,&lpIoData,sess,cop);
            continue;
        }

        switch (lpIoData->OperationType)
        ...{        
        case RECV_POSTED:

            /**//*设置当前通讯的起始时间*/
            /**//*if(0 == lpHandleData->recvBytes)
                lpHandleData->startTime = dm_local_time();*/
            
            /**//*拷贝当前接受内容至cop的数据缓冲区*/
            memcpy(cop->cop_req->data+lpHandleData->recvBytes,lpIoData->Buffer,BytesTransferred);
            lpHandleData->recvBytes += BytesTransferred;    
            //如果已接收值<需要接受值，则继续接受
            dm_assert(lpHandleData->recvBytes <= lpHandleData->needBytes);/**//*确保接收信息不会超出范围*/
            if(lpHandleData->recvBytes < lpHandleData->needBytes)
            ...{
                memcpy(cop->cop_req->data+lpHandleData->recvBytes-BytesTransferred,lpIoData->Buffer,BytesTransferred);
                complete_port_recv_continue(lpHandleData,lpIoData);/**//*如果失败，函数体内部实现断开通信操作*/                
                continue;
            }

            //已接收完毕，得到消息中设定的消息长度            
            if(FALSE == lpHandleData->bGetLen)
            ...{
                msg_get_len(lpIoData->Buffer,&msg_len);
                msg_len += msg_get_head_size();
                lpHandleData->bGetLen = TRUE;
                lpHandleData->needBytes = msg_len;
                
                /**//*已接收完毕*/
                if(msg_len <= lpHandleData->recvBytes)
                ...{
                    if(FALSE == complete_port_process(lpHandleData,lpIoData))/**//*处理错误-关闭会话*/
                    ...{
                        //printf("process error! ");
                        /**//*断开客户端的消息接受*/
                        port_data_close(&lpHandleData,&lpIoData,sess,cop);
                        continue;            
                    }
                }
                else
                ...{                    
                    if (msg_len > MSG_MAX_LEN * 2048)//长度错误->退出
                    ...{
                        /**//*断开客户端的消息接受*/            
                        port_data_close(&lpHandleData,&lpIoData,sess,cop);    
                        continue;
                    }
                    else
                    ...{
                        if(msg_len > MSG_MAX_LEN) /**//*超出cop缓冲区范围*/
                        ...{
                            /**//*开辟新的cop数据缓冲区*/
                            new_buf = os_malloc(msg_len);
                            dm_assert(NULL != new_buf);
                            /**//*拷贝原来cop数据缓冲区的内容至新的cop数据缓冲区*/
                            memcpy(new_buf,cop->cop_req->data,lpHandleData->recvBytes);
                            cop->cop_req->data = new_buf;
                        }
                         /**//*未接受完毕*/                        
                        complete_port_recv_continue(lpHandleData,lpIoData);
                        continue;
                    }                    
                }
            }
            else/**//*已接收完毕*/
            ...{                
                if(FALSE == complete_port_process(lpHandleData,lpIoData))/**//*处理错误-关闭会话*/
                ...{
                    //printf("process error! ");
                    /**//*断开客户端的消息接受*/
                    port_data_close(&lpHandleData,&lpIoData,sess,cop);
                    continue;
                }
            }
            

            //处理接收到的消息，新建一个任务            
            cop = cop_create(COP_RECV, sess->id, NULL, 0);
            dm_assert(cop != NULL);
            port_data_init(lpHandleData,lpIoData,lpHandleData->Socket,lpHandleData->sess,cop,FALSE);    
            //8.关联完成端口
            //CreateIoCompletionPort((HANDLE)(lpHandleData->Socket),CompletionPortID,(DWORD)lpHandleData,0);
            ret = WSARecv(lpHandleData->Socket,&(lpIoData->DataBuf),1,&RecvBytes,&Flags,&(lpIoData->Overlapped),NULL);

            if (ret == SOCKET_ERROR)
            ...{
                ret = WSAGetLastError();
                if (WSA_IO_PENDING != ret)
                ...{
                    printf("WSARecv error! ");
                    /**//*断开客户端的消息接受*/
                    port_data_close(&lpHandleData,&lpIoData,sess,cop);
                    continue;
                }
            }
            break;
        default:
            break;        
        } // switch        
    } // while

    port_thread->bExit = TRUE;
    //port_data_lst_free();/*释放对列*/
    return 0;
}



/**//*lt-继续接受字节*/
dmbool complete_port_recv_continue(LPPER_HANDLE_DATA lpHandleData,LPPER_IO_DATA lpIoData)
...{
    int ret;
    DWORD Flags = 0;
    DWORD RecvBytes=0;

    sess3_t* sess;
    cop_t* cop;

    /**//*为下一次重叠调用设置单I/O操作数据*/
    //memset(&(lpIoData->Overlapped),0,sizeof(OVERLAPPED));    
    //memset(lpIoData->Buffer,0,DATA_BUF_SIZE);
    lpIoData->OperationType = RECV_POSTED;            
    lpIoData->DataBuf.buf = lpIoData->Buffer;          /**//*接受缓冲区起始位置*/
    ret = lpHandleData->needBytes-lpHandleData->recvBytes;
    lpIoData->DataBuf.len = (ret>DATA_BUF_SIZE)?DATA_BUF_SIZE:ret;   /**//*还需接受字节数*/    
    dm_assert(lpIoData->DataBuf.len>0);
    
    ret = WSARecv(lpHandleData->Socket,&(lpIoData->DataBuf),1,&RecvBytes,&Flags,&(lpIoData->Overlapped),NULL);

    if (ret == SOCKET_ERROR)
    ...{
        ret = WSAGetLastError();
        if (WSA_IO_PENDING != ret)
        ...{
            fprintf(stderr,"WSARecv error ");
                
            sess = lpHandleData->sess;
            dm_assert(sess != NULL);
            cop = lpHandleData->cop;

            /**//*断开客户端的消息接受*/
            port_data_close(&lpHandleData,&lpIoData,sess,cop);
            return FALSE;
        }
    }

    return TRUE;
}


/**//*lt-消息处理*/
dmbool complete_port_process(LPPER_HANDLE_DATA lpHandleData,LPPER_IO_DATA lpIoData)
...{
    usint cmd;
    byte* msg;
    dmbool        client_check = FALSE;
    sess3_t* sess = lpHandleData->sess;
    cop_t* cop = lpHandleData->cop;
    byte** buf;        
    comm_port_t* port;
    ulint* len = NULL;    

#ifndef DM_DUP
    usint            encrypt_type;
    char            crc;
#endif

    dm_assert(sess != NULL);
    dm_assert(cop != NULL);
    buf = &(cop->cop_req->data);
    port = &(sess->port);
    len = &(lpHandleData->needBytes);    
    
    /**//**<p>crc校验</p>*/
#ifndef DM_DUP
    msg_msgbody_crc(&crc, (char*)*buf);

    if (crc != msg_get_crc((char*)*buf))
    ...{
        fprintf(stderr, "crc fail! ");
        return FALSE;
    }

    msg_get_encrypt_type(*buf, &encrypt_type);

#ifndef YACC_JAVA_EXPORTS
    if (encrypt_type != MSG_ENCRYPT_NONE &&    *len > MSG_HEAD_SIZE)
    ...{
        ulint    tmp_len = *len - MSG_HEAD_SIZE - (ENABLE_MESSGE_DIGEST ? MD5_DIGEST_LENGTH : 0) + MAX_BLOCK_SIZE;

        // malloc new buffer if necessary
        if (port->encrypt_buf == NULL)
        ...{
            port->encrypt_buf = os_malloc(tmp_len);
            port->encrypt_len = tmp_len;

            //            fprintf(stderr, "msg recv : malloc msg buffer %d ", tmp_len);
        }
        else if (port->encrypt_buf != NULL && port->encrypt_len < tmp_len)
        ...{
            os_free(port->encrypt_buf);

            port->encrypt_buf = os_malloc(tmp_len);
            port->encrypt_len = tmp_len;

            //            fprintf(stderr, "msg recv : remalloc msg buffer %d ", tmp_len);
        }
        dm_assert(port->encrypt_buf != NULL);    

        // decrypt msg
        if (!cipher_decrypt(port->cipher_type,
            (char*)(*buf) + MSG_HEAD_SIZE, 
            tmp_len - MAX_BLOCK_SIZE, 
            port->encrypt_buf, 
            &tmp_len, 
            port->session_key, 
            16,
            NULL))
        ...{
            fprintf(stderr, "decrypt fail! ");
            return FALSE;
        }

        // check digest if necessary
        if (ENABLE_MESSGE_DIGEST && 
            !cipher_check_msg_digest(MD5, port->encrypt_buf, tmp_len, *buf + (*len - MD5_DIGEST_LENGTH), MD5_DIGEST_LENGTH))
        ...{
            fprintf(stderr, "check digest fail! ");
            return FALSE;
        } 

        // set msg body and len
        dm_assert(tmp_len <= *len - MSG_HEAD_SIZE);
        memcpy(*buf + MSG_HEAD_SIZE, port->encrypt_buf, tmp_len);

        msg_set_len(*buf, tmp_len);
        msg_msgbody_crc(*buf + MSG_HEAD_CRC, *buf);
        *len = tmp_len + MSG_HEAD_SIZE;

        if (port->encrypt_buf != NULL && port->encrypt_len > 2 * MSG_MAX_LEN)
        ...{
            os_free(port->encrypt_buf);
            port->encrypt_buf = NULL;
            port->encrypt_len = 0;
        }
    }
#endif

    // modify encrypt type
    if (encrypt_type > port->encrypt_type)
    ...{
        port->encrypt_type = encrypt_type;
    }
    
#endif

    sess->sock_check_cnt = 0;/**//*添加 */

    /**//*第一次连接处理客户端版本信息*/
    if(cop->cop_req->cop_type == COP_SESS_ALLOC)
    ...{
        sess->last_msg_time = dm_local_time();
        sess->last_check_time = dm_local_time();
        
        sess->clnt_waiting = TRUE;
        cop->cop_req->cop_type = COP_RECVED;        
        cop->cnt_flag = 1;

        client_check = svr_process_startup(sess, cop);
        cop_free(cop);

        /**//*打印该任务消耗的时间*/
        /**//*lpHandleData->endTime = dm_local_time();
        printf("time:%d ",lpHandleData->endTime-lpHandleData->startTime);*/

        if (client_check)//登录成功
        ...{
            sess->port.read_timeout = NET_READ_TIMEOUT;            
            return TRUE;        
        }
        else
            return FALSE;        
    }
    else
    ...{
        msg = cop_get_msg(cop);
        msg_get_cmd(msg, &cmd);

        if (cmd == CMD_IS_SERVER_ACTIVE)
        ...{
            sess->last_check_time = dm_local_time();
        }
        else
        ...{
            sess->last_msg_time = dm_local_time();
        }

        sess->clnt_waiting = TRUE;
        cop->cop_req->cop_type = COP_RECVED;

        cop->cnt_flag = 1;

        /**//*打印该任务消耗的时间*/
        /**//*lpHandleData->endTime = dm_local_time();
        printf("time:%d ",lpHandleData->endTime-lpHandleData->startTime);*/

        tsk_cop_enqueue(cop);/**//*抛出任务*/            
    }
    
    return TRUE;
}