linux c++ epoll 网络框架

1.基于多路复用模型
2.配合使用libaio的信号触发来搭建框架
3.良好的负载均衡设计，使得多用户之间的资源占用更为公平，减少长时间无响应的发生概率
4.超轻量简洁、防粘包、无特征（隐匿性）应用层协议头设计（仅12/24bytes）
5.基于应用层头协议，支持单通道消息穿插，无需切片，毫秒级响应（例如单通道传输文件时收发文字消息的场景）
    减少由多通道而带来的额外开销，使用管理更加简单、轻便
6.超轻量代码，整个Server端1000行左右，引入so即可使用，无其他第三方库依赖

ps：该框架不适用于B/S，个人认为.net core/nginx 这些已经做得比较完美了，整个框架以及以后的规划都会只针对于C/S端使用

开发环境：C++ 11  CentOS7.1 x64
稳定性、压力测试环境：CentOS7.1 x64

目前待开发：
1.更加完善的抗干扰、抗攻击逻辑
2.基于ECC（secp256k1）/AES_CBC网络加密传输，仅引入gmp大数库，无其他第三方依赖
    C端内置PUB_KEY，将序列号+机器码+时间戳通过ECC加密后发送至服务端
    服务端返回此处通信的PUB_KEY，客户端生成AES秘钥，使用此处的PUB_KEY将密钥通过ECC加密后发送至服务端
    此后通过AES秘钥进行加密数据传输

#include <iostream>
#include <condition_variable>
#include <unistd.h>
#include <fcntl.h>
#include <sys/prctl.h>
#include <sys/signal.h>
#include <atomic>

#include <netinet/in.h>
#include <netinet/tcp.h>
#include <arpa/inet.h>
#include <sys/epoll.h>
#include <sys/eventfd.h>
#include <sys/socket.h>
#include <sys/ioctl.h>
#include <libaio.h>

#include <algorithm>
#include <vector>
#include <unordered_set>
#include "network.h"
#include "protocol.hpp"

#define LISTEN_MAX 65535
#define LISTEN_FD_MAX 1
#define PRODUCE_MAX 65535
#define PRODUCE_FD_MAX 160000
#define THREAD_NUM 32
#define PROC_MAX 2048
#define PROC_FD_MAX 4
#define RECV_AIO_MAX 512
#define SEND_AIO_MAX 512
#define RECV_LEN 16*1024
#define SEND_LEN RECV_LEN
#define BODY_LEN SEND_LEN-HEADER_LEN

int SEND_SYS_LEN = 16 * 1024;
int RECV_SYS_LEN = 16 * 1024;

struct ProcThreadArgs
{
	int index;
};

ProcThreadArgs* ptaArr[THREAD_NUM];

int _epfdProduce;
std::unordered_map<int, Connection*> _connectionFdMap;
std::mutex _connectionFdMapLock;
int _evfd4Recv[THREAD_NUM];
int _evfd4Push[THREAD_NUM];
int _evfd4RecvOk[THREAD_NUM];
int _evfd4SendOk[THREAD_NUM];
int _epfd4Proc[THREAD_NUM];
std::unordered_map<int, EventType> _eventTypeMap;

std::deque<epoll_event*> _evDeque;
std::mutex _evDequeLock;
std::mutex _procLock;
std::condition_variable _cv4Proc;
int _finishThreadCount;

std::unordered_set<Connection*> _pushSet;
std::mutex _pushSetLock;
std::condition_variable _cv4Push;
std::mutex _procPushLock;
std::condition_variable _cv4PushProc;
int _finishPushThreadCount;

std::unordered_set<Connection*> _releaseSet;
std::mutex _releaseSetLock;


void FillHeader(char* buf, long id, long totalLen, long bodyLen)
{
	if (bodyLen == 0) {
		assert(false);
	}
	memcpy(buf, &id, sizeof(long));
	memcpy(buf + 8, &totalLen, sizeof(long));
	memcpy(buf + 16, &bodyLen, sizeof(long));
}

void ResolveHeader(Connection* connection, char* buf, int validLen, int& resolveStatus, int& curLen, long& id)
{
	if (validLen < HEADER_LEN)
	{
		//std::cerr << "ResolveHeader head not enough " << validLen << std::endl;
		resolveStatus = 0;
	}
	else
	{
		long totalLen;
		long bodyLen;
		memcpy(&id, buf, sizeof(long));
		memcpy(&totalLen, buf + 8, sizeof(long));
		memcpy(&bodyLen, buf + 16, sizeof(long));

		if (totalLen < 0 || bodyLen < 0 || bodyLen >BODY_LEN)
		{
			resolveStatus = 2;
		}
		else if (validLen < HEADER_LEN + bodyLen)
		{
			//std::cout << "ResolveHeader need recv more" << std::endl;
			resolveStatus = 0;
		}
		else
		{
			//std::cout << "ResolveHeader recv body finished" << std::endl;
			RecvEntity* re = 0;
			auto iter = connection->recvIdMap->find(id);
			if (iter == connection->recvIdMap->end())
			{
				re = new RecvEntity();
				re->totalLen = totalLen;
				re->isFirst = true;
				connection->recvIdMap->insert(std::pair<long, RecvEntity*>(id, re));
			}
			else
			{
				re = iter->second;
			}
			re->curLen += bodyLen;

			if (re->totalLen < re->curLen || re->totalLen < bodyLen)
			{
				resolveStatus = 2;
			}
			else
			{
				re->buf = buf;
				resolveStatus = 1;
				curLen += HEADER_LEN + bodyLen;
			}
		}
	}
}

SendEntity* GetSendEntity(Connection* connection, long sendLen, char* buf)
{
	SendEntity* se = new SendEntity();
	se->curLen = 0;
	se->len = sendLen;
	se->releaseType = NORMAL;
	connection->sendLock.lock();
	se->id = ++connection->sendId;
	if (buf == 0) {
		if (connection->sendStatus == 0 && sendLen < BODY_LEN)
		{
			connection->sendStatus = 2;
			connection->sendingSeId = se->id;
			se->buf = connection->sendBuf + HEADER_LEN;
		}
		else
		{
			se->buf = new char[sendLen];
		}
	}
	else {
		se->buf = buf;
	}
	connection->sendLock.unlock();
	return se;
}

void PushData(Connection* connection, SendEntity* se, PushPos pp)
{
	bool isNeedInsert = false;
	connection->sendLock.lock();
	{
		if (pp == PushPos::FRONT)
		{
			connection->sendDeque->push_front(se);
		}
		else if (pp == PushPos::BACK)
		{
			connection->sendDeque->push_back(se);
		}
		if (connection->sendStatus == 2) {
			connection->sendStatus = 3;
		}
		if (connection->sendStatus == 0 || connection->sendStatus == 3)
		{
			isNeedInsert = true;
		}
	}
	connection->sendLock.unlock();

	if (isNeedInsert)
	{
		_pushSetLock.lock();
		auto iter = _pushSet.find(connection);
		if (iter == _pushSet.end()) {
			++connection->refNum;
			_pushSet.insert(connection);
			_cv4Push.notify_one();
		}
		_pushSetLock.unlock();
	}
}

void ReleaseConnection(Connection* connection)
{
	connection->status = -1;

	_releaseSetLock.lock();
	_releaseSet.insert(connection);
	_releaseSetLock.unlock();
}

void ReleaseConnection2(Connection* connection)
{
	connection->sendLock.lock();
	while (connection->sendDeque->size() > 0)
	{
		SendEntity* se = connection->sendDeque->front();
		connection->sendDeque->pop_front();
		if (se->buf == connection->sendBuf + HEADER_LEN) {

		}
		else {
			if (se->releaseType == NOT_ALL) {

			}
			else {
				delete se->buf;
			}
		}
		if (se->releaseType == NOT_ALL) {
		}
		else {
			se->buf = 0;
			delete se;
		}
	}
	for (auto iter = connection->multiSendInfoMap->begin(); iter != connection->multiSendInfoMap->end(); ++iter) {
		SendEntity* se = iter->second->se;
		if (se->buf == connection->sendBuf + HEADER_LEN) {

		}
		else {
			delete se->buf;
		}
		delete se;
		delete iter->second;
	}
	connection->sendLock.unlock();

	connection->recvLock.lock();
	connection->recvIdMap->clear();
	connection->recvLock.unlock();

	int fd = connection->sockFd;

	free(connection->recvBuf);
	free(connection->sendBuf);
	delete connection->recvIdMap;
	delete connection->sendDeque;
	delete connection->multiSendInfoMap;
	delete connection;

	epoll_ctl(_epfdProduce, EPOLL_CTL_DEL, fd, 0);
	close(fd);
}

void userConnectCallback(Connection* connection)
{

}

void userDisconnectCallback(Connection* connection)
{

}

void defaultRecvCallback(io_context_t ioctx, struct iocb* obj, long res, long res2, int& resolveStatus)
{
	Connection* connection = (Connection*)obj;

	long id;
	int curLen = 0;
	/*
	 * 0 this buf has been handled
	 * 1 a header has been handled
	 * 2 illegal data received
	 * */
	resolveStatus = 0;
	while (1 == 1)
	{
		ResolveHeader(connection, connection->recvBuf + curLen, connection->recvCurLen + res - curLen, resolveStatus,
			curLen, id);
		if (resolveStatus == 0)
		{
			int surplus = connection->recvCurLen + res - curLen;
			//std::cout << "defaultRecvCallback need recv more,curLen:" << curLen << ",surplus:" << surplus << std::endl;
			if (surplus > 0)
			{
				memmove(connection->recvBuf, connection->recvBuf + curLen, surplus);
			}
			connection->recvCurLen = surplus;
			break;
		}
		else if (resolveStatus == 1)
		{
			userRecvCallback(connection, id);
		}
		else if (resolveStatus == 2)
		{
			std::cerr << "defaultRecvCallback recv error" << std::endl;
			break;
		}
	}
}

void defaultSendCallback(io_context_t ioctx, struct iocb* obj, long res, long res2, long finishId)
{
	Connection* connection = (Connection*)(obj - 1);
	if (finishId > 0) {
		userSendCallback(connection, finishId);
	}
}

void StartSend(Connection* connection, ProcThreadArgs* pta, iocb** sendIoSubmitArr, int& sendAioNum)
{
	if (connection->status >= 0)
	{
		if (connection->sendStatus == 4)
		{
			//std::cout << "start send " << pthread_self() << " " << connection->sendStatus << std::endl;
			int len = connection->ei.submitSendLen - connection->ei.sendCurLen;

			//memmove(connection->sendBuf, connection->sendBuf + connection->ei.sendCurLen, len);

			io_prep_pwrite(&connection->sendCb, connection->sockFd, connection->sendBuf, len, connection->ei.sendCurLen);
			io_set_eventfd(&connection->sendCb, _evfd4SendOk[pta->index]);
			sendIoSubmitArr[sendAioNum] = &connection->sendCb;
			++sendAioNum;
			++connection->refNum;
			connection->ei.sendCurLen = 0;
			connection->ei.submitSendLen = len;
			connection->sendStatus = 1;

			//std::cout << "TryConsumeSendqueue:1:submitSendLen" << len << std::endl;
		}
		else if (connection->sendStatus == 3) {
			//std::cout << "start send " << pthread_self() << " " << connection->sendStatus << std::endl;
			SendEntity* se = 0;
			for (auto iter = connection->sendDeque->begin(); iter != connection->sendDeque->end(); ++iter) {
				if ((*iter)->id == connection->sendingSeId) {
					se = *iter;
					break;
				}
			}
			if (se == 0) {
				std::cerr << "not find se!!" << std::endl;
			}
			else {
				long surplus = se->len - se->curLen;
				FillHeader(connection->sendBuf, se->id, se->len, surplus);
				io_prep_pwrite(&connection->sendCb, connection->sockFd, connection->sendBuf, HEADER_LEN + surplus,
					0);
				connection->ei.submitSendLen = HEADER_LEN + surplus;
				se->curLen += surplus;

				io_set_eventfd(&connection->sendCb, _evfd4SendOk[pta->index]);

				sendIoSubmitArr[sendAioNum] = &connection->sendCb;
				++sendAioNum;
				++connection->refNum;
				connection->sendStatus = 1;
			}
		}
		else if (connection->sendStatus == 0 && connection->sendDeque->size() > 0)
		{
			//std::cout << "start send " << pthread_self() << " " << connection->sendStatus << std::endl;
			char* bodyBuf = connection->sendBuf + HEADER_LEN;

			SendEntity* se = connection->sendDeque->front();

			//before send submit set
			connection->sendingSeId = se->id;
			long surplus = se->len - se->curLen;
			if (surplus <= BODY_LEN)
			{
				memcpy(bodyBuf, se->buf + se->curLen, surplus);
				FillHeader(connection->sendBuf, se->id, se->len, surplus);
				io_prep_pwrite(&connection->sendCb, connection->sockFd, connection->sendBuf, HEADER_LEN + surplus,
					0);
				connection->ei.submitSendLen = HEADER_LEN + surplus;
				se->curLen += surplus;
			}
			else
			{
				memcpy(bodyBuf, se->buf + se->curLen, BODY_LEN);
				FillHeader(connection->sendBuf, se->id, se->len, BODY_LEN);
				io_prep_pwrite(&connection->sendCb, connection->sockFd,
					connection->sendBuf,
					HEADER_LEN + BODY_LEN, 0);
				connection->ei.submitSendLen = HEADER_LEN + BODY_LEN;
				se->curLen += BODY_LEN;
			}
			//std::cout << "TryConsumeSendqueue:2:submitSendLen" << connection->submitSendLen << std::endl;
//                                                io_set_callback(&connection->sendCb, send_call_back);
			io_set_eventfd(&connection->sendCb, _evfd4SendOk[pta->index]);

			sendIoSubmitArr[sendAioNum] = &connection->sendCb;
			++sendAioNum;
			++connection->refNum;
			connection->sendStatus = 1;
		}
	}
}

void StartRecv(Connection* connection, ProcThreadArgs* pta, iocb** recvIoSubmitArr, int& recvAioNum)
{
	connection->recvStatus = 1;
	io_prep_pread(&connection->recvCb, connection->sockFd,
		connection->recvBuf + connection->recvCurLen,
		RECV_LEN - connection->recvCurLen, 0);
	io_set_eventfd(&connection->recvCb, _evfd4RecvOk[pta->index]);
	recvIoSubmitArr[recvAioNum] = &connection->recvCb;
	++connection->refNum;
	++recvAioNum;
}

void* CreateServThread(void* args)
{
	prctl(PR_SET_NAME, "ServThread");
	int sockSrv = socket(AF_INET, SOCK_STREAM, 0);

	int opt = 1;
	setsockopt(sockSrv, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(int));

	fcntl(sockSrv, F_SETFL, O_NONBLOCK | O_DIRECT);

	sockaddr_in addrSrv;
	addrSrv.sin_addr.s_addr = htonl(INADDR_ANY);
	addrSrv.sin_family = AF_INET;
	addrSrv.sin_port = htons(9001);

	::bind(sockSrv, (sockaddr*)&addrSrv, sizeof(sockaddr));

	int err = listen(sockSrv, LISTEN_MAX);
	if (err == -1)
	{
		return 0;
	}

	int epfdSrv = epoll_create(LISTEN_FD_MAX);
	epoll_event evSrv;
	evSrv.data.fd = sockSrv;
	evSrv.events = EPOLLIN | EPOLLOUT | EPOLLET;
	epoll_ctl(epfdSrv, EPOLL_CTL_ADD, sockSrv, &evSrv);


	epoll_event evConnection;

	epoll_event evTriggerArr[LISTEN_MAX] = {};
	int evNumSrv = 0;
	while (1 == 1)
	{
		evNumSrv = epoll_wait(epfdSrv, evTriggerArr, LISTEN_MAX, -1);
		if (evNumSrv <= 0)
		{
			continue;
		}

		while (1 == 1)
		{
			int sockConnection = accept(sockSrv, 0, 0);
			if (sockConnection <= 0)
			{
				break;
			}

			setsockopt(sockConnection, SOL_SOCKET, SO_SNDBUF, &SEND_SYS_LEN, sizeof(int));
			setsockopt(sockConnection, SOL_SOCKET, SO_RCVBUF, &RECV_SYS_LEN, sizeof(int));

			Connection* connection = new Connection();

			memset(connection, 0, sizeof(Connection));
			connection->sockFd = sockConnection;
			connection->recvIdMap = new std::unordered_map<long, RecvEntity*>();
			connection->multiSendInfoMap = new std::unordered_map<long, MultiSendInfo*>();
			connection->sendDeque = new std::deque<SendEntity*>();
			posix_memalign((void**)&connection->recvBuf, getpagesize(), RECV_LEN);
			posix_memalign((void**)&connection->sendBuf, getpagesize(), SEND_LEN);


			int flags = fcntl(sockConnection, F_GETFL);
			fcntl(sockConnection, F_SETFL, flags | O_NONBLOCK | O_DIRECT);

			evConnection.data.fd = sockConnection;
			evConnection.events = EPOLLIN | EPOLLOUT | EPOLLET | EPOLLRDHUP | EPOLLERR;
			epoll_ctl(_epfdProduce, EPOLL_CTL_ADD, sockConnection, &evConnection);

			_connectionFdMapLock.lock();
			auto iter = _connectionFdMap.find(sockConnection);
			if (iter == _connectionFdMap.end())
			{
				_connectionFdMap[sockConnection] = connection;
				_connectionFdMapLock.unlock();

				userConnectCallback(connection);
			}
			else
			{
				_connectionFdMapLock.unlock();

				//release....
				ReleaseConnection2(connection);
			}
		}
	}
}

void* CreateProduceThread(void* args)
{
	prctl(PR_SET_NAME, "ProduceThread");
	epoll_event evTriggerArr[PRODUCE_MAX] = {};
	int evNumProduce = 0;
	unsigned long long num = 1;

	while (1 == 1)
	{
		evNumProduce = epoll_wait(_epfdProduce, evTriggerArr, PRODUCE_MAX, -1);
		if (evNumProduce > 0)
		{
			for (int i = 0; i < evNumProduce; ++i)
			{
				//std::cout << "produce:" << evTriggerArr[i].data.fd << std::endl;
				_evDeque.push_back(&evTriggerArr[i]);
			}

			_finishThreadCount = 0;
			for (int i = 0; i < THREAD_NUM; ++i)
			{
				write(_evfd4Recv[i], &num, sizeof(unsigned long long));
			}
			{
				std::unique_lock<std::mutex> lock(_procLock);
				_cv4Proc.wait(lock, []() -> bool
					{
						return _finishThreadCount == THREAD_NUM;
					});
			}
		}
	}
}

void* CreateProcThread(void* args)
{
	srand(time(0));

	prctl(PR_SET_NAME, "ProcThread");

	ProcThreadArgs* pta = (ProcThreadArgs*)args;

	io_context_t recvIoctx;
	if (io_setup(RECV_AIO_MAX, &recvIoctx) == 0) {

	}
	else {
		std::cerr << "ProcThread io_setup err" << std::endl;
		abort();
	}
	io_event evRecvArr[RECV_AIO_MAX];
	iocb* recvIoSubmitArr[RECV_AIO_MAX];
	int recvAioNum;

	io_context_t sendIoctx;
	if (io_setup(SEND_AIO_MAX, &sendIoctx) == 0) {

	}
	else {
		std::cerr << "ProcThread io_setup err" << std::endl;
		abort();
	}
	io_event evSendArr[SEND_AIO_MAX];
	iocb* sendIoSubmitArr[SEND_AIO_MAX];
	int sendAioNum;

	timespec tms;
	tms.tv_sec = 0;
	tms.tv_nsec = 0;

	epoll_event evTriggerArr[PROC_MAX];
	int evNumProc = 0;
	unsigned long long num;
	while (1 == 1)
	{
		evNumProc = epoll_wait(_epfd4Proc[pta->index], evTriggerArr, PROC_MAX, -1);
		for (int i = 0; i < evNumProc; ++i)
		{
			read(evTriggerArr[i].data.fd, &num, sizeof(unsigned long long));

			EventType et = _eventTypeMap[evTriggerArr[i].data.fd];

			if (et == EventType::RECV)
			{
				if (evTriggerArr[i].events == EPOLLIN)
				{
					//std::cout<<"enter RECV"<<std::endl;
					recvAioNum = 0;
					sendAioNum = 0;
					while (1 == 1)
					{

						_evDequeLock.lock();
						if (_evDeque.size() <= 0)
						{
							_evDequeLock.unlock();
							break;
						}
						epoll_event* ev = _evDeque.front();
						_evDeque.pop_front();
						_evDequeLock.unlock();

						_connectionFdMapLock.lock();
						auto iter = _connectionFdMap.find(ev->data.fd);
						if (iter == _connectionFdMap.end())
						{
							_connectionFdMapLock.unlock();
							//unexpected fd
						}
						else
						{
							Connection* connection = iter->second;
							++connection->refNum;
							_connectionFdMapLock.unlock();

							if (connection->status >= 0)
							{

							}
							else
							{
								--connection->refNum;
								continue;
							}
							if ((ev->events & EPOLLRDHUP) == EPOLLRDHUP)
							{
								std::cout << ev->data.fd << ":peer closed!" << std::endl;
								ReleaseConnection(connection);
								userDisconnectCallback(connection);
							}
							else
							{
								if ((ev->events & EPOLLIN) == EPOLLIN)
								{
									//std::cout << "EPOLLIN!" << std::endl;

									connection->recvLock.lock();
									int length;
									ioctl(connection->sockFd, FIONREAD, &length);
									if (connection->recvStatus == 0 && length > 0)
									{
										StartRecv(connection, pta, recvIoSubmitArr, recvAioNum);
									}
									else
									{
										//                                        std::cout << connection->sockFd << " reading..." << std::endl;
																				//unexpected status
									}
									connection->recvLock.unlock();
								}
								if ((ev->events & EPOLLOUT) == EPOLLOUT)
								{
									//std::cout << "EPOLLOUT!" << std::endl;
									connection->sendLock.lock();
									StartSend(connection, pta, sendIoSubmitArr, sendAioNum);
									connection->sendLock.unlock();
								}
							}
							--connection->refNum;
						}
					}
					io_submit(recvIoctx, recvAioNum, recvIoSubmitArr);
					io_submit(sendIoctx, sendAioNum, sendIoSubmitArr);
				}
				{
					std::unique_lock<std::mutex> lock(_procLock);
					++_finishThreadCount;
					_cv4Proc.notify_all();
				}
			}
			else if (et == EventType::PUSH)
			{
				sendAioNum = 0;
				while (1 == 1)
				{
					_pushSetLock.lock();
					if (_pushSet.size() <= 0)
					{
						_pushSetLock.unlock();
						break;
					}
					auto iter = _pushSet.begin();
					Connection* connection = (Connection*)*iter;
					_pushSet.erase(iter);
					_pushSetLock.unlock();

					connection->sendLock.lock();
					StartSend(connection, pta, sendIoSubmitArr, sendAioNum);
					connection->sendLock.unlock();

					--connection->refNum;
				}
				io_submit(sendIoctx, sendAioNum, sendIoSubmitArr);
				{
					std::unique_lock<std::mutex> lock(_procPushLock);
					++_finishPushThreadCount;
					_cv4PushProc.notify_all();
				}
			}
			else if (et == EventType::RECV_OK)
			{
				int gevNum = io_getevents(recvIoctx, 1, RECV_AIO_MAX, evRecvArr, &tms);
				//std::cout << "RECV_OK :" << pthread_self() << std::endl;
				recvAioNum = 0;
				sendAioNum = 0;
				for (int i = 0; i < gevNum; ++i)
				{
					Connection* connection = (Connection*)evRecvArr[i].obj;

					if (connection->status >= 0)
					{

					}
					else
					{
						connection->recvLock.lock();
						connection->recvStatus = 0;
						connection->recvLock.unlock();
						--connection->refNum;
						continue;
					}

					if ((long)evRecvArr[i].res <= 0)
					{
						std::cout << connection->sockFd << ":peer closed!!!RECV_OK:" << (long)evRecvArr[i].res
							<< std::endl;
						ReleaseConnection(connection);
						userDisconnectCallback(connection);
						connection->recvLock.lock();
						connection->recvStatus = 0;
						connection->recvLock.unlock();
						--connection->refNum;
						continue;
					}

					int resolveStatus = 0;
					defaultRecvCallback(recvIoctx, evRecvArr[i].obj, evRecvArr[i].res,
						evRecvArr[i].res2, resolveStatus);

					if (resolveStatus == 2)
					{
						std::cout << connection->sockFd << ":peer closed!!!" << std::endl;
						ReleaseConnection(connection);
						userDisconnectCallback(connection);
					}

					{
						connection->recvLock.lock();
						if (resolveStatus == 2)
						{
							connection->recvStatus = 0;
						}
						else
						{
							int length;
							ioctl(connection->sockFd, FIONREAD, &length);
							if (length > 0)
							{
								StartRecv(connection, pta, recvIoSubmitArr, recvAioNum);
							}
							else
							{
								connection->recvStatus = 0;
								//std::cout << connection->sockFd << "recv end,wait next epollin !!" << std::endl;
							}
						}
						connection->recvLock.unlock();
					}

					{
						connection->sendLock.lock();
						StartSend(connection, pta, sendIoSubmitArr, sendAioNum);
						connection->sendLock.unlock();
					}

					--connection->refNum;

				}
				io_submit(sendIoctx, sendAioNum, sendIoSubmitArr);
				io_submit(recvIoctx, recvAioNum, recvIoSubmitArr);
			}
			else if (et == EventType::SEND_OK)
			{
				int gevNum = io_getevents(sendIoctx, 1, SEND_AIO_MAX, evSendArr, &tms);
				//std::cout << "SEND OK :" << pthread_self() << std::endl;
				sendAioNum = 0;
				for (int i = 0; i < gevNum; ++i)
				{
					Connection* connection = (Connection*)((char*)evSendArr[i].obj - sizeof(iocb));

					if (connection->status >= 0)
					{

					}
					else
					{
						connection->sendLock.lock();
						connection->sendStatus = 0;
						connection->sendLock.unlock();
						--connection->refNum;
						continue;
					}

					if ((long)evSendArr[i].res < 0)
					{
						connection->sendLock.lock();
						connection->sendStatus = 4;
						connection->sendLock.unlock();
						--connection->refNum;
						continue;
					}


					if (evSendArr[i].res == 0)
					{
						std::cout << connection->sockFd << ":peer closed!!!SEND_OK:" << (long)evSendArr[i].res
							<< std::endl;
						ReleaseConnection(connection);
						userDisconnectCallback(connection);
						connection->sendLock.lock();
						connection->sendStatus = 0;
						connection->sendLock.unlock();
						--connection->refNum;
						continue;
					}

					long finishId = 0;
					connection->sendLock.lock();
					if (evSendArr[i].res < connection->ei.submitSendLen)
					{
						connection->ei.sendCurLen = evSendArr[i].res;
						connection->sendStatus = 4;
					}
					else
					{
						if (connection->sendingSeId > 0)
						{
							for (auto iter = connection->sendDeque->begin();
								iter != connection->sendDeque->end(); ++iter)
							{
								if ((*iter)->id == connection->sendingSeId)
								{
									SendEntity* se = *iter;

									if (se->curLen < se->len)
									{

									}
									else
									{
										finishId = se->id;
										connection->sendDeque->erase(iter);
										connection->sendingSeId = 0;
										if (se->buf == connection->sendBuf + HEADER_LEN) {

										}
										else {
											if (se->releaseType == NOT_ALL) {

											}
											else {
												delete se->buf;
											}
										}
										if (se->releaseType == NOT_ALL) {
											se->id = 0;
											se->curLen = 0;
										}
										else {
											se->buf = 0;
											delete se;
										}
									}
									break;
								}
							}
						}
						connection->sendStatus = 0;
					}
					connection->sendLock.unlock();

					defaultSendCallback(sendIoctx, evSendArr[i].obj, evSendArr[i].res,
						evSendArr[i].res2, finishId);

					connection->sendLock.lock();
					StartSend(connection, pta, sendIoSubmitArr, sendAioNum);
					connection->sendLock.unlock();

					--connection->refNum;
				}
				io_submit(sendIoctx, sendAioNum, sendIoSubmitArr);
			}
		}
	}
	delete pta;
}

void* CreatePushThread(void* args)
{
	prctl(PR_SET_NAME, "PushThread");
	unsigned long long num = 1;
	while (1 == 1)
	{
		{
			std::unique_lock<std::mutex> lock(_pushSetLock);
			_cv4Push.wait(lock, []()->bool {
				return _pushSet.size() > 0;
				});
		}

		if (_pushSet.size() > 0)
		{
			_finishPushThreadCount = 0;
			for (int i = 0; i < THREAD_NUM; ++i)
			{
				write(_evfd4Push[i], &num, sizeof(unsigned long long));
			}
			{
				std::unique_lock<std::mutex> lock(_procPushLock);
				_cv4PushProc.wait(lock, []() -> bool
					{
						return _finishPushThreadCount == THREAD_NUM;
					});
			}
		}
	}
}

void* CreateReleaseThread(void* args)
{
	prctl(PR_SET_NAME, "ReleaseThread");
	while (1 == 1)
	{
		_releaseSetLock.lock();
		for (auto iter = _releaseSet.begin(); iter != _releaseSet.end();)
		{

			Connection* connection = *iter;

			_connectionFdMapLock.lock();
			if (connection->refNum == 0)
			{
				_connectionFdMap.erase(connection->sockFd);
				_connectionFdMapLock.unlock();
				ReleaseConnection2(connection);
				iter = _releaseSet.erase(iter);
			}
			else
			{
				_connectionFdMapLock.unlock();
				++iter;
			}
		}
		_releaseSetLock.unlock();

		//        std::cout << "_connectionFdMap size:" << _connectionFdMap.size() << std::endl;
		//        std::cout << "_releaseSet size:" << _releaseSet.size() << std::endl;
		usleep(1000);
	}
}

int StartConnect(const char* ip, int port, Connection*& connection)
{
	int sockClient = socket(AF_INET, SOCK_STREAM, 0);

	int opt = 1;
	setsockopt(sockClient, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(int));
	setsockopt(sockClient, SOL_SOCKET, SO_SNDBUF, &SEND_SYS_LEN, sizeof(int));

	sockaddr_in addrClient;
	addrClient.sin_addr.s_addr = inet_addr(ip);
	addrClient.sin_family = AF_INET;
	addrClient.sin_port = htons(port);
	int err = connect(sockClient, (sockaddr*)&addrClient, sizeof(sockaddr));
	if (err < 0)
	{
		return err;
	}

	int flags = fcntl(sockClient, F_GETFL);
	fcntl(sockClient, F_SETFL, flags | O_NONBLOCK);

	connection = new Connection();
	memset(connection, 0, sizeof(Connection));
	connection->sockFd = sockClient;
	connection->recvIdMap = new std::unordered_map<long, RecvEntity*>();
	connection->sendDeque = new std::deque<SendEntity*>();
	posix_memalign((void**)&connection->recvBuf, getpagesize(), RECV_LEN);
	posix_memalign((void**)&connection->sendBuf, getpagesize(), SEND_LEN);

	epoll_event evConnection;
	evConnection.data.fd = sockClient;
	evConnection.events = EPOLLIN | EPOLLOUT | EPOLLET | EPOLLRDHUP | EPOLLERR;
	epoll_ctl(_epfdProduce, EPOLL_CTL_ADD, sockClient, &evConnection);

	_connectionFdMapLock.lock();
	auto iter = _connectionFdMap.find(sockClient);
	if (iter == _connectionFdMap.end())
	{
		_connectionFdMap[sockClient] = connection;
		_connectionFdMapLock.unlock();
	}
	else
	{
		_connectionFdMapLock.unlock();
		//release....
		ReleaseConnection2(connection);
	}
	return err;
}

void StartServ()
{
	pthread_t pidSrv;
	pthread_create(&pidSrv, 0, CreateServThread, 0);
}

int main()
{
	signal(SIGPIPE, SIG_IGN);

	_epfdProduce = epoll_create(PRODUCE_FD_MAX);

	epoll_event ev;
	for (int i = 0; i < THREAD_NUM; ++i)
	{
		_epfd4Proc[i] = epoll_create(PROC_FD_MAX);

		_evfd4Recv[i] = eventfd(0, 0);
		ev.data.fd = _evfd4Recv[i];
		ev.events = EPOLLIN | EPOLLET;
		epoll_ctl(_epfd4Proc[i], EPOLL_CTL_ADD, _evfd4Recv[i], &ev);
		_eventTypeMap[_evfd4Recv[i]] = EventType::RECV;

		_evfd4Push[i] = eventfd(0, 0);
		ev.data.fd = _evfd4Push[i];
		ev.events = EPOLLIN | EPOLLET;
		epoll_ctl(_epfd4Proc[i], EPOLL_CTL_ADD, _evfd4Push[i], &ev);
		_eventTypeMap[_evfd4Push[i]] = EventType::PUSH;

		_evfd4RecvOk[i] = eventfd(0, 0);
		ev.data.fd = _evfd4RecvOk[i];
		ev.events = EPOLLIN | EPOLLET;
		epoll_ctl(_epfd4Proc[i], EPOLL_CTL_ADD, _evfd4RecvOk[i], &ev);
		_eventTypeMap[_evfd4RecvOk[i]] = EventType::RECV_OK;

		_evfd4SendOk[i] = eventfd(0, 0);
		ev.data.fd = _evfd4SendOk[i];
		ev.events = EPOLLIN | EPOLLET;
		epoll_ctl(_epfd4Proc[i], EPOLL_CTL_ADD, _evfd4SendOk[i], &ev);
		_eventTypeMap[_evfd4SendOk[i]] = EventType::SEND_OK;
	}

	for (int i = 0; i < THREAD_NUM; ++i)
	{
		ProcThreadArgs* pta = new ProcThreadArgs();
		memset(pta, 0, sizeof(ProcThreadArgs));
		pta->index = i;
		ptaArr[i] = pta;
	}

	pthread_t pidProduce;
	pthread_create(&pidProduce, 0, CreateProduceThread, 0);

	for (int i = 0; i < THREAD_NUM; ++i)
	{
		pthread_t pidProc;
		pthread_create(&pidProc, 0, CreateProcThread, ptaArr[i]);
	}

	pthread_t pidPush;
	pthread_create(&pidPush, 0, CreatePushThread, 0);

	pthread_t pidRelease;
	pthread_create(&pidRelease, 0, CreateReleaseThread, 0);

	StartServ();
	
	while (1 == 1)
	{
		sleep(3600);
	}
	return 0;
}