epoll EPOLLOUT事件未触发

记录一次epoll事件未触发的经历，由于是tcp通信，使用了io多路复用epoll，客户端只发送数据，不处理服务端响应，服务端处理对应的事件，并且会响应消息给客户端，遇到了EPOLLOUT事件未触发的情况，进行排查。

经过查阅相关资料得知，socket通信时是全双工的，都有send、recv缓冲区，当recv缓冲区满了，就会响应ACK给对方，将发送窗口置为0，这个时候EPOLLOUT事件就无法触发，EPOLLOUT触发的前提条件是发送缓冲区可写，即对应的内核缓冲区为可写状态，这个时候不满足条件，EPOLLOUT事件就触发不了，响应消息就发送不过去。

附相关代码

客户端代码

#include<stdio.h>
#include<stdlib.h>
#include<unistd.h>
#include<arpa/inet.h>
#include<sys/socket.h>
#include<netinet/in.h>
#include<string.h>
#include<errno.h>

#define MAXLINE 1024

void cli_send_msg(FILE *fp, int sockfd)
{
    char    sendline[MAXLINE], recvline[MAXLINE];

    //while (fgets(sendline, MAXLINE, fp) != NULL) {
    while(1) {
        memset(sendline, 97, MAXLINE);
        int n = write(sockfd, sendline, strlen(sendline));
        if (n < 0) {
            fprintf(stderr, "write err errno(%d) %s\n", errno, strerror(errno));
        }
        memset(sendline, 0, sizeof(sendline));

        usleep(10);
        //if (read(sockfd, recvline, MAXLINE) == -1) {
        //    fprintf(stderr, "read err errno(%d) %s\n", errno, strerror(errno));
        //    close(sockfd);
        //}
        //memset(recvline, 0, MAXLINE);

        //Fputs(recvline, stdout);
    }
}

int main(int argc, const char **argv)
{
    int sockfd;
    struct sockaddr_in servaddr;

    if (argc != 3) {
        fprintf(stderr, "usage: tcpcli <ip> <port>\n");
        return -1;
    }

    sockfd = socket(AF_INET, SOCK_STREAM, 0);
    bzero(&servaddr, sizeof(servaddr));
    servaddr.sin_family = AF_INET;
    servaddr.sin_port = htons(atoi(argv[2]));
    int ret = inet_pton(AF_INET, argv[1], &servaddr.sin_addr);
    if (ret <= 0) {
        fprintf(stderr, "inet_pton err\n");
    }
    int len = 4;
    int rcvbuf_size = 3145728;
    setsockopt(sockfd, SOL_SOCKET, SO_RCVBUF, &rcvbuf_size, len);
    rcvbuf_size = 0;
    getsockopt(sockfd, SOL_SOCKET, SO_RCVBUF, &rcvbuf_size, &len);
    printf("recive buff size is %d\n", rcvbuf_size);
    ret = connect(sockfd, (struct sockaddr *)&servaddr, sizeof(servaddr));

    if (ret < 0) {
        fprintf(stderr, "connect err\n");
    }

    cli_send_msg(stdin, sockfd);

    return 0;
}

服务端代码 

#include<stdio.h>
#include<stdlib.h>
#include<string.h>
#include<errno.h>
#include<netinet/in.h>
#include<arpa/inet.h>
#include<sys/socket.h>
#include<sys/epoll.h>
#include<unistd.h>
#include<sys/types.h>


#define IPADDRESS "0.0.0.0"
#define PORT 7001
#define MAXSIZE 1024
#define LISTENQ 5
#define FDSIZE 1000
#define EPOLLEVENTS 1024

int rcv_c = 0;
int wt_c = 0;
int socket_bind(const char *ip, int port);

void do_epoll(int listenfd);

void handle_events(int epollfd, struct epoll_event *event, int num,int listenfd,char *buf);

void handle_accept(int epollfd, int listenfd);

void do_read(int epollfd, int connfd, char *buf);

void do_write(int epollfd, int connfd, char *buf);

void add_event(int epollfd, int connfd, int state);

void delete_event(int epollfd, int connfd, int state);

void modify_event(int epollfd, int connfd, int state);

int main(void)
{   
    int listenfd;
    listenfd = socket_bind(IPADDRESS, PORT);
    listen(listenfd, LISTENQ);
    do_epoll(listenfd);
    return 0;
}

int socket_bind(const char *ip, int port)
{   
    int fd;
    struct sockaddr_in servaddr;
    if ((fd = socket(AF_INET, SOCK_STREAM, 0)) == -1)
    {   
        perror("socket create error: ");
        exit(1);
    }
    servaddr.sin_family = AF_INET;
    inet_pton(AF_INET, ip ,&servaddr.sin_addr);
    servaddr.sin_port = htons(port);
    int reuse  = 1;
    setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse));
    if(bind(fd, (struct sockaddr*) &servaddr, sizeof(servaddr)) == -1)
    {   
        perror("bind error: ");
        exit(1);
    }
    return fd;
}

void do_epoll(int listenfd)
{   
    int epollfd;
    struct epoll_event events[EPOLLEVENTS];
    char buf[MAXSIZE];
    int ready = 0;
    
    memset(buf, 0, MAXSIZE);
    
    epollfd = epoll_create(FDSIZE);
    
    //添加服务监听事件
    add_event(epollfd,listenfd,EPOLLIN);
    //轮询处理事件
    for( ; ; )
    {
            printf("waiting...\n");
            ready = epoll_wait(epollfd, events, EPOLLEVENTS, -1);
            if(ready == 0)
            {
                fprintf(stderr, "epoll_wait timeout\n");
                continue;
            }
            printf("wait_fd %d\n", ready);
            handle_events(epollfd, events, ready, listenfd, buf);
    }


}

void add_event(int epollfd, int fd, int state)
{
    struct epoll_event ev;
    ev.events = state;
    ev.data.fd = fd;
    epoll_ctl(epollfd, EPOLL_CTL_ADD, fd, &ev);
}

void delete_event(int epollfd, int fd, int state)
{
    struct epoll_event ev;
    ev.events = state;
    ev.data.fd = fd;
    epoll_ctl(epollfd, EPOLL_CTL_DEL, fd, &ev);
}

void modify_event(int epollfd, int fd, int state)
{
    struct epoll_event ev;
    ev.events = state;
    ev.data.fd = fd;
    epoll_ctl(epollfd, EPOLL_CTL_MOD, fd, &ev);
}

void handle_events(int epollfd, struct epoll_event *events, int num,int listenfd,char *buf)
{
    int fd = -1;
    for(int i = 0; i < num; i++ )
    {
        fd = events[i].data.fd;
        if (fd == listenfd && events[i].events & EPOLLIN )
        {
            handle_accept(epollfd, listenfd);
        }else if (events[i].events & EPOLLIN )
        {
            //处理读事件fd->buf
            do_read(epollfd, fd, buf);
        }else if ( events[i].events & EPOLLOUT )
        {
            //处理写事件buf->fd
            do_write(epollfd, fd, buf);
        } else if( events[i].events & EPOLLERR) {
            close(fd);
            fprintf(stderr, "close connect, error causeed!\n");
        }

    }

}

void handle_accept(int epollfd, int listenfd)
{   
    int connfd;
    struct sockaddr_in cliaddr;
    socklen_t socklen;
    socklen = sizeof(cliaddr);
    if ( (connfd = accept(listenfd,(struct sockaddr*)&cliaddr, &socklen)) == -1 )
    {   
        perror("accept error: ");
        return;
    }
    
    fprintf(stdout, "accept a client : %s:%d\n", inet_ntoa(cliaddr.sin_addr), cliaddr.sin_port);
    
    //添加对客户端监听事件
    add_event(epollfd, connfd, EPOLLIN|EPOLLERR);
}

void do_read(int epollfd, int fd, char *buf)
{
    int n;
    n = recv(fd, buf, MAXSIZE, 0);

    if ( n == -1 )
    {
        char msg[64] = {0};
        int n = 0;
        if (errno == ECONNRESET) {
            n = sprintf(msg, "a client( %d ) close connection. ", fd);
            if (n < 0) {
                fprintf(stderr, "sprintf err\n");
            }
        }
        sprintf(msg+n, "[ERROR][%d]", errno);
        perror(msg);
        close(fd);
        delete_event(epollfd, fd, EPOLLIN);
    }else if ( n == 0 )
    {
        fprintf(stdout, "a client( %d ) close connection. [OK]\n", fd);
        close(fd);
        delete_event(epollfd, fd, EPOLLIN);
    }else
    {
        rcv_c++;
        printf("receive[%d]: %s\n", rcv_c, buf);
        modify_event(epollfd, fd, EPOLLOUT);
        printf("add write\n");
    }

}

void do_write(int epollfd, int fd, char *buf)
{
    int nwrite;
    nwrite = write(fd, buf, strlen(buf));
    if (nwrite == -1)
    {
        perror("write error:");
        close(fd);
        delete_event(epollfd, fd, EPOLLOUT);
    } else {
        wt_c++;
        printf("write count %d\n", wt_c);
        modify_event(epollfd, fd, EPOLLIN);
    }
    memset(buf, 0, MAXSIZE);
}