Memcached线程模型（1）

http://liuzhaomin.blog.163.com/blog/static/18363949820121913425501/

 

memcached使用libevent事件库来处理Socket事件。

libevent中的核心结构event_base不是线程安全的。memcached和libevent集成，每一个线程拥有一个event_base实例。

下面这个图，基本在分析memcached的文章中都是用了，经典啊。真是前人栽树，后人乘凉。

 

memcached中主线程负责监听客户端连接，把已建立连接的fd包装成CQ_ITEM。每一个工作线程worker thread持有一个 CQ_ITEM队列（fd队列，连接队列），此线程只处理这个队列中socket的事件 。主线程把 CQ_ITEM交给哪个worker thread去处理，是通过轮询的方式。

核心数据结构

---

/* 连接队列中的元素. */
typedef struct conn_queue_item CQ_ITEM;
struct conn_queue_item {
    int               sfd;
    enum conn_states  init_state;
    int               event_flags;
    int               read_buffer_size;
    enum network_transport     transport;
    CQ_ITEM          *next;
};

/* 连接队列，单向链表 */
typedef struct conn_queue CQ;
struct conn_queue {
    CQ_ITEM *head;
    CQ_ITEM *tail;
    pthread_mutex_t lock;   // 连接队列的互斥锁
    pthread_cond_t  cond;   // 连接队列的等待条件
};

// 线程状态
struct thread_stats {
    pthread_mutex_t   mutex;
    uint64_t          get_cmds;
    uint64_t          get_misses;
    uint64_t          touch_cmds;
    uint64_t          touch_misses;
    uint64_t          delete_misses;
    uint64_t          incr_misses;
    uint64_t          decr_misses;
    uint64_t          cas_misses;
    uint64_t          bytes_read;
    uint64_t          bytes_written;
    uint64_t          flush_cmds;
    uint64_t          conn_yields; /* # of yields for connections (-R option)*/
    uint64_t          auth_cmds;
    uint64_t          auth_errors;
    struct slab_stats slab_stats[MAX_NUMBER_OF_SLAB_CLASSES];
};

/* memcached中的线程封装 */
typedef struct {
    pthread_t thread_id;        /* 线程id */
    struct event_base *base;    /* event_base，当前线程唯一使用 */
    struct event notify_event;  /* listen event for notify pipe */
    int notify_receive_fd;      /* 通知管道接收端fd */
    int notify_send_fd;         /* 通知管道发送端fd */
    struct thread_stats stats;  /* Stats generated by this thread */
    struct conn_queue *new_conn_queue; /* 待处理连接的队列 */
    cache_t *suffix_cache;      /* suffix cache */
} LIBEVENT_THREAD;

/* redis对网络连接的封装 */
typedef struct conn conn;
struct conn {
    int    sfd;
    sasl_conn_t *sasl_conn;
    enum conn_states  state;
    enum bin_substates substate;

    struct event event;
    short  ev_flags;
    short  which;   /** which events were just triggered */

    char   *rbuf;   /** buffer to read commands into */
    char   *rcurr;  /** but if we parsed some already, this is where we stopped */
    int    rsize;   /** total allocated size of rbuf */
    int    rbytes;  /** how much data, starting from rcur, do we have unparsed */

    char   *wbuf;
    char   *wcurr;
    int    wsize;
    int    wbytes;
    /** which state to go into after finishing current write */
    enum conn_states  write_and_go;
    void   *write_and_free; /** free this memory after finishing writing */

    char   *ritem;  /** when we read in an item's value, it goes here */
    int    rlbytes;

    /* data for the nread state */

    /**
     * item is used to hold an item structure created after reading the command
     * line of set/add/replace commands, but before we finished reading the actual
     * data. The data is read into ITEM_data(item) to avoid extra copying.
     */

    void   *item;     /* for commands set/add/replace  */

    /* data for the swallow state */
    int    sbytes;    /* how many bytes to swallow */

    /* data for the mwrite state */
    struct iovec *iov;
    int    iovsize;   /* number of elements allocated in iov[] */
    int    iovused;   /* number of elements used in iov[] */

    struct msghdr *msglist;
    int    msgsize;   /* number of elements allocated in msglist[] */
    int    msgused;   /* number of elements used in msglist[] */
    int    msgcurr;   /* element in msglist[] being transmitted now */
    int    msgbytes;  /* number of bytes in current msg */

    item   **ilist;   /* list of items to write out */
    int    isize;
    item   **icurr;
    int    ileft;

    char   **suffixlist;
    int    suffixsize;
    char   **suffixcurr;
    int    suffixleft;

    enum protocol protocol;   /* which protocol this connection speaks */
    enum network_transport transport; /* what transport is used by this connection */

    /* data for UDP clients */
    int    request_id; /* Incoming UDP request ID, if this is a UDP "connection" */
    struct sockaddr request_addr; /* Who sent the most recent request */
    socklen_t request_addr_size;
    unsigned char *hdrbuf; /* udp packet headers */
    int    hdrsize;   /* number of headers' worth of space is allocated */

    bool   noreply;   /* True if the reply should not be sent. */
    /* current stats command */
    struct {
        char *buffer;
        size_t size;
        size_t offset;
    } stats;

    /* Binary protocol stuff */
    /* This is where the binary header goes */
    protocol_binary_request_header binary_header;
    uint64_t cas; /* the cas to return */
    short cmd; /* current command being processed */
    int opaque;
    int keylen;
    conn   *next;     /* Used for generating a list of conn structures */
    LIBEVENT_THREAD *thread; /* Pointer to the thread object serving this connection */
};

初始化流程，memcached.c-->main()

/* 初始化主线程libevent实例 */
main_base = event_init();

/* 启动所有的worker线程 */
thread_init(settings.num_threads, main_base);

/* 创建监听socket，绑定地址，设置非阻塞模式并注册监听socket的libevent 读事件 */
server_sockets(settings.port, tcp_transport, portnumber_file)
 内部调用event_set方法，设置server socket fd的EV_READ事件的处理函数是event_handler

/* 进入事件循环 */
event_base_loop(main_base, 0)
------------------------------------------------------------------------------------
event_handler---->drive_machine()

主分发线程
typedef struct {
pthread_t thread_id;        /* 线程id */
struct event_base *base;    /* event_base */
} LIBEVENT_DISPATCHER_THREAD;