libevent代码阅读（16）——缓冲区

在hello-world的listener_cb函数中，我们看到该函数使用一个bufferevent对象（缓冲区事件），使用bufferevent的好处就是不用直接调用io操作函数即可进行读写，往缓冲区中写入数据就是往文件（或者套接字）中写入数据，从缓冲区中读取数据就是从文件或者套接字中读取数据。不仅如此，bufferevent还有很好的缓冲功能。

bufferevent的定义如下：

/*
 * 缓冲区事件处理器
 */
struct bufferevent {
	/** Event base for which this bufferevent was created. */
	// 指向Reactor
	struct event_base *ev_base;

	/** Pointer to a table of function pointers to set up how this
	    bufferevent behaves. */
	// 缓冲区的操作
	const struct bufferevent_ops *be_ops;

	/** A read event that triggers when a timeout has happened or a socket
	    is ready to read data.  Only used by some subtypes of
	    bufferevent. */
	// 读事件处理器
	struct event ev_read;
	/** A write event that triggers when a timeout has happened or a socket
	    is ready to write data.  Only used by some subtypes of
	    bufferevent. */
	// 写事件处理器
	struct event ev_write;

	/** An input buffer. Only the bufferevent is allowed to add data to
	    this buffer, though the user is allowed to drain it. */
	// 输入缓冲区
	struct evbuffer *input;

	/** An input buffer. Only the bufferevent is allowed to drain data
	    from this buffer, though the user is allowed to add it. */
	// 输出缓冲区
	struct evbuffer *output;

	// 读水位
	struct event_watermark wm_read;

	// 写水位
	struct event_watermark wm_write;

	// 读写缓存区的回调函数
	bufferevent_data_cb readcb;
	bufferevent_data_cb writecb;
	/* This should be called 'eventcb', but renaming it would break
	 * backward compatibility */
	bufferevent_event_cb errorcb;

	// 回调函数的参数
	void *cbarg;

	// 读写的超时时间
	struct timeval timeout_read;
	struct timeval timeout_write;

	/** Events that are currently enabled: currently EV_READ and EV_WRITE
	    are supported. */
	// 是否启用
	short enabled;
};

我们看到bufferevent包含了输入输出两个缓存区，其类型是evbuffer, 定义如下：

/*
 * 缓冲区
 */
struct evbuffer {
	/** The first chain in this buffer's linked list of chains. */
	// 第一块数据
	struct evbuffer_chain *first;

	/** The last chain in this buffer's linked list of chains. */
	// 最后一块数据
	struct evbuffer_chain *last;

	/** Pointer to the next pointer pointing at the 'last_with_data' chain.
	 *
	 * To unpack:
	 *
	 * The last_with_data chain is the last chain that has any data in it.
	 * If all chains in the buffer are empty, it is the first chain.
	 * If the buffer has no chains, it is NULL.
	 *
	 * The last_with_datap pointer points at _whatever 'next' pointer_
	 * points at the last_with_datap chain.  If the last_with_data chain
	 * is the first chain, or it is NULL, then the last_with_datap pointer
	 * is &buf->first.
	 */
	// 和尾队列有相似的功能？
	struct evbuffer_chain **last_with_datap;

	/** Total amount of bytes stored in all chains.*/
	// 缓冲区长度
	size_t total_len;

	/** Number of bytes we have added to the buffer since we last tried to
	 * invoke callbacks. */
	// 被添加到缓存区的字节数
	size_t n_add_for_cb;

	/** Number of bytes we have removed from the buffer since we last
	 * tried to invoke callbacks. */
	// 从缓冲区中删除的字节数
	size_t n_del_for_cb;

#ifndef _EVENT_DISABLE_THREAD_SUPPORT
	/** A lock used to mediate access to this buffer. */
	// 锁——线程支持
	void *lock;
#endif
	/** True iff we should free the lock field when we free this
	 * evbuffer. */
	// 是否拥有锁
	unsigned own_lock : 1;

	/** True iff we should not allow changes to the front of the buffer
	 * (drains or prepends). */
	// 冻结缓冲区头部
	unsigned freeze_start : 1;

	/** True iff we should not allow changes to the end of the buffer
	 * (appends) */
	// 冻结缓冲区尾部
	unsigned freeze_end : 1;
	/** True iff this evbuffer's callbacks are not invoked immediately
	 * upon a change in the buffer, but instead are deferred to be invoked
	 * from the event_base's loop.	Useful for preventing enormous stack
	 * overflows when we have mutually recursive callbacks, and for
	 * serializing callbacks in a single thread. */
	unsigned deferred_cbs : 1;
#ifdef WIN32
	/** True iff this buffer is set up for overlapped IO. */
	unsigned is_overlapped : 1;
#endif
	/** Zero or more EVBUFFER_FLAG_* bits */
	ev_uint32_t flags;

	/** Used to implement deferred callbacks. */
	// 延迟调用的回调函数队列
	struct deferred_cb_queue *cb_queue;

	/** A reference count on this evbuffer.	 When the reference count
	 * reaches 0, the buffer is destroyed.	Manipulated with
	 * evbuffer_incref and evbuffer_decref_and_unlock and
	 * evbuffer_free. */
	// 引用计数
	int refcnt;

	/** A deferred_cb handle to make all of this buffer's callbacks
	 * invoked from the event loop. */
	// 延迟调用的回调函数
	struct deferred_cb deferred;

	/** A doubly-linked-list of callback functions */
	// 回调函数队列
	TAILQ_HEAD(evbuffer_cb_queue, evbuffer_cb_entry) callbacks;

	/** The parent bufferevent object this evbuffer belongs to.
	 * NULL if the evbuffer stands alone. */
	// 它的父亲
	struct bufferevent *parent;
};

可以看到evbuffer是由很多个evbuffer_chain（缓冲区链）组成的，而evbuffer有点像尾队列，可以很方便的增删一个缓冲区链，evbuffer_chain定义如下：

// 缓存区链
struct evbuffer_chain {
	/** points to next buffer in the chain */

	// 下一个缓存区链
	struct evbuffer_chain *next;

	/** total allocation available in the buffer field. */
	// 缓存区长度
	size_t buffer_len;

	/** unused space at the beginning of buffer or an offset into a
	 * file for sendfile buffers. */
	// 不进行内存对齐
	ev_misalign_t misalign;

	/** Offset into buffer + misalign at which to start writing.
	 * In other words, the total number of bytes actually stored
	 * in buffer. */
	// 偏移量
	size_t off;

	/** Set if special handling is required for this chain */
	// 标志
	unsigned flags;

	// 共享内存
#define EVBUFFER_MMAP		0x0001	/**< memory in buffer is mmaped */
	// 用于sendfile函数
#define EVBUFFER_SENDFILE	0x0002	/**< a chain used for sendfile */
	// 内存引用
#define EVBUFFER_REFERENCE	0x0004	/**< a chain with a mem reference */
	// 只读
#define EVBUFFER_IMMUTABLE	0x0008	/**< read-only chain */
	/** a chain that mustn't be reallocated or freed, or have its contents
	 * memmoved, until the chain is un-pinned. */
#define EVBUFFER_MEM_PINNED_R	0x0010
#define EVBUFFER_MEM_PINNED_W	0x0020
#define EVBUFFER_MEM_PINNED_ANY (EVBUFFER_MEM_PINNED_R|EVBUFFER_MEM_PINNED_W)
	/** a chain that should be freed, but can't be freed until it is
	 * un-pinned. */
#define EVBUFFER_DANGLING	0x0040

	/** Usually points to the read-write memory belonging to this
	 * buffer allocated as part of the evbuffer_chain allocation.
	 * For mmap, this can be a read-only buffer and
	 * EVBUFFER_IMMUTABLE will be set in flags.  For sendfile, it
	 * may point to NULL.
	 */
	// 缓冲区
	unsigned char *buffer;
};

可以看到evbuffer_chain中才是真正存储数据的地方
下面我们看一下创建bufferevent对象的函数，bufferevent_socket_new，顾名思义，该函数通过套接字来新建一个bufferevent对象。它首先创建一个bufferevent_private对象，然后通过bufferevent_init_common进行初始化（创建读写缓存区和读写事件处理器），然后设置一些标志，然后给读写事件处理器赋值，最后冻结输入输出缓冲区。

/*
 * 根据套接字创建一个缓冲区
 */
struct bufferevent *
bufferevent_socket_new(struct event_base *base, evutil_socket_t fd,
    int options)
{
	// 内部私有的buffer，管理了bufferevent
	struct bufferevent_private *bufev_p;

	// 需要返回的结果
	struct bufferevent *bufev;

#ifdef WIN32
	if (base && event_base_get_iocp(base))
		return bufferevent_async_new(base, fd, options);
#endif

	// 内存分配
	if ((bufev_p = mm_calloc(1, sizeof(struct bufferevent_private)))== NULL)
		return NULL;

	// 内部私有的buffer初始化
	if (bufferevent_init_common(bufev_p, base, &bufferevent_ops_socket,
				    options) < 0) {
		mm_free(bufev_p);
		return NULL;
	}

	// 取出内部私有buffer的bufferevent作为返回结果
	bufev = &bufev_p->bev;

	// 设置输出缓冲区的标志
	evbuffer_set_flags(bufev->output, EVBUFFER_FLAG_DRAINS_TO_FD);

	// 读写信号处理器赋值
	event_assign(&bufev->ev_read, bufev->ev_base, fd,
	    EV_READ|EV_PERSIST, bufferevent_readcb, bufev);
	event_assign(&bufev->ev_write, bufev->ev_base, fd,
	    EV_WRITE|EV_PERSIST, bufferevent_writecb, bufev);

	// 添加输出缓冲区的回调函数
	evbuffer_add_cb(bufev->output, bufferevent_socket_outbuf_cb, bufev);

	// 冻结输入/输出缓存区
	evbuffer_freeze(bufev->input, 0);
	evbuffer_freeze(bufev->output, 1);

	return bufev;
}

bufferevent_init_common函数，它创建了输入输出事件处理器和输入输出缓存区，定义如下：

// bufferevent_private对象初始化
int
bufferevent_init_common(struct bufferevent_private *bufev_private,
    struct event_base *base,
    const struct bufferevent_ops *ops,
    enum bufferevent_options options)
{
	struct bufferevent *bufev = &bufev_private->bev;

	// 创建输入缓存区
	if (!bufev->input) {
		if ((bufev->input = evbuffer_new()) == NULL)
			return -1;
	}

	// 创建输出缓冲区
	if (!bufev->output) {
		if ((bufev->output = evbuffer_new()) == NULL) {
			evbuffer_free(bufev->input);
			return -1;
		}
	}

	// 增加引用计数
	bufev_private->refcnt = 1;

	// 设置所属的event_base对象
	bufev->ev_base = base;

	/* Disable timeouts. */
	// 读写超时时间清空
	evutil_timerclear(&bufev->timeout_read);
	evutil_timerclear(&bufev->timeout_write);

	// 一些选项
	bufev->be_ops = ops;

	/*
	 * Set to EV_WRITE so that using bufferevent_write is going to
	 * trigger a callback.  Reading needs to be explicitly enabled
	 * because otherwise no data will be available.
	 */
	// 默认的写开启
	bufev->enabled = EV_WRITE;

	// 多线程支持
#ifndef _EVENT_DISABLE_THREAD_SUPPORT
	// 如果是线程安全选项
	if (options & BEV_OPT_THREADSAFE) {
		// 启动锁功能
		if (bufferevent_enable_locking(bufev, NULL) < 0) {
			/* cleanup */
			evbuffer_free(bufev->input);
			evbuffer_free(bufev->output);
			bufev->input = NULL;
			bufev->output = NULL;
			return -1;
		}
	}
#endif
	// 如果设置了BEV_OPT_DEFER_CALLBACKS，事件循环中的回调函数将会延迟调用
	if ((options & (BEV_OPT_DEFER_CALLBACKS|BEV_OPT_UNLOCK_CALLBACKS))
	    == BEV_OPT_UNLOCK_CALLBACKS) {
		event_warnx("UNLOCK_CALLBACKS requires DEFER_CALLBACKS");
		return -1;
	}

	// 如果设置了延期调用回调函数
	if (options & BEV_OPT_DEFER_CALLBACKS) {

		// 如果是不带锁执行回调函数
		if (options & BEV_OPT_UNLOCK_CALLBACKS)
			event_deferred_cb_init(&bufev_private->deferred,
			    bufferevent_run_deferred_callbacks_unlocked,
			    bufev_private);
		else
		// 如果是带锁执行回调函数
			event_deferred_cb_init(&bufev_private->deferred,
			    bufferevent_run_deferred_callbacks_locked,
			    bufev_private);
	}

	// 选项
	bufev_private->options = options;

	// 设置输入缓冲区的父亲
	evbuffer_set_parent(bufev->input, bufev);

	// 设置输出缓冲区的父亲
	evbuffer_set_parent(bufev->output, bufev);

	return 0;
}

在给读写事件处理器赋值的时候有两个函数非常重要——bufferevent_readcb，bufferevent_writecb这两个函数就是实际调用套接字函数进行数据发送和接收的地方。每当事件多路分发器得到读写事件，就回调用这两个回调函数，实际就是从客户接收数据或者向客户端发送数据。

bufferevent_writecb函数：
首先调用了evbuffer_write_atmost(bufev->output, fd, atmost)往套接字写入数据。
然后调用_bufferevent_run_writecb，调用用户的回调函数bufev->writecb(bufev, bufev->cbarg);（注意writecb我们已经通过bufferevent_setcb来设置了）

bufferevent_readcb也一样：
首先evbuffer_read从套接字读取数据。
然后通过_bufferevent_run_readcb调用用户的回调函数bufev->readcb

而bufferevent_write和bufferevent_read则是简单的将数据插入到缓存区和从缓冲区中取出数据

真正管理bufferevent的是bufferevent_private这个对象，bufferevent_private对象内部有一个bufferevent对象，创建bufferevent_private之后即可完成bufferevent对象的创建。bufferevent_private的定义如下：

// 一个私有的结构体
struct bufferevent_private {
	/** The underlying bufferevent structure. */
	// 管理的bufferevent
	struct bufferevent bev;

	/** Evbuffer callback to enforce watermarks on input. */
	// 读写水位处理
	struct evbuffer_cb_entry *read_watermarks_cb;

	/** If set, we should free the lock when we free the bufferevent. */
	// 是否拥有锁
	unsigned own_lock : 1;

	/** Flag: set if we have deferred callbacks and a read callback is
	 * pending. */
	// 读事件投递
	unsigned readcb_pending : 1;
	/** Flag: set if we have deferred callbacks and a write callback is
	 * pending. */
	// 写事件投递
	unsigned writecb_pending : 1;
	/** Flag: set if we are currently busy connecting. */
	// 正在链接
	unsigned connecting : 1;
	/** Flag: set if a connect failed prematurely; this is a hack for
	 * getting around the bufferevent abstraction. */
	// 链接被拒绝
	unsigned connection_refused : 1;
	/** Set to the events pending if we have deferred callbacks and
	 * an events callback is pending. */
	// 回调函数事件的投递
	short eventcb_pending;

	/** If set, read is suspended until one or more conditions are over.
	 * The actual value here is a bitfield of those conditions; see the
	 * BEV_SUSPEND_* flags above. */
	// 读暂停标记
	bufferevent_suspend_flags read_suspended;

	/** If set, writing is suspended until one or more conditions are over.
	 * The actual value here is a bitfield of those conditions; see the
	 * BEV_SUSPEND_* flags above. */
	// 写暂停标记
	bufferevent_suspend_flags write_suspended;

	/** Set to the current socket errno if we have deferred callbacks and
	 * an events callback is pending. */
	// 错误码投递
	int errno_pending;

	/** The DNS error code for bufferevent_socket_connect_hostname */
	// dns错误码
	int dns_error;

	/** Used to implement deferred callbacks */
	// 用于回调函数
	struct deferred_cb deferred;

	/** The options this bufferevent was constructed with */
	// 缓冲区选项
	enum bufferevent_options options;

	/** Current reference count for this bufferevent. */
	// 引用计数
	int refcnt;

	/** Lock for this bufferevent.  Shared by the inbuf and the outbuf.
	 * If NULL, locking is disabled. */
	// 锁
	void *lock;

	/** Rate-limiting information for this bufferevent */
	// 速率限制
	struct bufferevent_rate_limit *rate_limiting;
};

针对bufferevent专门定义了一个用于操作bufferevent的结构体bufferevent_ops，例如启用、禁用、冲刷等等，定义如下：

// bufferevent的操作
struct bufferevent_ops {
	/** The name of the bufferevent's type. */
	// 类型
	const char *type;
	/** At what offset into the implementation type will we find a
	    bufferevent structure?

	    Example: if the type is implemented as
	    struct bufferevent_x {
	       int extra_data;
	       struct bufferevent bev;
	    }
	    then mem_offset should be offsetof(struct bufferevent_x, bev)
	*/
	// 内存偏移
	off_t mem_offset;

	/** Enables one or more of EV_READ|EV_WRITE on a bufferevent.  Does
	    not need to adjust the 'enabled' field.  Returns 0 on success, -1
	    on failure.
	 */
	// 启用
	int (*enable)(struct bufferevent *, short);

	/** Disables one or more of EV_READ|EV_WRITE on a bufferevent.  Does
	    not need to adjust the 'enabled' field.  Returns 0 on success, -1
	    on failure.
	 */
	// 禁用
	int (*disable)(struct bufferevent *, short);

	/** Free any storage and deallocate any extra data or structures used
	    in this implementation.
	 */
	// 销毁
	void (*destruct)(struct bufferevent *);

	/** Called when the timeouts on the bufferevent have changed.*/
	// 调整超时时间
	int (*adj_timeouts)(struct bufferevent *);

	/** Called to flush data. */
	// 冲刷
	int (*flush)(struct bufferevent *, short, enum bufferevent_flush_mode);

	/** Called to access miscellaneous fields. */
	// 控制
	int (*ctrl)(struct bufferevent *, enum bufferevent_ctrl_op, union bufferevent_ctrl_data *);

};

针对套接字的bufferevent_ops实例bufferevent_ops_socket定义如下：

const struct bufferevent_ops bufferevent_ops_socket = {
	"socket",
	evutil_offsetof(struct bufferevent_private, bev),
	be_socket_enable,
	be_socket_disable,
	be_socket_destruct,
	be_socket_adj_timeouts,
	be_socket_flush,
	be_socket_ctrl,
};