流缓冲区的特化版本(适用于通用tcp业务包)

最新推荐文章于 2025-01-01 00:05:56 发布

登高思危-少言勿躁

最新推荐文章于 2025-01-01 00:05:56 发布

阅读量318

点赞数 1

CC 4.0 BY-SA版权

本文链接：https://blog.youkuaiyun.com/undercallfile/article/details/78138348

这是c++标准库的流缓冲区接口

http://www.cplusplus.com/reference/streambuf/streambuf/

基本理念是使用游标标识缓冲区的读写点，让缓冲区尽量以惰性扩容和减容来减少性能开支.

一下是我的特化版本的的主体结构

// Ethernet len[46~1500] MTU 20
// udp head 8 1500 - 20 - 8 = 1472
// tcp head 20 1500 - 20 - 20 = 1460
// we define packet page size to 1024.
#define MM_PACKET_PAGE_SIZE 1024

// streambuf page size.default is 1024.
#define MM_STREAMBUF_PAGE_SIZE 1024

// output sequence (put)
// +++++-------
// | | |
// 0 pptr size
///
// input sequence (get)
// +++++-------
// | | |
// 0 gptr size

灰色代表已经读取,绿色代表已经写入等待读取,白色代表等待写入.

先说说一页的大小缺省为1024的原因：

1.经过统计，我们接触的90%以上的通用tcp业务包数据基本为int,单包总大小不会超过1k.

2.Ethernet len[46~1500] MTU 20,现代路由器基本都是1500MTU.

包体在缓冲区中通常会面临以下几种状况:

红色代表需要写入的数据超出的部分

第一张图表示数据游标已经写到末尾，超出数据缓冲区时，超出部分小于可用长度，仅做一次数据迁移(有效数据拷贝到缓冲区头，变成第三图)即可

第二张图表示缓冲区超出，总长超过一页(比如两页，闲置了一页)，除了做数据迁移，还会做一次减容，将缓冲区减少为一页

第四张图表示由于特殊情况导致可用缓冲区够用(但是需要做一次数据迁移)，但是有效数据超过一页，就会简单附加一页而不是做数据迁移(数据过长迁移代价变大了)

我的实现是尽量惰性的做减容,尽可能的减少大块数据(大于一页)的数据迁移(比如最后一种情况)。只有在数据超出时才会做一次数据迁移.

扩容和减容策略:

mm_streambuf.h

#ifndef __mm_streambuf_h__
#define __mm_streambuf_h__

#include "core/mm_prefix.h"

#include "core/mm_core.h"
#include "core/mm_platform.h"
#include "core/mm_types.h"
#include <stdio.h>

// streambuf page size.default is 1024.
#define MM_STREAMBUF_PAGE_SIZE 1024

// output sequence (put)
//    +++++-------
//    |    |      |
//    0  pptr   size
///
// input  sequence (get)
//    +++++-------
//    |    |      |
//    0  gptr   size
struct mm_streambuf
{
	// output sequence (put)
	size_t pptr;
	// input  sequence (get)
	size_t gptr;

	// max size for this streambuf.default is MM_STREAMBUF_PAGE_SIZE = 1024
	size_t size;
	mm_uint8_t* buff;// buffer for real data.
};
//
MM_EXPORT_DLL void mm_streambuf_init(struct mm_streambuf* p);
MM_EXPORT_DLL void mm_streambuf_destroy(struct mm_streambuf* p);
//
// copy q to p.
MM_EXPORT_DLL void mm_streambuf_copy(struct mm_streambuf* p, struct mm_streambuf* q);
// add max streambuf size.not checking overflow,use it when overflow only.
MM_EXPORT_DLL void mm_streambuf_addsize(struct mm_streambuf* p, size_t size);

MM_EXPORT_DLL void mm_streambuf_removeget(struct mm_streambuf* p);
MM_EXPORT_DLL void mm_streambuf_removeget_size(struct mm_streambuf* p, size_t rsize);
MM_EXPORT_DLL void mm_streambuf_clearget(struct mm_streambuf* p);

MM_EXPORT_DLL void mm_streambuf_removeput(struct mm_streambuf* p);
MM_EXPORT_DLL void mm_streambuf_removeput_size(struct mm_streambuf* p, size_t wsize);
MM_EXPORT_DLL void mm_streambuf_clearput(struct mm_streambuf* p);

MM_EXPORT_DLL size_t mm_streambuf_getsize(struct mm_streambuf* p);
MM_EXPORT_DLL size_t mm_streambuf_putsize(struct mm_streambuf* p);

// size for input and output interval.
MM_EXPORT_DLL size_t mm_streambuf_size(struct mm_streambuf* p);

// set get pointer to new pointer.
MM_EXPORT_DLL void mm_streambuf_setg_ptr(struct mm_streambuf* p, size_t new_ptr);
// set put pointer to new pointer.
MM_EXPORT_DLL void mm_streambuf_setp_ptr(struct mm_streambuf* p, size_t new_ptr);
// set get and put pointer to zero.
MM_EXPORT_DLL void mm_streambuf_reset(struct mm_streambuf* p);

// get get pointer(offset).
MM_EXPORT_DLL size_t mm_streambuf_gptr(struct mm_streambuf* p);
// get put pointer(offset).
MM_EXPORT_DLL size_t mm_streambuf_pptr(struct mm_streambuf* p);
//
// get data s + offset length is n.and gbump n.
MM_EXPORT_DLL size_t mm_streambuf_sgetn(struct mm_streambuf* p, mm_uint8_t* s, size_t o, size_t n);
// put data s + offset length is n.and pbump n.
MM_EXPORT_DLL size_t mm_streambuf_sputn(struct mm_streambuf* p, mm_uint8_t* s, size_t o, size_t n);
// gptr += n
MM_EXPORT_DLL void mm_streambuf_gbump(struct mm_streambuf* p, size_t n);
// pptr += n
MM_EXPORT_DLL void mm_streambuf_pbump(struct mm_streambuf* p, size_t n);
//
MM_EXPORT_DLL void mm_streambuf_underflow(struct mm_streambuf* p);
MM_EXPORT_DLL void mm_streambuf_overflow(struct mm_streambuf* p);
//
// if p->gptr >= MM_STREAMBUF_PAGE_SIZE we realloc a thin memory.ohter we just remove get.make sure the p->pptr + n <= p->size
MM_EXPORT_DLL void mm_streambuf_try_decrease(struct mm_streambuf* p, size_t n);
// check overflow when need write size n.
MM_EXPORT_DLL void mm_streambuf_try_increase(struct mm_streambuf* p, size_t n);
// aligned streambuf memory if need sputn n size buffer before.
MM_EXPORT_DLL void mm_streambuf_aligned_memory(struct mm_streambuf* p, size_t n);
//
#include "core/mm_suffix.h"

#endif//__mm_streambuf_h__

mm_streambuf.c

#include "core/mm_streambuf.h"
#include "core/mm_alloc.h"

// realloc size (nsize+1024*1024*n,n<32) will case a malloc and memcpy.
MM_EXPORT_DLL void mm_streambuf_init(struct mm_streambuf* p)
{
	// setp
	p->pptr = 0;
	// setg
	p->gptr = 0;
	//
	p->size = MM_STREAMBUF_PAGE_SIZE;
	p->buff = (mm_uint8_t*)mm_malloc(p->size);
}
MM_EXPORT_DLL void mm_streambuf_destroy(struct mm_streambuf* p)
{
	// setp
	p->pptr = 0;
	// setg
	p->gptr = 0;
	//
	mm_free(p->buff);
	//
	p->size = 0;
	p->buff = NULL;
}
MM_EXPORT_DLL void mm_streambuf_copy(struct mm_streambuf* p, struct mm_streambuf* q)
{
	mm_uint8_t* buff = (mm_uint8_t*)mm_malloc(q->size);
	mm_memcpy(buff,q->buff,q->size);
	mm_free(p->buff);
	p->buff = buff;
	p->size = q->size;
	// setp
	p->pptr = q->pptr;
	// setg
	p->gptr = q->gptr;
}
MM_EXPORT_DLL void mm_streambuf_addsize(struct mm_streambuf* p, size_t size)
{
	mm_uint8_t* buff = (mm_uint8_t*)mm_malloc(p->size + size);
	mm_memcpy(buff,p->buff,p->size);
	mm_free(p->buff);
	p->buff = buff;
	p->size += size;
}
MM_EXPORT_DLL void mm_streambuf_removeget(struct mm_streambuf* p)
{
	size_t rsize = p->gptr;
	if (0 < rsize)
	{
		size_t wr = p->pptr;
		mm_memmove(p->buff, p->buff + rsize, p->pptr - p->gptr);
		wr -= rsize;
		// setp
		p->pptr = wr;
		// setg
		p->gptr = 0;
	}
}
MM_EXPORT_DLL void mm_streambuf_removeget_size(struct mm_streambuf* p, size_t rsize)
{
	size_t rdmax = 0;
	mm_streambuf_clearget(p);
	rdmax = p->pptr - p->gptr;
	if (rdmax < rsize)
	{
		rsize = rdmax;
	}
	if (0 < rsize)
	{
		size_t wr = p->pptr;
		mm_memmove(p->buff, p->buff + rsize, p->pptr - p->gptr);
		wr -= rsize;
		// setp
		p->pptr = wr;
		// setg
		p->gptr = 0;
	}
}
MM_EXPORT_DLL void mm_streambuf_clearget(struct mm_streambuf* p)
{
	// setg
	p->gptr = 0;
}

MM_EXPORT_DLL void mm_streambuf_removeput(struct mm_streambuf* p)
{
	size_t wsize = p->pptr;
	if (0 < wsize)
	{
		size_t rd = p->gptr;
		mm_memmove(p->buff, p->buff + wsize, p->pptr - p->gptr);
		rd -= wsize;
		// setp
		p->pptr = 0;
		// setg
		p->gptr = rd;
	}
}
MM_EXPORT_DLL void mm_streambuf_removeput_size(struct mm_streambuf* p, size_t wsize)
{
	size_t wrmax = 0;
	mm_streambuf_clearput(p);
	wrmax = p->pptr - p->gptr;
	if (wrmax < wsize)
	{
		wsize = wrmax;
	}
	if (0 < wsize)
	{
		size_t rd = p->gptr;
		mm_memmove(p->buff, p->buff + wsize, p->pptr - p->gptr);
		rd -= wsize;
		// setp
		p->pptr = 0;
		// setg
		p->gptr = rd;
	}
}
MM_EXPORT_DLL void mm_streambuf_clearput(struct mm_streambuf* p)
{
	// setp
	p->pptr = 0;
}

MM_EXPORT_DLL size_t mm_streambuf_getsize(struct mm_streambuf* p)
{
	return p->gptr;	
}
MM_EXPORT_DLL size_t mm_streambuf_putsize(struct mm_streambuf* p)
{
	return p->pptr;	
}
MM_EXPORT_DLL size_t mm_streambuf_size(struct mm_streambuf* p)
{
	return p->pptr - p->gptr;
}

// set get pointer to new pointer.
MM_EXPORT_DLL void mm_streambuf_setg_ptr(struct mm_streambuf* p, size_t new_ptr)
{
	// setg
	p->gptr = new_ptr;
}
// set put pointer to new pointer.
MM_EXPORT_DLL void mm_streambuf_setp_ptr(struct mm_streambuf* p, size_t new_ptr)
{
	// setp
	p->pptr = new_ptr;
}
MM_EXPORT_DLL void mm_streambuf_reset(struct mm_streambuf* p)
{
	// setp
	p->pptr = 0;
	// setg
	p->gptr = 0;
}
// get get pointer(offset).
MM_EXPORT_DLL size_t mm_streambuf_gptr(struct mm_streambuf* p)
{
	return p->gptr;
}
// get put pointer(offset).
MM_EXPORT_DLL size_t mm_streambuf_pptr(struct mm_streambuf* p)
{
	return p->pptr;
}

MM_EXPORT_DLL size_t mm_streambuf_sgetn(struct mm_streambuf* p, mm_uint8_t* s, size_t o, size_t n)
{
	while( p->gptr + n > p->size )
	{
		mm_streambuf_underflow(p);
	}
	mm_memcpy(s + o, p->buff + p->gptr, n);
	p->gptr += n;
	return n;
}

MM_EXPORT_DLL size_t mm_streambuf_sputn(struct mm_streambuf* p, mm_uint8_t* s, size_t o, size_t n)
{
	while( p->pptr + n > p->size )
	{
		mm_streambuf_overflow(p);
	}
	mm_memcpy(p->buff + p->pptr, s + o, n);
	p->pptr += n;
	return n;
}

MM_EXPORT_DLL void mm_streambuf_gbump(struct mm_streambuf* p, size_t n)
{
	p->gptr += n;
}
MM_EXPORT_DLL void mm_streambuf_pbump(struct mm_streambuf* p, size_t n)
{
	p->pptr += n;
}

MM_EXPORT_DLL void mm_streambuf_underflow(struct mm_streambuf* p)
{
	if(0 < p->gptr)
	{
		// if gptr have g data.we try remove for free space.
		mm_streambuf_removeget(p);
	}
	else
	{
		// if gptr no data we just add page size.
		mm_streambuf_addsize(p,MM_STREAMBUF_PAGE_SIZE);
	}
}
MM_EXPORT_DLL void mm_streambuf_overflow(struct mm_streambuf* p)
{
	mm_streambuf_addsize(p,MM_STREAMBUF_PAGE_SIZE);
}
// if p->gptr >= MM_STREAMBUF_PAGE_SIZE we realloc a thin memory.ohter we just remove get.make sure the p->pptr + n <= p->size
MM_EXPORT_DLL void mm_streambuf_try_decrease(struct mm_streambuf* p, size_t n)
{
	size_t cursz = p->pptr - p->gptr;
	size_t fresz = p->size - cursz;
	// free size > n and MM_STREAMBUF_PAGE_SIZE < fresz - n.
	if ( fresz > n && MM_STREAMBUF_PAGE_SIZE < fresz - n )
	{
		// the surplus size for decrease.
		size_t ssize = fresz - n;
		// memmove size is <  MM_STREAMBUF_PAGE_SIZE.
		// decrease
		mm_uint8_t* buff = NULL;
		// use memove to remove the already get buffer.
		size_t wr = p->pptr;
		p->size -= (ssize / MM_STREAMBUF_PAGE_SIZE) * MM_STREAMBUF_PAGE_SIZE;
		buff = (mm_uint8_t*)mm_malloc(p->size);
		mm_memcpy(buff, p->buff + p->gptr, cursz);
		mm_free(p->buff);
		p->buff = buff;
		wr -= p->gptr;
		// setp
		p->pptr = wr;
		// setg
		p->gptr = 0;
	}
	else
	{
		// use memove to remove the already get buffer.
		mm_streambuf_removeget(p);
	}
}
// check overflow when need write size n.
MM_EXPORT_DLL void mm_streambuf_try_increase(struct mm_streambuf* p, size_t n)
{
	if ( p->pptr + n > p->size )
	{
		// the increase process is here.
		// assert( p->pptr + n > p->size && "(p->pptr + n > p->size) is invalid.");
		// size_t dsz = p->pptr + n - p->size;
		// assert( 0 < dsz );
		// size_t asz = dsz / MM_STREAMBUF_PAGE_SIZE;
		// size_t bsz = dsz % MM_STREAMBUF_PAGE_SIZE;
		// assert( 0 != bsz );
		// size_t nsz = ( asz + ( 0 == bsz ? 0 : 1 ) );
		// assert( 0 < nsz );
		// mm_streambuf_addsize(p, nsz * MM_STREAMBUF_PAGE_SIZE);

		// the quick process is here.
		size_t dsz = p->pptr + n - p->size;
		size_t asz = dsz / MM_STREAMBUF_PAGE_SIZE;
		size_t nsz = asz + 1;
		mm_streambuf_addsize(p, nsz * MM_STREAMBUF_PAGE_SIZE);
	}
}
// aligned streambuf memory if need sputn n size buffer before.
MM_EXPORT_DLL void mm_streambuf_aligned_memory(struct mm_streambuf* p, size_t n)
{
	// if current pptr + n not enough for target n.we make a aligned memory.
	if ( p->pptr + n > p->size )
	{
		size_t cursz = p->pptr - p->gptr;
		// the free size is enough and memmove size is <  MM_STREAMBUF_PAGE_SIZE.
		if ( p->size - cursz >= n && MM_STREAMBUF_PAGE_SIZE > cursz)
		{
			mm_streambuf_try_decrease(p, n);
		} 
		else
		{
			mm_streambuf_try_increase(p, n);
		}
	}
}

mm_streambuf.java

package org.mm.core;

//output sequence (put)
//+++++-------
//|    |      |
//0  pptr   size
///
//input  sequence (get)
//+++++-------
//|    |      |
//0  gptr   size
public class mm_streambuf
{
	public static final int MM_STREAMBUF_PAGE_SIZE = 1024;
	
	// output sequence (put)
	public int pptr = 0;
	// input  sequence (get)
	public int gptr = 0;

	// max size for this streambuf.default is MM_STREAMBUF_PAGE_SIZE = 1024
	public int size = 0;
	public byte[] buff = null;// buffer for real data.
	
	public void init()
	{
		this.pptr = 0;
		this.gptr = 0;
		this.size = MM_STREAMBUF_PAGE_SIZE;
		this.buff = new byte[MM_STREAMBUF_PAGE_SIZE];
	}
	public void destroy()
	{
		this.pptr = 0;
		this.gptr = 0;
		this.size = 0;
		this.buff = null;
	}
	
	// copy q to p.
	public void copy(mm_streambuf q)
	{
		byte[] buff = new byte[q.size];
		System.arraycopy(q.buff,0,buff,0,q.size);
		this.buff = buff;
		this.size = q.size;
		// setp
		this.pptr = q.pptr;
		// setg
		this.gptr = q.gptr;
	}
	// add max streambuf size.not checking overflow,use it when overflow only.
	public void addsize(int size)
	{
		byte[] buff = new byte[this.size + size];
		System.arraycopy(this.buff,0,buff,0,this.size);
		this.buff = buff;
		this.size += size;
	}

	public void removeget()
	{
		int rsize = this.gptr;
		if (0 < rsize)
		{
			int wr = this.pptr;
			System.arraycopy(this.buff, rsize, this.buff, 0, this.pptr - this.gptr);
			wr -= rsize;
			// setp
			this.pptr = wr;
			// setg
			this.gptr = 0;
		}
	}
	public void removeget_size( int rsize )
	{
		int rdmax = 0;
		this.clearget();
		rdmax = this.pptr - this.gptr;
		if (rdmax < rsize)
		{
			rsize = rdmax;
		}
		if (0 < rsize)
		{
			int wr = this.pptr;
			System.arraycopy(this.buff, rsize, this.buff, 0, this.pptr - this.gptr);
			wr -= rsize;
			// setp
			this.pptr = wr;
			// setg
			this.gptr = 0;
		}
	}
	public void clearget()
	{
		// setg
		this.gptr = 0;
	}

	public void removeput()
	{
		int wsize = this.pptr;
		if (0 < wsize)
		{
			int rd = this.gptr;
			System.arraycopy(this.buff, wsize, this.buff, 0, this.pptr - this.gptr);
			rd -= wsize;
			// setp
			this.pptr = 0;
			// setg
			this.gptr = rd;
		}
	}
	public void removeput_size( int wsize )
	{
		int wrmax = 0;
		this.clearput();
		wrmax = this.pptr - this.gptr;
		if (wrmax < wsize)
		{
			wsize = wrmax;
		}
		if (0 < wsize)
		{
			int rd = this.gptr;
			System.arraycopy(this.buff, wsize, this.buff, 0, this.pptr - this.gptr);
			rd -= wsize;
			// setp
			this.pptr = 0;
			// setg
			this.gptr = rd;
		}
	}
	public void clearput()
	{
		// setp
		this.pptr = 0;
	}

	public int getsize()
	{
		return this.gptr;
	}
	public int putsize()
	{
		return this.pptr;	
	}

	// size for input and output interval.
	public int size()
	{
		return this.pptr - this.gptr;
	}

	// set get pointer to new pointer.
	public void setg_ptr( int new_ptr )
	{
		// setg
		this.gptr = new_ptr;
	}
	// set put pointer to new pointer.
	public void setp_ptr( int new_ptr )
	{
		// setp
		this.pptr = new_ptr;
	}
	// set get and put pointer to zero.
	public void reset()
	{
		// setp
		this.pptr = 0;
		// setg
		this.gptr = 0;
	}

	// get get pointer(offset).
	public int gptr()
	{
		return this.gptr;
	}
	// get put pointer(offset).
	public int pptr()
	{
		return this.pptr;
	}
	//
	// get data s + offset length is n.and gbump n.
	public int sgetn( byte[] s, int o, int n )
	{
		while( this.gptr + n > this.size )
		{
			this.underflow();
		}
		System.arraycopy(this.buff, this.gptr, s, o, n);
		this.gptr += n;
		return n;
	}
	// put data s + offset length is n.and pbump n.
	public int sputn( byte[] s, int o, int n )
	{
		while( this.pptr + n > this.size )
		{
			this.overflow();
		}
		System.arraycopy(s, o, this.buff, this.pptr, n);
		this.pptr += n;
		return n;
	}
	// gptr += n
	public void gbump( int n )
	{
		this.gptr += n;
	}
	// pptr += n
	public void pbump( int n )
	{
		this.pptr += n;
	}
	//
	public void underflow()
	{
		if(0 < this.gptr)
		{
			// if gptr have g data.we try remove for free space.
			this.removeget();
		}
		else
		{
			// if gptr no data we just add page size.
			this.addsize(MM_STREAMBUF_PAGE_SIZE);
		}
	}
	public void overflow()
	{
		this.addsize(MM_STREAMBUF_PAGE_SIZE);
	}
	// if p->gptr >= MM_STREAMBUF_PAGE_SIZE we realloc a thin memory.ohter we just remove get.make sure the p->pptr + n <= p->size
	public void try_decrease( int n )
	{
		int cursz = this.pptr - this.gptr;
		int fresz = this.size - cursz;
		// free size > n and MM_STREAMBUF_PAGE_SIZE < fresz - n.
		if ( fresz > n && MM_STREAMBUF_PAGE_SIZE < fresz - n )
		{
			// the surplus size for decrease.
			int ssize = fresz - n;
			// memmove size is <  MM_STREAMBUF_PAGE_SIZE.
			// decrease
			byte[] buff = null;
			// use memove to remove the already get buffer.
			int wr = this.pptr;
			this.size -= (ssize / MM_STREAMBUF_PAGE_SIZE) * MM_STREAMBUF_PAGE_SIZE;
			buff = new byte[this.size];
			System.arraycopy(this.buff, this.gptr, buff, 0, cursz);
			this.buff = buff;
			wr -= this.gptr;
			// setp
			this.pptr = wr;
			// setg
			this.gptr = 0;
		}
		else
		{
			// use memove to remove the already get buffer.
			this.removeget();
		}
	}
	// check overflow when need write size n.
	public void try_increase( int n )
	{
		if ( this.pptr + n > this.size )
		{
			// the increase process is here.
			// assert( p->pptr + n > p->size && "(p->pptr + n > p->size) is invalid.");
			// size_t dsz = p->pptr + n - p->size;
			// assert( 0 < dsz );
			// size_t asz = dsz / MM_STREAMBUF_PAGE_SIZE;
			// size_t bsz = dsz % MM_STREAMBUF_PAGE_SIZE;
			// assert( 0 != bsz );
			// size_t nsz = ( asz + ( 0 == bsz ? 0 : 1 ) );
			// assert( 0 < nsz );
			// mm_streambuf_addsize(p, nsz * MM_STREAMBUF_PAGE_SIZE);

			// the quick process is here.
			int dsz = this.pptr + n - this.size;
			int asz = dsz / MM_STREAMBUF_PAGE_SIZE;
			int nsz = asz + 1;
			this.addsize( nsz * MM_STREAMBUF_PAGE_SIZE );
		}
	}
	// aligned streambuf memory if need sputn n size buffer before.
	public void aligned_memory( int n )
	{
		// if current pptr + n not enough for target n.we make a aligned memory.
		if ( this.pptr + n > this.size )
		{
			int cursz = this.pptr - this.gptr;
			// the free size is enough and memmove size is <  MM_STREAMBUF_PAGE_SIZE.
			if ( this.size - cursz >= n && MM_STREAMBUF_PAGE_SIZE > cursz)
			{
				this.try_decrease( n );
			} 
			else
			{
				this.try_increase( n );
			}
		}
	}
}