使用自定义堆，代替系统缺省堆？

原创已于 2022-05-04 07:00:53 修改 · 340 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#数据结构

于 2022-05-04 00:54:28 首次发布

本文探讨了为避免内存泄漏，通过自定义全局堆来替代系统默认堆的实现。作者详细介绍了自定义堆的管理结构、分配与释放策略，并在遇到问题时进行了优化，包括合并连续的空闲块以减少碎片。同时，文章提到了使用自定义堆可能带来的二进制文件大小增加和全局区域大小的设定问题。虽然解决了内存管理问题，但作者也提醒了线程安全和数据溢出等潜在风险。

前一篇文章追踪heap使用，检测内存泄漏？既然没有解决问题（上一篇主要是帮助发现问题），那么能不能避开缺省堆呢，使用自己管理的堆，调用接口又能完全兼容呢？于是，有了如下想法：
1. 使用一大段全局区作为自定义的堆（代码中的g_heap[MAX_HEAP_SIZE]），所有的堆分配请求，和用于堆管理的内存（HEAP_BLOCK）都使用这段内存。
2. 将这一大段区域，处理成HEAP_BLOCK为结点的双向链表，每个结点HEAP_BLOCK要么处于已分配状态（ALLOCATED），要么可分配状态（FREE），每个结点还包含内存大小等信息，具体见HEAP_BLOCK定义。注意：为了避免使用系统缺省堆，结点在自定义heap区域是依次连续的，只是结点状态和内存大小会随着分配和释放不停变化。初始状态下，整个自定义heap区域被视为一个FREE态的HEAP_BLOCK。
3. 分配请求的处理：在链表中寻找第一个未分配且buff足够大的HEAP_BLOCK，如找到就进行分配（即1个FREE的HEAP_BLOCK可能分裂成1个ALLOCATED的HEAP_BLOCK和1个FREE的HEAP_BLOCK）；如果没找到，先尝试合并连续的FREE态的HEAP_BLOCK（即将多个连续的FREE态HEAP_BLOCK合并成一个大的FREE态HEAP_BLOCK）。
4. 释放请求的处理：根据传入的地址，将对应的HEAP_BLOCK标记为FREE。
5. 已考虑：统计自定义堆的内存使用峰值，避免多个很小的FREE态HEAP_BLOCK(避免太多内存碎片)，等等，具体见代码。
6. 未考虑：线程安全；已尽可能考虑数据大小溢出的问题。
以下是代码（macro USE_MY_HEAP包含的是本文描述的内容，macro USE_SYS_HEAP_TRACK包含的是上一篇文章描述的内容)，和遇到问题的简单总结。

完整的头文件的内容如下（heap_wrapper.h）:

#pragma once

#define USE_MY_HEAP               //use the heap implemented based on a big range of global data
//#define USE_SYS_HEAP_TRACK        //use the system heap and we track the heap usage
//#define USE_SYS_HEAP_NO_TRACK     //just the system heap


#if defined(USE_MY_HEAP)
void * custom_calloc(uint32_t num, uint32_t size, const char* function_name, uint32_t line_num);
void custom_free(void* ptr);
void print_heap_info();

#define NEW_CALLOC(num,size)  custom_calloc(num,size,__FUNCTION__,__LINE__)
#define NEW_FREE(ptr)         custom_free(ptr)
#define PRINT_HEAP_INFO()     print_heap_info()
#endif

#if defined(USE_SYS_HEAP_TRACK)
void * custom_calloc(uint32_t num, uint32_t size, const char* function_name, uint32_t line_num);
void custom_free(void* ptr);
void print_heap_info();

#define NEW_CALLOC(num,size)  custom_calloc(num,size,__FUNCTION__,__LINE__)
#define NEW_FREE(ptr)         custom_free(ptr)
#define PRINT_HEAP_INFO()     print_heap_info()
#endif

#if defined(USE_SYS_HEAP_NO_TRACK)
#define NEW_CALLOC            calloc
#define NEW_FREE              free
#define PRINT_HEAP_INFO()     print_heap_info()
#endif

完整的源代码文件如下（heap_wrapper.c）：

#include "heap_wrapper.h"

#if defined(USE_MY_HEAP)
//use a big range of global area as the heap
//all heap request from callers and heap management is based on it

//the global heap area is a list of blocks, either in allocated state (used for a heap memory request)  or free state(not used and available for request)
//each block begins with a fixed-size block information (state, start/size, function_name/line_num, next/previous) and a variable buffer (specified by start/size in block information)

//initially, the whole heap area is one block of free area

//Process a calloc request:
//step 1: search the first free block that is big enough, 
//step 2: if found, 
//step 2.1: do the allocation: that is 1 free block changed into 1 allocated block and 1 free block(if more bytes left)
//Step 2.2: inserted the 2 new blocks into the list to replace the original one
//Step 3:  if not found, merge the continous free blocks: multiple continous free block are replaced with 1 big free block in the list, goto step 1 and try again

//Process a free request:
//per the input address, find the block in the list and mark it as free

//limitation not thread safe



//if after a allocate request, the left memory is not big enough, not to mark it as a new free block
//to avoid too many small free blocks
#define MIN_BLOCK_AVAILABLE_BUFF_SIZE   (32)   
#define MAX_HEAP_SIZE                   (2*1024*1024)  //2MB

#dfine MAX_FUNCTION_NAME_LEN            (32)
static char    g_function_name[MAX_FUNCTION_NAME_LEN] = "unkown";
static uint8_t g_heap[MAX_HEAP_SIZE];

typedef enum _HEAP_BLOCK_STATE_
{
	HEAP_BLOCK_FREE = 0,
	HEAP_BLOCK_ALLOCATED,
}HEAP_BLOCK_STATE;

typedef struct _HEAP_BLOCK_ HEAP_BLOCK, *PHEAP_BLOCK;
typedef struct _HEAP_BLOCK_
{
	HEAP_BLOCK_STATE state;
	void*            start;       //start of buffer for caller's request (not including this HEAP_BLOCK)
	uint32_t         size;        //buffer size available for caller's request (not including this HEAP_BLOCK)
	uint32_t         line_num;
	char             function_name[MAX_FUNCTION_NAME_LEN];   //NOTE: trunc it if too long
	PHEAP_BLOCK      next;
	PHEAP_BLOCK      previous;	
}HEAP_BLOCK, *PHEAP_BLOCK; 

typedef struct _HEAP_CONTEXT_
{
	uint64_t heap_size;          //size of all the reserved heap memory
	uint64_t peak_size;          //peak size of all the requests from callers
	uint64_t total_alloc_size;   //current allocated size from callers
	uint32_t total_alloc_count;  //increment for every request, decrement for every free
	uint32_t total_unexpected;   //tracking unexpected cases
	uint32_t total_merge_count; //count of merging continous free areas
	PHEAP_BLOCK head;
	PHEAP_BLOCK tail;	
}HEAP_CONTEXT;

static HEAP_CONTEXT g_heap_context;

//initially, the whole heap area is one block of free area
static void init_heap_context()
{
	if(NULL == g_heap_context.head)
	{
		PHEAP_BLOCK first_block = (PHEAP_BLOCK)g_heap;
		memset(first_block, 0, sizeof(HEAP_BLOCK));
		first_block->state = HEAP_BLOCK_FREE;
		first_block->start = (uint8_t*)first_block + sizeof(HEAP_BLOCK);
		first_block->size = (uint32_t)(sizeof(g_heap) - sizeof(HEAP_BLOCK));
		
		//populate the g_heap_context
		memset(&g_heap_context, 0, sizeof(g_heap_context));
		g_heap_context.heap_size = (uint64_t)sizeof(g_heap);
		g_heap_context.head = first_block;
		g_heap_context.tail = first_block;
	}
}

//merge the continous free blocks: multiple continous free block are replaced with 1 big free block in the list
static void merge_continuous_free_block()
{
	PHEAP_BLOCK free_block = NULL;            //first free block found
	PHEAP_BLOCK next_allocated_block = NULL;  //next allocated block after <free_block> in the list, 
	PHEAP_BLOCK temp = NULL;
	uint32_t total_size = 0;
	uint8_t  meet_merge_condition = 0;
	if(g_heap_context.head)
	{
		free_block = g_heap_context.head;
		while(free_block)
		{
			while(free_block && (HEAP_BLOCK_FREE != free_block.state))  //to find the free block
			{
				free_block = free_block.next;
			}
			
			if(free_block)  //found a free block, then to find next allocated block
			{
				next_allocated_block = free_block.next;
				while(next_allocated_block && (HEAP_BLOCK_ALLOCATED != next_allocated_block.state)) //to find the allocated block
				{
					next_allocated_block = next_allocated_block->next;
				}
				
				//merge condition: more than 1 free blocks (including <free_block>), multiple free blocks in the end of list
				//merge condition: more than 1 free blocks (including <free_block>) | allocated block
				meet_merge_condition = 0;
				if(NULL == next_allocated_block && NULL != free_block->next)
				{
					meet_merge_condition = 1;
				}
				if((NULL != next_allocated_block) && (next_allocated_block != free_block->next))  //not the merge condition
				{
					meet_merge_condition = 1;
				}
				
		        if(!meet_merge_condition)  //not the merge condition
				{
					free_block = next_allocated_block;
					continue;
				}
				else  //meet the merge condition: from <free_block> to revious of <next_allocated_block> (could be NULL) to merge into 1 big free block
				{
					temp = free_block;
					total_size = 0;
					
					//calc the total size
					while(temp->next != next_allocated_block)
					{
						total_size += (sizeof(HEAP_BLOCK) + temp->size);
						temp = temp->next;
					}
					
					free_block->size = total_size - sizeof(HEAP_BLOCK);
					free_block->next = next_allocated_block;
					//no changes to other fields
					memset(free_block->start, 0, free_block->size);
					
					if(NULL != next_allocated_block)
					{
						next_allocated_block->previous = free_block;						
					}
					else
					{
						g_heap_context.tail = free_block;
					}
					
					free_block = next_allocated_block;
					g_heap_context.total_merge_count++;					
				}				
			}
		}
	}
}

void * custom_calloc(uint32_t num, uint32_t size, const char* function_name, uint32_t line_num)
{
	void* ret_addr = NULL;
	uint32_t request_size = num * size;
	uint32_t temp_available_size = 0;
	uint32_t merge_count = 0;
	uint8_t  retried_flag = 0;
	PHEAP_BLOCK temp_block = NULL;
	PHEAP_BLOCK new_block = NULL;
	char* func_name = NULL;
	
	if(request_size <= 0 || request_size > sizeof(g_heap))
	{
		LOGD("heap input wrong size %d", request_size);
		return NULL;
	}
	
	if(NULL == g_heap_context.head)
	{
		init_heap_context();
	}
	
	func_name = (char*)function_name;
	if(NULL == func_name)
	{
		func_name = g_function_name;
	}
	
	LOGD("heap input size %d, %s:%d", request_size, func_name, line_num);
	
try_again:
     
	//search the first free block that is big enough
	temp_block = g_heap_context.head;
	while(temp_block)
	{
		if((HEAP_BLOCK_FREE == temp_block->state) && (temp_block->size >= request_size))
		{
			break;
		}
		else
		{
			temp_block = temp_block->next;
		}
	}
	
	//if not found, merge the continous free blocks: multiple continous free block are replaced with 1 big free block in the list, goto step 1 and try again
	if(NULL == temp_block)
	{
		if(0 == retried_flag)
		{
			retried_flag = 1;
			g_heap_context.total_unexpected++;
			merge_count = g_heap_context.total_merge_count;
			merge_continuous_free_block();
			if(merge_count != g_heap_context.total_merge_count)
			{
				goto try_again;	
			}
		}
	}
	else
	{
		//step 2.1: do the allocation: that is 1 free block changed into 1 allocated block and 1 free block(if more bytes left)
		//Step 2.2: inserted the 2 new blocks into the list to replace the original one
		//big enough but not too much, not split it
		if(temp_block->size < (request_size + sizeof(HEAP_BLOCK) + MIN_BLOCK_AVAILABLE_BUFF_SIZE))  //NOTE: this is to avoid too many small free blocks
		{
			ret_addr = temp_block->start;
			temp_block->state = HEAP_BLOCK_ALLOCATED;
			//temp_block->size not changed  //the whold block is allocated and no new free block generated
			temp_block->line_num = line_num;
			strncpy(temp_block->function_name, func_name, sizeof(temp_block->function_name));
		}
		else  //big enough and too much, split into 2
		{
			ret_addr = temp_block->start;
			temp_available_size = temp_block->size;   //split into request_size | new_block | available size of new_block
			
			//first block of the 2
			temp_block->state = HEAP_BLOCK_ALLOCATED;
			//temp_block->start   //not changed
			temp_block->size = request_size;
			temp_block->line_num = line_num;
			strncpy(temp_block->function_name, func_name, sizeof(temp_block->function_name));
			
			//second block of the 2
			new_block = (PHEAP_BLOCK)((uint8_t*)temp_block->start + temp_block->size);
			new_block->state = HEAP_BLOCK_FREE;
			new_block->start = (uint8_t*)new_block + sizeof(HEAP_BLOCK);
			new_block->size = temp_available_size - request_size - sizeof(HEAP_BLOCK);
			new_block->next = temp_block->next;
			new_block->previous = temp_block;
			
			temp_block->next = new_block;
			if(g_heap_context.tail == temp_block)
			{
				g_heap_context.tail = new_block;
			}
		}
		
		g_heap_context.total_alloc_count++;
		g_heap_context.total_alloc_size += temp_block->size;
		if(g_heap_context.total_alloc_size > g_heap_context.peak_size)
		{
			g_heap_context.peak_size = g_heap_context.total_alloc_size;
		}		
	}
	
	if(ret_addr)
	{
		print_heap_info();
	}
	else
	{
		LOGD("heap alloc failed");
	}
	
	return ret_addr;
}

void custom_free(void* ptr)
{
	PHEAP_BLOCK temp_block = NULL;
	if(ptr)
	{
		temp_block = g_heap_context.head;
		while(temp_block && (ptr != temp_block->start))
		{
			temp_block = temp_block->next;
		}
		
		if(NULL == temp_block)
		{
			g_heap_context.total_unexpected++;
			LOGD("heap address not found");
		}
		else
		{
			if(HEAP_BLOCK_ALLOCATED != temp_block->state)
			{
				g_heap_context.total_unexpected++;
			}
			
			LOGD("heap free node size %d, %s:%d", temp_block->size, temp_block->function_name, temp_block->line_num);
			temp_block->state = HEAP_BLOCK_FREE;
			memset(temp_block->function_name, 0, sizeof(temp_block->function_name));
			temp_block->line_num = 0;
			memset(temp_block->start, 0, temp_block->size);
			
			g_heap_context.total_alloc_count--;
			g_heap_context.total_alloc_size -= temp_block->size;
		}
	}
}

void print_heap_info()
{
	PHEAP_BLOCK temp_block = g_heap_context.head;
	while(NULL != temp_block)
	{
		if(HEAP_BLOCK_ALLOCATED == temp_block->state)
		{
			LOGD("heap alloc node size %d, %s:%d", temp_block->size, temp_block->function_name, temp_block->line_num);
		}
		else
		{
			LOGD("heap free node size %d", temp_block->size);
		}
		
		temp_block = temp_block->next;
	}
	
	LOGD("heap count %d, size %lld, peak %lld, fail %d, merge %d", g_heap_context.total_alloc_count, 
	g_heap_context.total_alloc_size, g_heap_context.peak_size, g_heap_context.total_unexpected, g_heap_context.total_merge_count);	
}

#endif

#if defined(USE_SYS_HEAP_TRACK)  //track the system heap usage information
//just track the function_name:line_number which requests buffer from heap
//track when it's freed
//then when know if there is any heap memory leak

#define MAX_FUNCTION_NAME_LEN        (32)

typedef struct _HEAP_BLOCK_ HEAP_BLOCK, *PHEAP_BLOCK;
typedef struct _HEAP_BLOCK_
{
	void*            start;       //system allocated buffer address
	uint32_t         size;        //request size from callers
	uint32_t         line_num;
	char             function_name[MAX_FUNCTION_NAME_LEN];   //NOTE: trunc it if too long
	PHEAP_BLOCK      next;
}HEAP_BLOCK, *PHEAP_BLOCK; 

typedef struct _HEAP_CONTEXT_
{
	uint64_t total_alloc_size;   //current allocated size from callers
	uint32_t total_alloc_count;  //increment for every request, decrement for every free
	uint32_t total_unexpected;   //tracking unexpected cases
	PHEAP_BLOCK head;
	PHEAP_BLOCK tail;	
}HEAP_CONTEXT;

static HEAP_CONTEXT g_heap_context;


void * custom_calloc(uint32_t num, uint32_t size, const char* function_name, uint32_t line_num)
{
	void* ret_addr = calloc(num * size);
	
	LOGD("heap input size %d, %s:%d", num * size, function_name, line_num);
	
	if(ret_addr)
	{
		PHEAP_BLOCK temp_block = (PHEAP_BLOCK)calloc(1, sizeof(HEAP_BLOCK));
		if(temp_block)
		{
			temp_block->start = ret_addr;
			temp_block->size = num * size;
			temp_block->line_num = line_num;
			strncpy(temp_block->function_name, func_name, sizeof(temp_block->function_name));
			
			g_heap_context.total_alloc_count++;
			g_heap_context.total_alloc_size += temp_block->size;
			
			if(NULL == g_heap_context.head)
			{
				g_heap_context.head = temp_block;
				g_heap_context.tail = temp_block;
			}
			else
			{
				g_heap_context.tail->next = temp_block;
				g_heap_context.tail = temp_block;
			}
		}
		else
		{
			g_heap_context.total_unexpected++;
		}
	}
	else
	{
		g_heap_context.total_unexpected++;
	}
	
	
	if(ret_addr)
	{
		print_heap_info();
	}
	else
	{
		LOGD("heap alloc failed");
	}
	
	return ret_addr;
}

void custom_free(void* ptr)
{
	PHEAP_BLOCK temp_block = NULL;
	PHEAP_BLOCK previous_block = NULL;
	if(ptr)
	{
		temp_block = g_heap_context.head;
		previous_block = NULL;
		while(temp_block && (ptr != temp_block->start))
		{
			previous_block = temp_block;
			temp_block = temp_block->next;
		}
		
		if(NULL == temp_block)
		{
			g_heap_context.total_unexpected++;
			LOGD("heap address not found");
		}
		else
		{			
			LOGD("heap free node size %d, %s:%d", temp_block->size, temp_block->function_name, temp_block->line_num);
			if(g_heap_context.head != temp_block && g_heap_context.tail != temp_block )
			{
				previous_block->next = temp_block->next;
			}
			else  //found block could be head or tail or head and tail(only 1 block)
			{		  
				if(g_heap_context.head == temp_block)
				{
					g_heap_context.head = temp_block->next;				
				}
				
				if(g_heap_context.tail == temp_block)
				{
					g_heap_context.tail = previous_block;
					if(NULL != previous_block)
					{
						previous_block->next = NULL;
					}
				}
			}
			free(temp_block);
			
			g_heap_context.total_alloc_count--;
			g_heap_context.total_alloc_size -= temp_block->size;
		}
		
		free(ptr);
	}
}

void print_heap_info()
{
	PHEAP_BLOCK temp_block = g_heap_context.head;
	while(NULL != temp_block)
	{
		LOGD("heap alloc node size %d, %s:%d", temp_block->size, temp_block->function_name, temp_block->line_num);
		
		temp_block = temp_block->next;
	}
	
	LOGD("heap count %d, size %lld, fail %d", g_heap_context.total_alloc_count, g_heap_context.total_alloc_size, g_heap_context.total_unexpected);	
}
#endif

调试过程中的问题总结：
1. 使用自定义堆，会直接增大编译生成的二进制文件大小；可以减少编译脚本（链接文件）中的HEAP_SIZE，有时可能要权衡一下。问题来了，这个HEAP_SZIE可以减少到0吗？肯定是不行的，虽然我们代码不用缺省堆，但调用我们代码的，和我们代码调用的，对我们来说都是黑盒子，能保证他们都不用缺省堆吗？保险起见，可以先使用缺省值吧。
2. 那么这个全局区域最大设为多少呢？可以根据peak_size统计进行设置，甚至也可以实现方案略加修改，变为自定义堆+缺省堆的方案。
3. 低级错误1：日志打印的%s和%d与实际的参数不对应导致的问题。由此可见，本地搭建完整的测试环境的重要性。
4. 低级错误2：最初内存分配custom_calloc的入参num/size为uint16_t，程序实际使用过程中会溢出。由此可见，测试驱动开发，测试用例设计的重要性。
5. 低级错误3：custom_calloc中最初实现的查找“第一个未分配且buff足够大的HEAP_BLOCK”的while循环的条件没有指定正确。这类条件设定，逻辑一定要越简单越清晰越好。

那最终问题解决了吗？解决了，不是内存泄漏，不是一个问题。

【2022年5.1节重庆】