malloc内存分配
文章目录
1. 内存分配策略由来
操作数据分配，在堆内存进行分配（因为栈存着私有，数据段存全局，未初始化数据），那既然涉及到分配内存的使用，通过汇编角度的esp指针移动进行内存分配，无外乎就可同理可得，分配内存方式也可以通过指针分配，除了此方式，为了加快速度，也可推理得出可以采用mmap映射
-
brk/sbrk指针：通过在堆内存用指针往高地址移动分配 -
mmap映射：通过在堆里虚拟映射（非主分配区）激活mmap映射的阈值为128byte
2. 代码结果验证
2.1 代码示例
这里我用如下代码进行debug调试，来验证sbrk，mmap
这里分别写了3个malloc的内存分配，为什么数字还不一样，128不是mmap的分配阈值么？怎么还用sbrk指针
先保留点好奇，后面会解答
#include <stdio.h>
#include <stdlib.h>
int main(){
void *p = malloc(0); // sbrk
void *a = malloc(128 * 1024); // sbrk
void *q = malloc(132 * 1024); // mmap
printf("%p\n", p);
printf("%p\n", a);
printf("%p\n", q);
return 1;
}
2.2 验证内存分配2种方式
sbrk指针分配的内存属于层级递增，而mmap属于在堆空间的高地址分配映射。
通过下面代码进行观察输出的地址信息
#include <stdio.h>
#include <stdlib.h>
int main(){
void *p = malloc(0);
void *q = malloc(128 * 1024);
void *y = malloc(132 * 1024);
printf("%p\n", p);
printf("%p\n", q);
printf("%p\n", y);
return 1;
}
=======结果=======
0x1c1c010 // sbrk
0x1c1c030 // sbrk递增分配
0x7f5f4066a010 // mmap映射
3. 源码讲解
3.1 前置预备
由于博主是Java Coder，这里只讲分配内存的直接关联代码，其余业务代码不会进行干涉。
建议小伙伴自行下载sourceinsight新建glibc2.19的源码项目，方便对照eclipsedebug代码进行学习，如果对source insight导入源码，可看博主这篇文章Source Insight 读取源码使用入门
source insight如图
3.2 132kb的sbrk指针开辟的由来
⭐️本源码围绕[2.代码](# 2. 代码示例)进行debug讲解。机器是64位机，size_t对应是64位，8byte
从第一个malloc进行断点，进去malloc方法
这一小节围绕，malloc(0)代码debug
🚀1. 进行入参的大小转化成后续开拓内存的基石
将我们的入参0,转化为内存要求的大小，也就是下面代码checked_request2size(bytes, nb)
这里相关checked_request2size(bytes, nb)的具体实现，我写在了涉及源码调用api部分的下面，之后的代码也是同样如此，形式就如下
主流程源码
== == 涉及源码调用api部分 == ==
调用的api的具体实现
// 将请求大小转换为内部形式加上结构体的大小，获取校准和，以此获取扩展内存的大小的参数之一
// 这里进行nb的赋值，byte为传进来的值，目前第一个malloc传的是0
// 通过下方源码算出来，nb = 32byte
checked_request2size (bytes, nb);
// ===================涉及源码调用api部分=========================
// ===================如何计算的==========================
// checked_request2size源码
#define checked_request2size(req, sz)
// 判断是否超出范围，超出结束进程，并添加错误信息
if (REQUEST_OUT_OF_RANGE (req)) {
__set_errno (ENOMEM);
return 0;
}
// 计算nb
(sz) = request2size (req);
// request2size源码
#define request2size(req)
// (0 + 8 + 15 < 32) ? 32 : ((8 + 15) & ~15) = 16
(((req) + SIZE_SZ + MALLOC_ALIGN_MASK < MINSIZE) ?
MINSIZE :
((req) + SIZE_SZ + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK)
// SIZE_SZ源码
#ifndef INTERNAL_SIZE_T
#define INTERNAL_SIZE_T size_t
#endif
/* The corresponding word size */
// 机器位数大小64,所以是8
#define SIZE_SZ (sizeof(INTERNAL_SIZE_T))
// MALLOC_ALIGN_MASK源码
// 2 * 8 = 16
#define MALLOC_ALIGNMENT (2 *SIZE_SZ)
// 2 * 机器位数 -1 = 16 - 1 = 15
#define MALLOC_ALIGN_MASK (MALLOC_ALIGNMENT - 1)
// MINSIZE源码
#define MINSIZE
// (32 + 15)& ~15 = 32
(unsigned long)(((MIN_CHUNK_SIZE+MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK))
// MIN_CHUNK_SIZE
// 获取前4个字段的占用内存大小，4 * 8 = 32 byte
#define MIN_CHUNK_SIZE (offsetof(struct malloc_chunk, fd_nextsize))
# define offsetof(type,ident) ((size_t)&(((type*)0)->ident))
// malloc_chunk 结构体
struct malloc_chunk {
INTERNAL_SIZE_T prev_size;
INTERNAL_SIZE_T size;
struct malloc_chunk* fd;
struct malloc_chunk* bk;
/* Only used for large blocks: pointer to next larger size. */
struct malloc_chunk* fd_nextsize;
struct malloc_chunk* bk_nextsize;
};
这里涉及到了一个内存分配的结构体，这个结构体[4.1 malloc_chunk](# 4.1 malloc_chunk)，内存分配的结构体，可点击跳转进行知识补充，这个结构体贯穿着所有内存分配的流程，也就是主骨干。这里不做篇幅，可以点击并观看相关知识。
🚀开拓内存大小
// 32 + 131072 + 32 = 131136
/* Request enough space for nb + pad + overhead */
size = nb + mp_.top_pad + MINSIZE;
// 是用来将一个大小（size）向上舍入到最近的页面边界
// (131136 + 4095) & ~4095 = 135168 = 132kb
size = (size + pagemask) & ~pagemask;
// ================== 涉及源码调用api部分 ================
// 这一行代码是在 glibc（GNU C Library）中用来计算一个掩码（mask），用于确定虚拟地址是否对齐到页面大小
// dl_pagesize 表示系统的页面大小
// 64位页大小是4kb，4096 -1 = 4095
size_t pagemask = GLRO(dl_pagesize) - 1;
🚀分配内存
进行获取brk指针移动后的指针地址
到最后进行
malloc_chunk里的size设置开销的大小
// brk地址
brk = (char *) (MORECORE (size));
// brk对齐后的指针地址赋予top属性
av->top = (mchunkptr) aligned_brk;
/* finally, do the allocation */
p = av->top;
size = chunksize (p);
// =============涉及源码调用api部分==============
#define SIZE_BITS (PREV_INUSE | IS_MMAPPED | NON_MAIN_ARENA)
/* Get size, ignoring use bits */
#define chunksize(p) ((p)->size & ~(SIZE_BITS))
#define PREV_INUSE 0x1
#define IS_MMAPPED 0x2
#define NON_MAIN_ARENA 0x4
🐯MORECORE：sbrk指针开辟
morecore.c - 45 line
extern void *__sbrk (ptrdiff_t increment) __THROW;
/* Allocate INCREMENT more bytes of data space,
and return the start of data space, or NULL on errors.
If INCREMENT is negative, shrink data space. */
void *
__default_morecore (ptrdiff_t increment)
{
void *result = (void *) __sbrk (increment);
if (result == (void *) -1)
return NULL;
return result;
}
sbrk底层开辟是调用os级别的函数api
linux v2.6 - mmap.c(mm)，通过搜sys_brk
进行
do_brk开辟
asmlinkage unsigned long sys_brk(unsigned long brk)
{
// 前置代码省略
/* Ok, looks good - let it rip. */
if (do_brk(oldbrk, newbrk-oldbrk) != oldbrk)
goto out;
set_brk:
mm->brk = brk;
out:
retval = mm->brk;
up_write(&mm->mmap_sem);
return retval;
}
do_brk
进行brk指针开辟堆内存
unsigned long do_brk(unsigned long addr, unsigned long len)
{
struct mm_struct * mm = current->mm;
struct vm_area_struct * vma, * prev;
unsigned long flags;
struct rb_node ** rb_link, * rb_parent;
len = PAGE_ALIGN(len);
if (!len)
return addr;
if ((addr + len) > TASK_SIZE || (addr + len) < addr)
return -EINVAL;
/*
* mlock MCL_FUTURE?
*/
if (mm->def_flags & VM_LOCKED) {
unsigned long locked = mm->locked_vm << PAGE_SHIFT;
locked += len;
if (locked > current->rlim[RLIMIT_MEMLOCK].rlim_cur)
return -EAGAIN;
}
/*
* Clear old maps. this also does some error checking for us
*/
munmap_back:
vma = find_vma_prepare(mm, addr, &prev, &rb_link, &rb_parent);
if (vma && vma->vm_start < addr + len) {
if (do_munmap(mm, addr, len))
return -ENOMEM;
goto munmap_back;
}
/* Check against address space limits *after* clearing old maps... */
if ((mm->total_vm << PAGE_SHIFT) + len
> current->rlim[RLIMIT_AS].rlim_cur)
return -ENOMEM;
if (mm->map_count > MAX_MAP_COUNT)
return -ENOMEM;
if (security_vm_enough_memory(len >> PAGE_SHIFT))
return -ENOMEM;
flags = VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
/* Can we just expand an old anonymous mapping? */
if (rb_parent && vma_merge(mm, prev, rb_parent, addr, addr + len,
flags, NULL, 0))
goto out;
/*
* create a vma struct for an anonymous mapping
*/
vma = kmem_cache_alloc(vm_area_cachep, SLAB_KERNEL);
if (!vma) {
vm_unacct_memory(len >> PAGE_SHIFT);
return -ENOMEM;
}
// vma 虚拟地址开辟
vma->vm_mm = mm;
vma->vm_start = addr;
vma->vm_end = addr + len;
vma->vm_flags = flags;
vma->vm_page_prot = protection_map[flags & 0x0f];
vma->vm_ops = NULL;
vma->vm_pgoff = 0;
vma->vm_file = NULL;
vma->vm_private_data = NULL;
INIT_LIST_HEAD(&vma->shared);
vma_link(mm, vma, prev, rb_link, rb_parent);
out:
mm->total_vm += len >> PAGE_SHIFT;
if (flags & VM_LOCKED) {
mm->locked_vm += len >> PAGE_SHIFT;
make_pages_present(addr, addr + len);
}
return addr;
}
3.3 malloc(128 * 1024)再次brk偏移讲解
这一小节围绕，malloc(128 * 1024)代码debug
判断主要来源于：
(unsigned long) (size) >= (unsigned long) (nb + MINSIZE)这里会进行之前的size分配，和当前要分配+
malloc_chunk分配的最小大小做比较
size：132 * 1024 - 32byte （已经分配的malloc(0)的结构体）= 135136 byte
nb：128 * 1024 = 131072 byte
MINSIZE：32byte最后也就是判断
135136 > 131072 - 32结构体是true
static void *
_int_malloc (mstate av, size_t bytes)
{
// 省略前置代码
use_top:
/*
If large enough, split off the chunk bordering the end of memory
(held in av->top). Note that this is in accord with the best-fit
search rule. In effect, av->top is treated as larger (and thus
less well fitting) than any other available chunk since it can
be extended to be as large as necessary (up to system
limitations).
We require that av->top always exists (i.e., has size >=
MINSIZE) after initialization, so if it would otherwise be
exhausted by current request, it is replenished. (The main
reason for ensuring it exists is that we may need MINSIZE space
to put in fenceposts in sysmalloc.)
*/
victim = av->top;
size = chunksize (victim);
/* check that one of the above allocation paths succeeded */
if ((unsigned long) (size) >= (unsigned long) (nb + MINSIZE))
{
remainder_size = size - nb;
remainder = chunk_at_offset (victim, nb);
av->top = remainder;
set_head (victim, nb | PREV_INUSE |
(av != &main_arena ? NON_MAIN_ARENA : 0));
set_head (remainder, remainder_size | PREV_INUSE);
check_malloced_chunk (av, victim, nb);
void *p = chunk2mem (victim);
alloc_perturb (p, bytes);
return p;
}
// 省略后续代码
}
chunk2men
顺便看下
chunk2men的宏定义，2 * SIZE_SZ也就是malloc_chunk里的前两个字段把新的
malloc_chunk里的元数据信息字段偏移掉，也就是返回malloc_chunk要存储的数据的指针地址
#define chunk2mem(p) ((void*)((char*)(p) + 2*SIZE_SZ))
3.4 mmap分配
这一小节围绕，malloc(132* 1024)代码
这里就简单讲解下mmap，毕竟博主不是C Coder，主要目的是为了证明内存有2种的分配策略
从上面ebrk分配不满足内存比较时，会走进sysmalloc函数，进行mmap判断并处理
static void *
sysmalloc (INTERNAL_SIZE_T nb, mstate av)
{
// 当判断完，就可比较出需要走mmap
// nb：132
// mmap_threshold:128
// n_mmaps:0
// n_mmaps_max: 65536
if ((unsigned long) (nb) >= (unsigned long) (mp_.mmap_threshold) &&
(mp_.n_mmaps < mp_.n_mmaps_max))
{
char *mm; /* return value from mmap call*/
try_mmap:
// 代码省略
}
// 代码省略
}
4. 补充小知识
4.1 malloc_chunk
在这里源码调用的具体实现有一个MIN_CHUNK_SIZE的计算，我需要着重讲下其中涉及的结构体malloc_chunk
#define MIN_CHUNK_SIZE (offsetof(struct malloc_chunk, fd_nextsize))
malloc_chunk
我这里把源码的注释粘贴过来
这个
chunk除了保存数据信息，还保存了几部分的元数据信息。这里以我们用的为主来说，剩下的源码注释大家自行观看即可。
🐯malloc_chunk的结构体属性结构
{上一块大小}{当前块大小}{fd}{bk}
fd指向下一个空闲chunk
bk指向前一个空闲chunk🐯
malloc_chunk后面2个字段是对于大块用到🐯
malloc_chunk结构体的内存分配因为我是64位机，所以需要8 * 4= 32byte
这个32byte起到了很大的作用，因为如果你后续开辟小于132的，会先减去已经使用的，在减去32byte，得到剩余空间，来进行比较是否符合，从而进行sbrk指针的移动分配地址
/*
malloc_chunk details:
(The following includes lightly edited explanations by Colin Plumb.)
Chunks of memory are maintained using a `boundary tag' method as
described in e.g., Knuth or Standish. (See the paper by Paul
Wilson ftp://ftp.cs.utexas.edu/pub/garbage/allocsrv.ps for a
survey of such techniques.) Sizes of free chunks are stored both
in the front of each chunk and at the end. This makes
consolidating fragmented chunks into bigger chunks very fast. The
size fields also hold bits representing whether chunks are free or
in use.
An allocated chunk looks like this:
chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Size of previous chunk, if allocated | |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Size of chunk, in bytes |M|P|
mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| User data starts here... .
. .
. (malloc_usable_size() bytes) .
. |
nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Size of chunk |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
Where "chunk" is the front of the chunk for the purpose of most of
the malloc code, but "mem" is the pointer that is returned to the
user. "Nextchunk" is the beginning of the next contiguous chunk.
Chunks always begin on even word boundaries, so the mem portion
(which is returned to the user) is also on an even word boundary, and
thus at least double-word aligned.
Free chunks are stored in circular doubly-linked lists, and look like this:
chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Size of previous chunk |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
`head:' | Size of chunk, in bytes |P|
mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Forward pointer to next chunk in list |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Back pointer to previous chunk in list |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Unused space (may be 0 bytes long) .
. .
. |
nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
`foot:' | Size of chunk, in bytes |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
The P (PREV_INUSE) bit, stored in the unused low-order bit of the
chunk size (which is always a multiple of two words), is an in-use
bit for the *previous* chunk. If that bit is *clear*, then the
word before the current chunk size contains the previous chunk
size, and can be used to find the front of the previous chunk.
The very first chunk allocated always has this bit set,
preventing access to non-existent (or non-owned) memory. If
prev_inuse is set for any given chunk, then you CANNOT determine
the size of the previous chunk, and might even get a memory
addressing fault when trying to do so.
Note that the `foot' of the current chunk is actually represented
as the prev_size of the NEXT chunk. This makes it easier to
deal with alignments etc but can be very confusing when trying
to extend or adapt this code.
The two exceptions to all this are
1. The special chunk `top' doesn't bother using the
trailing size field since there is no next contiguous chunk
that would have to index off it. After initialization, `top'
is forced to always exist. If it would become less than
MINSIZE bytes long, it is replenished.
2. Chunks allocated via mmap, which have the second-lowest-order
bit M (IS_MMAPPED) set in their size fields. Because they are
allocated one-by-one, each must contain its own trailing size field.
*/
struct malloc_chunk {
INTERNAL_SIZE_T prev_size; /* Size of previous chunk (if free). */
INTERNAL_SIZE_T size; /* Size in bytes, including overhead. */
struct malloc_chunk* fd; /* double links -- used only if free. */
struct malloc_chunk* bk;
/* Only used for large blocks: pointer to next larger size. */
struct malloc_chunk* fd_nextsize; /* double links -- used only if free. */
struct malloc_chunk* bk_nextsize;
};
4.2 elf头部地址和程序虚拟地址
⭐️通过pmap读取debug的程序pid，进行观看虚拟地址映射
pmap -x pid：查看程序的虚拟内存分配信息
readelf -l elf文件：查看elf文件程序头信息
可看到elf文件有2个load，分别对应text segment，data segment
下面这张图取elf可执行文件
而数据段往上就是堆内存，可看到sbrk移动了132kb
4.3 程序的虚拟地址的映射信息在哪看？
通过上面命令
pmap -x 6538查看程序内存映射会有601000相关的数据段映射的虚拟地址然后通过
sudo strace -f pmap -x 6538查看pmap的链路，查看到调用了stat(/proc/6538)于是打开这个目录，查看
maps文件对应的映射信息ps：proc是一个虚拟文件系统，保存了所有进程的所有信息
proc包含：
- 内存映射信息
- 系统调用信息
- 栈信息
- cpu信息
- io信息
- fd信息
- …
🚀额外小补充
里面还有fd(file descriptor 文件描述符)：你可以找到与特定进程相关的所有打开文件的符号链接。文件描述符是一个抽象的指示器（通常是正整数），用于指代进程所打开的文件或其它输入/输出资源（如网络套接字、管道等）
linux 0.11 - main.c
void init(void)
{
int pid,i;
setup((void *) &drive_info);
(void) open("/dev/tty0",O_RDWR,0); // 0 标准输入流
(void) dup(0); // 1 标准输出流
(void) dup(0); // 2 标准错误输出流
// 后续代码省略
}
扫一扫
1227
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
