eBPF内存泄露检测代码实现v1

eBPF内存泄露检测代码实现v1

视频讲解：

eBPF内存泄露检测代码实现 <一>

说明：

eBPF内存泄露检测代码实现是在 libbpf-bootstrap 框架下开发，需要的基础知识请参考之前的ebpf系列视频

本节视频使用的是 Ubuntu18.04 x86-64 平台

目标

这节视频的目标是用ebpf代码实现内存泄露检测工具的第一个简单版本，只检测 malloc 和 free，并打印如下的内存泄露堆栈；（指令地址对应符号名，文件名，行号的解析放到下节视频中讲解）

stack_id=0x2d81 with outstanding allocation: total_size=20 nr_alloc=5
[  0] 0x55e8fbd196f2
[  1] 0x55e8fbd19711
[  2] 0x55e8fbd19730
[  3] 0x55e8fbd19770
[  4] 0x7f5bb944ec87
[  5] 0x5f6258d4c544155

代码实现参考开源项目：

https://github.com/eunomia-bpf/bpf-developer-tutorial

​ bpf-developer-tutorial/src/16-memleak

测试代码

test_memleak.c

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

static void * alloc_v3(int alloc_size)
{
    void * ptr = malloc(alloc_size);
    
    return ptr;
}

static void * alloc_v2(int alloc_size)
{
    void * ptr = alloc_v3(alloc_size);

    return ptr;
}

static void * alloc_v1(int alloc_size)
{
    void * ptr = alloc_v2(alloc_size);

    return ptr;
}

int main(int argc, char * argv[])
{
    const int alloc_size = 4;
    void * ptr = NULL;
    int i = 0;

    for (i = 0; ; i++)
    {
        ptr = alloc_v1(alloc_size);

        sleep(2);

        if (0 == i % 2)
        {
            free(ptr);
        }
    }

    return 0;
}

编译：

gcc -g test_memleak.c -o test_memleak

内存泄露检测工具的第一个版本

下面的memleak.h, memleak.bpf.c, memleak.c 文件需要放到开源代码 libbpf-bootstrap 中的 examples/c/ 目录下编译；
并在 libbpf-bootstrap/examples/c/Makefile 中的 APPS 加上 memleak

memleak.h

#ifndef __MEMLEAK_H
#define __MEMLEAK_H
 
#define ALLOCS_MAX_ENTRIES 1000000
#define COMBINED_ALLOCS_MAX_ENTRIES 10240
 
struct alloc_info {
    __u64 size;
    int stack_id;
};
 
/* 为了节省内存和方便整形数据的原子操作,把 combined_alloc_info 定义为联合体
 * 其中 total_size 占 40bit, number_of_allocs 占 24bit, 联合体总大小为 64bit
 * 2个combined_alloc_info联合体的 bits 字段相加, 相当于对应的 total_size 相加, 
 * 和对应的 number_of_allocs 相加;
 */
union combined_alloc_info {
    struct {
        __u64 total_size : 40;
        __u64 number_of_allocs : 24;
    };
    __u64 bits;
};
 
#endif /* __MEMLEAK_H */

memleak.bpf.c

// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
/* Copyright (c) 2020 Facebook */
#include "vmlinux.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include "memleak.h"
 
#define KERN_STACKID_FLAGS (0 | BPF_F_FAST_STACK_CMP)
#define USER_STACKID_FLAGS (0 | BPF_F_FAST_STACK_CMP | BPF_F_USER_STACK)
 
struct {
    __uint(type, BPF_MAP_TYPE_HASH);
    __type(key, pid_t); // pid
    __type(value, u64); // size for alloc
    __uint(max_entries, 10240);
} sizes SEC(".maps");
 
struct {
    __uint(type, BPF_MAP_TYPE_HASH);
    __type(key, u64); /* alloc return address */
    __type(value, struct alloc_info);
    __uint(max_entries, ALLOCS_MAX_ENTRIES);
} allocs SEC(".maps");
 
struct {
    __uint(type, BPF_MAP_TYPE_HASH);
    __type(key, u64); /* stack id */
    __type(value, union combined_alloc_info);
    __uint(max_entries, COMBINED_ALLOCS_MAX_ENTRIES);
} combined_allocs SEC(".maps");
 
/* value： stack id 对应的堆栈的深度
 * max_entries: 最大允许存储多少个stack_id（每个stack id都对应一个完整的堆栈）
 * 这2个值可以根据应用层的使用场景,在应用层的ebpf中open之后load之前动态设置
 */
struct {
    __uint(type, BPF_MAP_TYPE_STACK_TRACE);
    __type(key, u32); /* stack id */
    //__type(value, xxx);       memleak_bpf__open 之后再动态设置
    //__uint(max_entries, xxx); memleak_bpf__open 之后再动态设置
} stack_traces SEC(".maps");
 
char LICENSE[] SEC("license") = "Dual BSD/GPL";
 
SEC("uprobe")
int BPF_KPROBE(malloc_enter, size_t size)
{
    const pid_t pid = bpf_get_current_pid_tgid() >> 32;
 
    bpf_map_update_elem(&sizes, &pid, &size, BPF_ANY);
 
    // bpf_printk("malloc_enter size=%d\n", size);
    return 0;
}
 
SEC("uretprobe")
int BPF_KRETPROBE(malloc_exit, void * address)
{
    const u64 addr = (u64)address;
    const pid_t pid = bpf_get_current_pid_tgid() >> 32;
    struct alloc_info info;
 
    const u64 * size = bpf_map_lookup_elem(&sizes, &pid);
    if (NULL == size) {
        return 0;
    }
 
    __builtin_memset(&info, 0, sizeof(info));
    info.size = *size;
 
    bpf_map_delete_elem(&sizes, &pid);
 
    if (NULL != address) {
        info.stack_id = bpf_get_stackid(ctx, &stack_traces, USER_STACKID_FLAGS);
 
        bpf_map_update_elem(&allocs, &addr, &info, BPF_ANY);
 
        union combined_alloc_info add_cinfo = {
            .total_size = info.size,
            .number_of_allocs = 1
        };
 
        union combined_alloc_info * exist_cinfo = bpf_map_lookup_elem(&combined_allocs, &info.stack_id);
        if (NULL == exist_cinfo) {
            bpf_map_update_elem(&combined_allocs, &info.stack_id, &add_cinfo, BPF_NOEXIST);
        }
        else {
            __sync_fetch_and_add(&exist_cinfo->bits, add_cinfo.bits);
        }
    }
 
    // bpf_printk("malloc_exit address=%p\n", address);
    return 0;
}
 
SEC("uprobe")
int BPF_KPROBE(free_enter, void * address)
{
    const u64 addr = (u64)address;
 
    const struct alloc_info * info = bpf_map_lookup_elem(&allocs, &addr);
    if (NULL == info) {
        return 0;
    }
 
    union combined_alloc_info * exist_cinfo = bpf_map_lookup_elem(&combined_allocs, &info->stack_id);
    if (NULL == exist_cinfo) {
        return 0;
    }
 
    const union combined_alloc_info sub_cinfo = {
        .total_size = info->size,
        .number_of_allocs = 1
    };
 
    __sync_fetch_and_sub(&exist_cinfo->bits, sub_cinfo.bits);
 
    bpf_map_delete_elem(&allocs, &addr);
 
    // bpf_printk("free_enter address=%p\n", address);
    return 0;
}

memleak.c

// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
/* Copyright (c) 2020 Facebook */
#include <errno.h>
#include <stdio.h>
#include <unistd.h>
#include <sys/resource.h>
#include <bpf/libbpf.h>
#include "memleak.skel.h"
#include "memleak.h"
 
static const int perf_max_stack_depth = 127;    //stack id 对应的堆栈的深度
static const int stack_map_max_entries = 10240; //最大允许存储多少个stack_id（每个stack id都对应一个完整的堆栈）
static __u64 * g_stacks = NULL;
static size_t g_stacks_size = 0;
static const char * p_print_file = "/tmp/memleak_print";
static const char * p_quit_file = "/tmp/memleak_quit";
 
static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
{
    return vfprintf(stderr, format, args);
}
 
int print_outstanding_combined_allocs(struct memleak_bpf * skel)
{
    const size_t combined_allocs_key_size = bpf_map__key_size(skel->maps.combined_allocs);
    const size_t stack_traces_key_size = bpf_map__key_size(skel->maps.stack_traces);
 
    for (__u64 prev_key = 0, curr_key = 0; ; prev_key = curr_key) {
 
        if (bpf_map__get_next_key(skel->maps.combined_allocs, 
            &prev_key, &curr_key, combined_allocs_key_size)) {
            if (errno == ENOENT) {
                break; //no more keys, done!
            }
            perror("map get next key failed!");
 
            return -errno;
        }
 
        // stack_id = curr_key
        union combined_alloc_info cinfo;
        memset(&cinfo, 0, sizeof(cinfo));
 
        if (bpf_map__lookup_elem(skel->maps.combined_allocs, 
            &curr_key, combined_allocs_key_size, &cinfo, sizeof(cinfo), 0)) {
            if (errno == ENOENT) {
                continue;
            }
 
            perror("map lookup failed!");
            return -errno;
        }
 
        if (bpf_map__lookup_elem(skel->maps.stack_traces, 
            &curr_key, stack_traces_key_size, g_stacks, g_stacks_size, 0)) {
            perror("failed to lookup stack traces!");
            return -errno;
        }
 
        printf("stack_id=0x%llx with outstanding allocations: total_size=%llu nr_allocs=%llu\n",
            curr_key, (__u64)cinfo.total_size, (__u64)cinfo.number_of_allocs);
         
        for (int i = 0; i < perf_max_stack_depth; i++) {
            if (0 == g_stacks[i]) {
                break;
            }
 
            printf("[%3d] 0x%llx\n", i, g_stacks[i]);
        }
 
    }
 
    return 0;
}
 
int main(int argc, char **argv)
{
    struct memleak_bpf *skel;
    int err, i;
    LIBBPF_OPTS(bpf_uprobe_opts, uprobe_opts);
    int attach_pid;
    char binary_path[128] = {0};
 
    if (2 != argc)
    {
        printf("usage:%s attach_pid\n", argv[0]);
        return -1;
    }
 
    attach_pid = atoi(argv[1]);
    strcpy(binary_path, "/lib/x86_64-linux-gnu/libc.so.6");
 
    /* Set up libbpf errors and debug info callback */
    libbpf_set_print(libbpf_print_fn);
 
    /* Load and verify BPF application */
    skel = memleak_bpf__open();
    if (!skel) {
        fprintf(stderr, "Failed to open BPF skeleton\n");
        return 1;
    }
 
    bpf_map__set_value_size(skel->maps.stack_traces, perf_max_stack_depth * sizeof(__u64));
    bpf_map__set_max_entries(skel->maps.stack_traces, stack_map_max_entries);
 
    err = memleak_bpf__load(skel);
    if (err) {
        fprintf(stderr, "Failed to load BPF skeleton\n");
        goto cleanup;
    }
 
    uprobe_opts.func_name = "malloc";
    uprobe_opts.retprobe = false;
    skel->links.malloc_enter = bpf_program__attach_uprobe_opts(skel->progs.malloc_enter,
                                 attach_pid, binary_path,
                                 0,
                                 &uprobe_opts);
    if (!skel->links.malloc_enter) {
        err = -errno;
        fprintf(stderr, "Failed to attach uprobe: %d\n", err);
        goto cleanup;
    }
 
    uprobe_opts.func_name = "malloc";
    uprobe_opts.retprobe = true;
    skel->links.malloc_exit = bpf_program__attach_uprobe_opts(
        skel->progs.malloc_exit, attach_pid, binary_path,
        0, &uprobe_opts);
    if (!skel->links.malloc_exit) {
        err = -errno;
        fprintf(stderr, "Failed to attach uprobe: %d\n", err);
        goto cleanup;
    }
 
    uprobe_opts.func_name = "free";
    uprobe_opts.retprobe = false;
    skel->links.free_enter = bpf_program__attach_uprobe_opts(skel->progs.free_enter,
                                 attach_pid, binary_path,
                                 0,
                                 &uprobe_opts);
    if (!skel->links.free_enter) {
        err = -errno;
        fprintf(stderr, "Failed to attach uprobe: %d\n", err);
        goto cleanup;
    }
 
    /* Let libbpf perform auto-attach for uprobe_sub/uretprobe_sub
     * NOTICE: we provide path and symbol info in SEC for BPF programs
     */
    err = memleak_bpf__attach(skel);
    if (err) {
        fprintf(stderr, "Failed to auto-attach BPF skeleton: %d\n", err);
        goto cleanup;
    }
 
    g_stacks_size = perf_max_stack_depth * sizeof(*g_stacks);
    g_stacks = (__u64 *)malloc(g_stacks_size);
    memset(g_stacks, 0, g_stacks_size);
 
    // printf("Successfully started! Please run `sudo cat /sys/kernel/debug/tracing/trace_pipe` "
    //        "to see output of the BPF programs.\n");
 
    for (i = 0;; i++) {
        if (0 == access(p_quit_file, F_OK)) {
            remove(p_quit_file);
            break;
        }
        else if (0 == access(p_print_file, F_OK)) {
            remove(p_print_file);
            print_outstanding_combined_allocs(skel);
        }
 
        usleep(100000);
    }
 
cleanup:
    memleak_bpf__destroy(skel);
    free(g_stacks);
    return -err;
}