netcap 编译
编译环境：ubuntu20.04
内核版本：5.15
问题记录
git clone 失败问题
root@debian:~# git clone https://github.com/bytedance/netcap.git
Cloning into 'netcap'...
fatal: unable to access 'https://github.com/bytedance/netcap.git/': GnuTLS recv error (-110): The TLS connection was non-properly terminated.
解决方法:
git config --global http.sslVerify false
模块依赖问题
embed: malformed module path "embed": missing dot in first path element
github.com/bytedance/netcap/pkg/cbpf_filter imports
github.com/cilium/ebpf/asm tested by
github.com/cilium/ebpf/asm.test imports
github.com/frankban/quicktest imports
github.com/google/go-cmp/cmp/cmpopts tested by
github.com/google/go-cmp/cmp/cmpopts.test imports
net/netip: malformed module path "net/netip": missing dot in first path element
系统中预装的 go 为 1.13，embed 模块依赖 go 1.16 及其之后的版本，更新 go 版本解决。
bcc 组件缺少问题
# github.com/iovisor/gobpf/bcc
../go/pkg/mod/github.com/iovisor/gobpf@v0.2.0/bcc/module.go:32:10: fatal error: bcc/bcc_common.h: No such file or directory
32 | #include <bcc/bcc_common.h>
| ^~~~~~~~~~~~~~~~~~
compilation terminated.
解决方法：
sudo apt-get install libbpfcc-dev
pcap.h 头文件缺少问题
# github.com/google/gopacket/pcap
../go/pkg/mod/github.com/google/gopacket@v1.1.19/pcap/pcap_unix.go:34:10: fatal error: pcap.h: No such file or directory
34 | #include <pcap.h>
解决方法：
sudo apt-get install libpcap-dev
bcc 与 gobpf 版本不兼容问题
# github.com/iovisor/gobpf/bcc
../go/pkg/mod/github.com/iovisor/gobpf@v0.2.0/bcc/module.go:261:109: too many arguments in call to (_C2func_bpf_attach_uprobe)
have (_Ctype_int, uint32, *_Ctype_char, *_Ctype_char, _Ctype_ulong, _Ctype_int, number)
want (_Ctype_int, uint32, *_Ctype_char, *_Ctype_char, _Ctype_ulong, _Ctype_int)
go/pkg/mod/github.com/iovisor/gobpf@v0.2.0/bcc/module.go 函数调用点修改为如下内容：
func (bpf *Module) attachUProbe(evName string, attachType uint32, path string, addr uint64, fd, pid int) error {
.............................................................
res, err := C.bpf_attach_uprobe(C.int(fd), attachType, evNameCS, binaryPathCS, (C.uint64_t)(addr), (C.pid_t)(pid))
.............................................................
}
netcap 运行
longyu@longyu-virtual-machine:~/netcap$ ./netcap
Capture skb/mbuf with tcpdump expression
Usage:
netcap [command]
Examples:
$ netcap help skb
$ netcap help mbuf
$ netcap help raw
Available Commands:
completion Generate the autocompletion script for the specified shell
help Help about any command
mbuf Dump mbuf with tcpdump expression
raw Dump raw(packet) with tcpdump expression
skb Dump skb with tcpdump expression
version Version of netcap
Flags:
-h, --help help for netcap
Use "netcap [command] --help" for more information about a command.
netcap 运行问题记录
运行时编译报错
longyu@longyu-virtual-machine:~/netcap$ ./netcap skb -f icmp_rcv@1 -i ens33 -e "host 10.227.0.45" -t "-nnv"
In file included from <built-in>:2:
In file included from /virtual/include/bcc/bpf.h:12:
In file included from include/linux/types.h:6:
In file included from include/uapi/linux/types.h:14:
In file included from ./include/uapi/linux/posix_types.h:5:
In file included from include/linux/stddef.h:5:
In file included from include/uapi/linux/stddef.h:5:
In file included from include/linux/compiler_types.h:80:
include/linux/compiler-clang.h:41:9: warning: '__HAVE_BUILTIN_BSWAP32__' macro redefined [-Wmacro-redefined]
#define __HAVE_BUILTIN_BSWAP32__
^
<command line>:4:9: note: previous definition is here
#define __HAVE_BUILTIN_BSWAP32__ 1
^
In file included from <built-in>:2:
In file included from /virtual/include/bcc/bpf.h:12:
In file included from include/linux/types.h:6:
In file included from include/uapi/linux/types.h:14:
In file included from ./include/uapi/linux/posix_types.h:5:
In file included from include/linux/stddef.h:5:
In file included from include/uapi/linux/stddef.h:5:
In file included from include/linux/compiler_types.h:80:
include/linux/compiler-clang.h:42:9: warning: '__HAVE_BUILTIN_BSWAP64__' macro redefined [-Wmacro-redefined]
#define __HAVE_BUILTIN_BSWAP64__
^
.....................................
^
In file included from /virtual/main.c:1:
In file included from include/net/sock.h:46:
In file included from include/linux/netdevice.h:37:
In file included from include/net/net_namespace.h:38:
In file included from include/net/netns/bpf.h:9:
include/linux/bpf-netns.h:21:7: error: use of undeclared identifier 'BPF_SK_LOOKUP'
case BPF_SK_LOOKUP:
..........................................................................
解决方法：
-
更新系统中的 /usr/include/linux 头文件为当前正在使用的内核版本内容
root@longyu-virtual-machine:/home/longyu/netcap# rm -rf /usr/include/linux/ root@longyu-virtual-machine:/home/longyu/netcap# cp -rf /usr/src/linux-hwe-5.15-headers-5.15.0-117/include/uapi/linux/ /usr/include/ -
重新编译 bcc
git clone https://github.com/iovisor/bcc.git mkdir bcc-build cd bcc-build/ cmake ../bcc -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_LLVM_SHARED=1 make -j10 make install -
修改内核头文件中的一处定义
/usr/include/linux/swab.h 增加如下定义：
#ifndef __attribute_const__ #define __attribute_const__ __attribute__((const)) #endif
netcap 下发命令测试记录
root@longyu-virtual-machine:/home/longyu/netcap# ./netcap skb -f tracepoint:skb:kfree_skb -e "tcp port 9000" -S 2
bpf: Failed to load program: Permission denied
reg type unsupported for arg#0 function xcap_tp_kfree_skb#31
; int xcap_tp_kfree_skb(struct kfree_skb_args *args)
0: (7b) *(u64 *)(r10 -56) = r1
; struct sk_buff *skb = (struct sk_buff*)(args->skbaddr);
1: (79) r1 = *(u64 *)(r1 +8)
2: (7b) *(u64 *)(r10 -32) = r1
3: (b7) r2 = 0
4: (b7) r1 = 0
; u32 key = 0;
5: (7b) *(u64 *)(r10 -24) = r1
last_idx 5 first_idx 0
regs=2 stack=0 before 4: (b7) r1 = 0
..............................................................
77: (85) call bpf_probe_read_kernel#113
R2 min value is negative, either use unsigned or 'var &= const'
processed 191 insns (limit 1000000) max_states_per_insn 0 total_states 13 peak_states 13 mark_read 9
2024/08/09 13:54:38 Dump err: error loading BPF program: permission denied
下发 ./netcap skb -f icmp_rcv@1 -i ens33 -e “host 10.65.10.192” -t "-nnv” 报了如下错误：
bpf: Failed to load program: Permission denied
; int xcap_kprobe_icmp_rcv(struct pt_regs *ctx)
0: (bf) r6 = r1
; if (!(skb = (struct sk_buff *)PT_REGS_PARM1(ctx))) {
1: (79) r9 = *(u64 *)(r6 +112)
; if (!(skb = (struct sk_buff *)PT_REGS_PARM1(ctx))) {
2: (15) if r9 == 0x0 goto pc+159
R1=ctx(id=0,off=0,imm=0) R6_w=ctx(id=0,off=0,imm=0) R9_w=inv(id=0) R10=fp0
3: (b7) r1 = 0
; u32 key = 0;
4: (63) *(u32 *)(r10 -12) = r1
last_idx 4 first_idx 0
regs=2 stack=0 before 3: (b7) r1 = 0
...............................................................
94: (85) call bpf_probe_read_kernel#113
R2 min value is negative, either use unsigned or 'var &= const'
processed 179 insns (limit 1000000) max_states_per_insn 0 total_states 14 peak_states 14 mark_read 10
2024/08/09 13:57:43 Dump err: error loading BPF program: permission denied
github 中的相关 issue：
https://github.com/bytedance/netcap/issues/2
issue 未解决
总结
- netcap 当前文档中未说明内核版本，从代码编译过程中分析至少需要 5.9 以上的版本（支持 BPF_SK_LOOKUP）
- netcap 依赖内核头文件与 bcc 及编译器，下发的规则会动态编译生成必要的文件后装载到内核中
- netcap 基于 usdt 用户态进程探针 dump dpdk mbuf 包，依赖 usdt 功能，此功能成熟度较差
- 基于 ubuntu 环境测试，netcap 并不能成功运行起来，相关问题也没有解决方案
补充
最近又重新访问了下项目，确认在 bpf/skb: Fix netcap load fail in some kernel verison 提交已经修复了上文描述的问题。
基于相同的环境测试记录如下：
root@longyu-virtual-machine:/home/longyu/netcap# ./netcap skb -f icmp_rcv@1 -i ens33 -e "host xx.xx.xx.xx" -t "-nnv"
16:56:28.610526 IP (tos 0x0, ttl 64, id 45515, offset 0, flags [none], proto ICMP (1), length 84)
xx.xx.xx.xx > xx.xx.xx.xx: ICMP echo request, id 45872, seq 14, length 64
16:56:29.615921 IP (tos 0x0, ttl 64, id 2036, offset 0, flags [none], proto ICMP (1), length 84)
xx.xx.xx.xx > xx.xx.xx.xx: ICMP echo request, id 45872, seq 15, length 64
16:56:30.617106 IP (tos 0x0, ttl 64, id 16074, offset 0, flags [none], proto ICMP (1), length 84)
xx.xx.xx.xx > xx.xx.xx.xx: ICMP echo request, id 45872, seq 16, length 64
netcap 已经能够成功使用！
扫一扫
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
