bpftrace 명령어

bpftrace는 Linux eBPF를 위한 고수준 트레이싱 언어입니다. 커널 및 사용자 공간 프로브를 사용하여 트레이싱 스크립트를 작성하기 위한 간결한 구문을 제공하며, 성능 분석 및 디버깅을 위한 동적 계측을 용이하게 합니다.

설치

Linux/Ubuntu

# Ubuntu 22.04+
sudo apt install bpftrace

# Fedora
sudo dnf install bpftrace

# Build from source
git clone https://github.com/bpftrace/bpftrace.git
cd bpftrace && mkdir build && cd build
cmake -DCMAKE_BUILD_TYPE=Release ..
make -j$(nproc) && sudo make install

설치 확인

# Check version and supported probes
bpftrace --version
bpftrace -l 'tracepoint:syscalls:*' | head -20

프로브 유형

Probe Type	Syntax	Description
kprobe	kprobe:func_name	Kernel function entry
kretprobe	kretprobe:func_name	Kernel function return
uprobe	uprobe:/path:func	User-space function entry
uretprobe	uretprobe:/path:func	User-space function return
tracepoint	tracepoint:category:name	Kernel static tracepoint
usdt	usdt:/path:probe	User-space static probe
profile	profile:hz:99	Timed sampling
interval	interval:s:1	Periodic output
software	software:faults:100	Software events
hardware	hardware:cache-misses:1000000	Hardware PMC events
BEGIN	BEGIN	Script startup
END	END	Script teardown

내장 변수

Variable	Description
pid	Process ID
tid	Thread ID
uid	User ID
gid	Group ID
comm	Process name
nsecs	Timestamp in nanoseconds
elapsed	Nanoseconds since bpftrace start
cpu	Current CPU ID
curtask	Current task_struct pointer
retval	Return value (kretprobe/uretprobe)
arg0-argN	Probe arguments
args	Tracepoint arguments struct
cgroup	Cgroup ID of current process
kstack	Kernel stack trace
ustack	User-space stack trace

필수 원라이너

# Trace new processes with arguments
sudo bpftrace -e 'tracepoint:syscalls:sys_enter_execve { printf("%s %s\n", comm, str(args.filename)); }'

# Count syscalls by process name
sudo bpftrace -e 'tracepoint:raw_syscalls:sys_enter { @[comm] = count(); }'

# Distribution of read() sizes
sudo bpftrace -e 'tracepoint:syscalls:sys_enter_read { @bytes = hist(args.count); }'

# Track file opens by process
sudo bpftrace -e 'tracepoint:syscalls:sys_enter_openat { printf("%s %s\n", comm, str(args.filename)); }'

# Measure VFS read latency
sudo bpftrace -e 'kprobe:vfs_read { @start[tid] = nsecs; } kretprobe:vfs_read /@start[tid]/ { @us = hist((nsecs - @start[tid]) / 1000); delete(@start[tid]); }'

# Count kernel function calls
sudo bpftrace -e 'kprobe:tcp_sendmsg { @[comm] = count(); }'

# Profile kernel stacks at 99 Hz
sudo bpftrace -e 'profile:hz:99 { @[kstack] = count(); }'

# TCP connection tracing
sudo bpftrace -e 'kprobe:tcp_v4_connect { printf("%s PID:%d -> connecting\n", comm, pid); }'

# Block I/O latency histogram
sudo bpftrace -e 'tracepoint:block:block_rq_issue { @start[args.dev, args.sector] = nsecs; } tracepoint:block:block_rq_complete /@start[args.dev, args.sector]/ { @usecs = hist((nsecs - @start[args.dev, args.sector]) / 1000); delete(@start[args.dev, args.sector]); }'

맵과 집계

# Count map
sudo bpftrace -e 'kprobe:vfs_read { @reads[comm] = count(); }'

# Sum map
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @bytes[comm] = sum(args.ret); }'

# Average
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @avg_bytes[comm] = avg(args.ret); }'

# Min/Max
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @max_read[comm] = max(args.ret); }'

# Linear histogram (0-100 in steps of 10)
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @size = lhist(args.ret, 0, 10000, 1000); }'

# Log2 histogram
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @size = hist(args.ret); }'

# Stats (count, avg, total)
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @bytes[comm] = stats(args.ret); }'

필터와 포매팅

# Filter by PID
sudo bpftrace -e 'tracepoint:syscalls:sys_enter_write /pid == 1234/ { @bytes = hist(args.count); }'

# Filter by process name
sudo bpftrace -e 'kprobe:vfs_read /comm == "nginx"/ { @[kstack] = count(); }'

# Ternary operator
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read { @result[args.ret >= 0 ? "success" : "error"] = count(); }'

# Timestamp formatting
sudo bpftrace -e 'kprobe:do_nanosleep { printf("%s PID:%d sleeping\n", strftime("%H:%M:%S", nsecs), pid); }'

# String comparison with wildcards
sudo bpftrace -e 'tracepoint:syscalls:sys_enter_openat /str(args.filename) == "/etc/passwd"/ { printf("%s opened /etc/passwd\n", comm); }'

스크립트 (여러 줄)

# Save as latency.bt and run with: sudo bpftrace latency.bt
# Measure read latency per-process
sudo bpftrace -e '
kprobe:vfs_read {
    @start[tid] = nsecs;
}

kretprobe:vfs_read /@start[tid]/ {
    $duration_us = (nsecs - @start[tid]) / 1000;
    @latency_us[comm] = hist($duration_us);
    if ($duration_us > 10000) {
        printf("SLOW READ: %s pid=%d %d us\n", comm, pid, $duration_us);
    }
    delete(@start[tid]);
}

interval:s:5 {
    print(@latency_us);
    clear(@latency_us);
}

END {
    clear(@start);
}
'

프로브 목록 조회 및 탐색

# List all tracepoints
sudo bpftrace -l 'tracepoint:*'

# List probes matching a pattern
sudo bpftrace -l 'kprobe:tcp_*'

# List tracepoint arguments
sudo bpftrace -lv 'tracepoint:syscalls:sys_enter_openat'

# List USDT probes in a binary
sudo bpftrace -l 'usdt:/usr/sbin/mysqld:*'

# Count available probe types
sudo bpftrace -l | awk -F: '{print $1}' | sort | uniq -c | sort -rn

사용자 공간 트레이싱

# Trace malloc calls in a running process
sudo bpftrace -e 'uprobe:/lib/x86_64-linux-gnu/libc.so.6:malloc { @size = hist(arg0); }' -p 1234

# Trace function entry/return in a binary
sudo bpftrace -e 'uprobe:/usr/local/bin/myapp:process_request { @start[tid] = nsecs; } uretprobe:/usr/local/bin/myapp:process_request /@start[tid]/ { @latency = hist((nsecs - @start[tid]) / 1000); delete(@start[tid]); }'

# Trace Python function calls via USDT
sudo bpftrace -e 'usdt:/usr/bin/python3:function__entry { printf("%s:%s\n", str(arg0), str(arg1)); }'

유용한 패턴

# Top processes by syscall count (with interval output)
sudo bpftrace -e 'tracepoint:raw_syscalls:sys_enter { @[comm] = count(); } interval:s:5 { print(@, 10); clear(@); }'

# Trace DNS lookups
sudo bpftrace -e 'uprobe:/lib/x86_64-linux-gnu/libc.so.6:getaddrinfo { printf("%s is resolving: %s\n", comm, str(arg0)); }'

# Page fault tracking
sudo bpftrace -e 'software:page-faults:1 { @[comm, kstack] = count(); }'

# CPU scheduler tracing
sudo bpftrace -e 'tracepoint:sched:sched_switch { @[args.prev_comm] = count(); }'