bpftrace-Befehle

bpftrace ist eine höhere Tracing-Sprache für Linux eBPF. Es bietet eine kompakte Syntax zum Schreiben von Tracing-Skripten mit Kernel- und User-Space-Probes und macht dynamische Instrumentierung für Leistungsanalyse und Debugging zugänglich.

Installation

Linux/Ubuntu

# Ubuntu 22.04+
sudo apt install bpftrace

# Fedora
sudo dnf install bpftrace

# Build from source
git clone https://github.com/bpftrace/bpftrace.git
cd bpftrace && mkdir build && cd build
cmake -DCMAKE_BUILD_TYPE=Release ..
make -j$(nproc) && sudo make install

Verify Installation

# Check version and supported probes
bpftrace --version
bpftrace -l 'tracepoint:syscalls:*' | head -20

Probe-Typen

Probe Type	Syntax	Description
kprobe	kprobe:func_name	Kernel function entry
kretprobe	kretprobe:func_name	Kernel function return
uprobe	uprobe:/path:func	User-space function entry
uretprobe	uretprobe:/path:func	User-space function return
tracepoint	tracepoint:category:name	Kernel static tracepoint
usdt	usdt:/path:probe	User-space static probe
profile	profile:hz:99	Timed sampling
interval	interval:s:1	Periodic output
software	software:faults:100	Software events
hardware	hardware:cache-misses:1000000	Hardware PMC events
BEGIN	BEGIN	Script startup
END	END	Script teardown

Eingebaute Variablen

Variable	Description
pid	Process ID
tid	Thread ID
uid	User ID
gid	Group ID
comm	Process name
nsecs	Timestamp in nanoseconds
elapsed	Nanoseconds since bpftrace start
cpu	Current CPU ID
curtask	Current task_struct pointer
retval	Return value (kretprobe/uretprobe)
arg0-argN	Probe arguments
args	Tracepoint arguments struct
cgroup	Cgroup ID of current process
kstack	Kernel stack trace
ustack	User-space stack trace

Wichtige Einzeiler

# Trace new processes with arguments
sudo bpftrace -e 'tracepoint:syscalls:sys_enter_execve { printf("%s %s\n", comm, str(args.filename)); }'

# Count syscalls by process name
sudo bpftrace -e 'tracepoint:raw_syscalls:sys_enter { @[comm] = count(); }'

# Distribution of read() sizes
sudo bpftrace -e 'tracepoint:syscalls:sys_enter_read { @bytes = hist(args.count); }'

# Track file opens by process
sudo bpftrace -e 'tracepoint:syscalls:sys_enter_openat { printf("%s %s\n", comm, str(args.filename)); }'

# Measure VFS read latency
sudo bpftrace -e 'kprobe:vfs_read { @start[tid] = nsecs; } kretprobe:vfs_read /@start[tid]/ { @us = hist((nsecs - @start[tid]) / 1000); delete(@start[tid]); }'

# Count kernel function calls
sudo bpftrace -e 'kprobe:tcp_sendmsg { @[comm] = count(); }'

# Profile kernel stacks at 99 Hz
sudo bpftrace -e 'profile:hz:99 { @[kstack] = count(); }'

# TCP connection tracing
sudo bpftrace -e 'kprobe:tcp_v4_connect { printf("%s PID:%d -> connecting\n", comm, pid); }'

# Block I/O latency histogram
sudo bpftrace -e 'tracepoint:block:block_rq_issue { @start[args.dev, args.sector] = nsecs; } tracepoint:block:block_rq_complete /@start[args.dev, args.sector]/ { @usecs = hist((nsecs - @start[args.dev, args.sector]) / 1000); delete(@start[args.dev, args.sector]); }'

Maps und Aggregationen

# Count map
sudo bpftrace -e 'kprobe:vfs_read { @reads[comm] = count(); }'

# Sum map
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @bytes[comm] = sum(args.ret); }'

# Average
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @avg_bytes[comm] = avg(args.ret); }'

# Min/Max
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @max_read[comm] = max(args.ret); }'

# Linear histogram (0-100 in steps of 10)
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @size = lhist(args.ret, 0, 10000, 1000); }'

# Log2 histogram
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @size = hist(args.ret); }'

# Stats (count, avg, total)
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @bytes[comm] = stats(args.ret); }'

Filter und Formatierung

# Filter by PID
sudo bpftrace -e 'tracepoint:syscalls:sys_enter_write /pid == 1234/ { @bytes = hist(args.count); }'

# Filter by process name
sudo bpftrace -e 'kprobe:vfs_read /comm == "nginx"/ { @[kstack] = count(); }'

# Ternary operator
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read { @result[args.ret >= 0 ? "success" : "error"] = count(); }'

# Timestamp formatting
sudo bpftrace -e 'kprobe:do_nanosleep { printf("%s PID:%d sleeping\n", strftime("%H:%M:%S", nsecs), pid); }'

# String comparison with wildcards
sudo bpftrace -e 'tracepoint:syscalls:sys_enter_openat /str(args.filename) == "/etc/passwd"/ { printf("%s opened /etc/passwd\n", comm); }'

Skripte (mehrzeilig)

# Save as latency.bt and run with: sudo bpftrace latency.bt
# Measure read latency per-process
sudo bpftrace -e '
kprobe:vfs_read {
    @start[tid] = nsecs;
}

kretprobe:vfs_read /@start[tid]/ {
    $duration_us = (nsecs - @start[tid]) / 1000;
    @latency_us[comm] = hist($duration_us);
    if ($duration_us > 10000) {
        printf("SLOW READ: %s pid=%d %d us\n", comm, pid, $duration_us);
    }
    delete(@start[tid]);
}

interval:s:5 {
    print(@latency_us);
    clear(@latency_us);
}

END {
    clear(@start);
}
'

Probes auflisten und erkunden

# List all tracepoints
sudo bpftrace -l 'tracepoint:*'

# List probes matching a pattern
sudo bpftrace -l 'kprobe:tcp_*'

# List tracepoint arguments
sudo bpftrace -lv 'tracepoint:syscalls:sys_enter_openat'

# List USDT probes in a binary
sudo bpftrace -l 'usdt:/usr/sbin/mysqld:*'

# Count available probe types
sudo bpftrace -l | awk -F: '{print $1}' | sort | uniq -c | sort -rn

User-Space-Tracing

# Trace malloc calls in a running process
sudo bpftrace -e 'uprobe:/lib/x86_64-linux-gnu/libc.so.6:malloc { @size = hist(arg0); }' -p 1234

# Trace function entry/return in a binary
sudo bpftrace -e 'uprobe:/usr/local/bin/myapp:process_request { @start[tid] = nsecs; } uretprobe:/usr/local/bin/myapp:process_request /@start[tid]/ { @latency = hist((nsecs - @start[tid]) / 1000); delete(@start[tid]); }'

# Trace Python function calls via USDT
sudo bpftrace -e 'usdt:/usr/bin/python3:function__entry { printf("%s:%s\n", str(arg0), str(arg1)); }'

Nützliche Muster

# Top processes by syscall count (with interval output)
sudo bpftrace -e 'tracepoint:raw_syscalls:sys_enter { @[comm] = count(); } interval:s:5 { print(@, 10); clear(@); }'

# Trace DNS lookups
sudo bpftrace -e 'uprobe:/lib/x86_64-linux-gnu/libc.so.6:getaddrinfo { printf("%s is resolving: %s\n", comm, str(arg0)); }'

# Page fault tracking
sudo bpftrace -e 'software:page-faults:1 { @[comm, kstack] = count(); }'

# CPU scheduler tracing
sudo bpftrace -e 'tracepoint:sched:sched_switch { @[args.prev_comm] = count(); }'