Salta ai contenuti
1337skills 1337skills - Cheat Sheet

Comandi bpftrace

bpftrace è un linguaggio di tracing di alto livello per Linux eBPF. Fornisce una sintassi concisa per scrivere script di tracing utilizzando probe del kernel e dello spazio utente, rendendo la strumentazione dinamica accessibile per l’analisi delle prestazioni e il debugging.

Installazione

Linux/Ubuntu

# Ubuntu 22.04+
sudo apt install bpftrace

# Fedora
sudo dnf install bpftrace

# Build from source
git clone https://github.com/bpftrace/bpftrace.git
cd bpftrace && mkdir build && cd build
cmake -DCMAKE_BUILD_TYPE=Release ..
make -j$(nproc) && sudo make install

Verify Installation

# Check version and supported probes
bpftrace --version
bpftrace -l 'tracepoint:syscalls:*' | head -20

Tipi di probe

Probe TypeSyntaxDescription
kprobekprobe:func_nameKernel function entry
kretprobekretprobe:func_nameKernel function return
uprobeuprobe:/path:funcUser-space function entry
uretprobeuretprobe:/path:funcUser-space function return
tracepointtracepoint:category:nameKernel static tracepoint
usdtusdt:/path:probeUser-space static probe
profileprofile:hz:99Timed sampling
intervalinterval:s:1Periodic output
softwaresoftware:faults:100Software events
hardwarehardware:cache-misses:1000000Hardware PMC events
BEGINBEGINScript startup
ENDENDScript teardown

Variabili integrate

VariableDescription
pidProcess ID
tidThread ID
uidUser ID
gidGroup ID
commProcess name
nsecsTimestamp in nanoseconds
elapsedNanoseconds since bpftrace start
cpuCurrent CPU ID
curtaskCurrent task_struct pointer
retvalReturn value (kretprobe/uretprobe)
arg0-argNProbe arguments
argsTracepoint arguments struct
cgroupCgroup ID of current process
kstackKernel stack trace
ustackUser-space stack trace

One-liner essenziali

# Trace new processes with arguments
sudo bpftrace -e 'tracepoint:syscalls:sys_enter_execve { printf("%s %s\n", comm, str(args.filename)); }'

# Count syscalls by process name
sudo bpftrace -e 'tracepoint:raw_syscalls:sys_enter { @[comm] = count(); }'

# Distribution of read() sizes
sudo bpftrace -e 'tracepoint:syscalls:sys_enter_read { @bytes = hist(args.count); }'

# Track file opens by process
sudo bpftrace -e 'tracepoint:syscalls:sys_enter_openat { printf("%s %s\n", comm, str(args.filename)); }'

# Measure VFS read latency
sudo bpftrace -e 'kprobe:vfs_read { @start[tid] = nsecs; } kretprobe:vfs_read /@start[tid]/ { @us = hist((nsecs - @start[tid]) / 1000); delete(@start[tid]); }'

# Count kernel function calls
sudo bpftrace -e 'kprobe:tcp_sendmsg { @[comm] = count(); }'

# Profile kernel stacks at 99 Hz
sudo bpftrace -e 'profile:hz:99 { @[kstack] = count(); }'

# TCP connection tracing
sudo bpftrace -e 'kprobe:tcp_v4_connect { printf("%s PID:%d -> connecting\n", comm, pid); }'

# Block I/O latency histogram
sudo bpftrace -e 'tracepoint:block:block_rq_issue { @start[args.dev, args.sector] = nsecs; } tracepoint:block:block_rq_complete /@start[args.dev, args.sector]/ { @usecs = hist((nsecs - @start[args.dev, args.sector]) / 1000); delete(@start[args.dev, args.sector]); }'

Mappe e aggregazioni

# Count map
sudo bpftrace -e 'kprobe:vfs_read { @reads[comm] = count(); }'

# Sum map
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @bytes[comm] = sum(args.ret); }'

# Average
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @avg_bytes[comm] = avg(args.ret); }'

# Min/Max
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @max_read[comm] = max(args.ret); }'

# Linear histogram (0-100 in steps of 10)
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @size = lhist(args.ret, 0, 10000, 1000); }'

# Log2 histogram
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @size = hist(args.ret); }'

# Stats (count, avg, total)
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @bytes[comm] = stats(args.ret); }'

Filtri e formattazione

# Filter by PID
sudo bpftrace -e 'tracepoint:syscalls:sys_enter_write /pid == 1234/ { @bytes = hist(args.count); }'

# Filter by process name
sudo bpftrace -e 'kprobe:vfs_read /comm == "nginx"/ { @[kstack] = count(); }'

# Ternary operator
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read { @result[args.ret >= 0 ? "success" : "error"] = count(); }'

# Timestamp formatting
sudo bpftrace -e 'kprobe:do_nanosleep { printf("%s PID:%d sleeping\n", strftime("%H:%M:%S", nsecs), pid); }'

# String comparison with wildcards
sudo bpftrace -e 'tracepoint:syscalls:sys_enter_openat /str(args.filename) == "/etc/passwd"/ { printf("%s opened /etc/passwd\n", comm); }'

Script (multiriga)

# Save as latency.bt and run with: sudo bpftrace latency.bt
# Measure read latency per-process
sudo bpftrace -e '
kprobe:vfs_read {
    @start[tid] = nsecs;
}

kretprobe:vfs_read /@start[tid]/ {
    $duration_us = (nsecs - @start[tid]) / 1000;
    @latency_us[comm] = hist($duration_us);
    if ($duration_us > 10000) {
        printf("SLOW READ: %s pid=%d %d us\n", comm, pid, $duration_us);
    }
    delete(@start[tid]);
}

interval:s:5 {
    print(@latency_us);
    clear(@latency_us);
}

END {
    clear(@start);
}
'

Elenco ed esplorazione delle probe

# List all tracepoints
sudo bpftrace -l 'tracepoint:*'

# List probes matching a pattern
sudo bpftrace -l 'kprobe:tcp_*'

# List tracepoint arguments
sudo bpftrace -lv 'tracepoint:syscalls:sys_enter_openat'

# List USDT probes in a binary
sudo bpftrace -l 'usdt:/usr/sbin/mysqld:*'

# Count available probe types
sudo bpftrace -l | awk -F: '{print $1}' | sort | uniq -c | sort -rn

Tracing user-space

# Trace malloc calls in a running process
sudo bpftrace -e 'uprobe:/lib/x86_64-linux-gnu/libc.so.6:malloc { @size = hist(arg0); }' -p 1234

# Trace function entry/return in a binary
sudo bpftrace -e 'uprobe:/usr/local/bin/myapp:process_request { @start[tid] = nsecs; } uretprobe:/usr/local/bin/myapp:process_request /@start[tid]/ { @latency = hist((nsecs - @start[tid]) / 1000); delete(@start[tid]); }'

# Trace Python function calls via USDT
sudo bpftrace -e 'usdt:/usr/bin/python3:function__entry { printf("%s:%s\n", str(arg0), str(arg1)); }'

Pattern utili

# Top processes by syscall count (with interval output)
sudo bpftrace -e 'tracepoint:raw_syscalls:sys_enter { @[comm] = count(); } interval:s:5 { print(@, 10); clear(@); }'

# Trace DNS lookups
sudo bpftrace -e 'uprobe:/lib/x86_64-linux-gnu/libc.so.6:getaddrinfo { printf("%s is resolving: %s\n", comm, str(arg0)); }'

# Page fault tracking
sudo bpftrace -e 'software:page-faults:1 { @[comm, kstack] = count(); }'

# CPU scheduler tracing
sudo bpftrace -e 'tracepoint:sched:sched_switch { @[args.prev_comm] = count(); }'