bpftrace Commands

bpftrace is a high-level tracing language for Linux eBPF. It provides a concise syntax for writing tracing scripts using kernel and user-space probes, making dynamic instrumentation accessible for performance analysis and debugging.

Installation

Linux/Ubuntu

# Ubuntu 22.04+
sudo apt install bpftrace

# Fedora
sudo dnf install bpftrace

# Build from source
git clone https://github.com/bpftrace/bpftrace.git
cd bpftrace && mkdir build && cd build
cmake -DCMAKE_BUILD_TYPE=Release ..
make -j$(nproc) && sudo make install

Verify Installation

# Check version and supported probes
bpftrace --version
bpftrace -l 'tracepoint:syscalls:*' | head -20

Probe Types

Probe Type	Syntax	Description
kprobe	kprobe:func_name	Kernel function entry
kretprobe	kretprobe:func_name	Kernel function return
uprobe	uprobe:/path:func	User-space function entry
uretprobe	uretprobe:/path:func	User-space function return
tracepoint	tracepoint:category:name	Kernel static tracepoint
usdt	usdt:/path:probe	User-space static probe
profile	profile:hz:99	Timed sampling
interval	interval:s:1	Periodic output
software	software:faults:100	Software events
hardware	hardware:cache-misses:1000000	Hardware PMC events
BEGIN	BEGIN	Script startup
END	END	Script teardown

Built-in Variables

Variable	Description
pid	Process ID
tid	Thread ID
uid	User ID
gid	Group ID
comm	Process name
nsecs	Timestamp in nanoseconds
elapsed	Nanoseconds since bpftrace start
cpu	Current CPU ID
curtask	Current task_struct pointer
retval	Return value (kretprobe/uretprobe)
arg0-argN	Probe arguments
args	Tracepoint arguments struct
cgroup	Cgroup ID of current process
kstack	Kernel stack trace
ustack	User-space stack trace

Essential One-Liners

# Trace new processes with arguments
sudo bpftrace -e 'tracepoint:syscalls:sys_enter_execve { printf("%s %s\n", comm, str(args.filename)); }'

# Count syscalls by process name
sudo bpftrace -e 'tracepoint:raw_syscalls:sys_enter { @[comm] = count(); }'

# Distribution of read() sizes
sudo bpftrace -e 'tracepoint:syscalls:sys_enter_read { @bytes = hist(args.count); }'

# Track file opens by process
sudo bpftrace -e 'tracepoint:syscalls:sys_enter_openat { printf("%s %s\n", comm, str(args.filename)); }'

# Measure VFS read latency
sudo bpftrace -e 'kprobe:vfs_read { @start[tid] = nsecs; } kretprobe:vfs_read /@start[tid]/ { @us = hist((nsecs - @start[tid]) / 1000); delete(@start[tid]); }'

# Count kernel function calls
sudo bpftrace -e 'kprobe:tcp_sendmsg { @[comm] = count(); }'

# Profile kernel stacks at 99 Hz
sudo bpftrace -e 'profile:hz:99 { @[kstack] = count(); }'

# TCP connection tracing
sudo bpftrace -e 'kprobe:tcp_v4_connect { printf("%s PID:%d -> connecting\n", comm, pid); }'

# Block I/O latency histogram
sudo bpftrace -e 'tracepoint:block:block_rq_issue { @start[args.dev, args.sector] = nsecs; } tracepoint:block:block_rq_complete /@start[args.dev, args.sector]/ { @usecs = hist((nsecs - @start[args.dev, args.sector]) / 1000); delete(@start[args.dev, args.sector]); }'

Maps and Aggregations

# Count map
sudo bpftrace -e 'kprobe:vfs_read { @reads[comm] = count(); }'

# Sum map
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @bytes[comm] = sum(args.ret); }'

# Average
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @avg_bytes[comm] = avg(args.ret); }'

# Min/Max
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @max_read[comm] = max(args.ret); }'

# Linear histogram (0-100 in steps of 10)
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @size = lhist(args.ret, 0, 10000, 1000); }'

# Log2 histogram
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @size = hist(args.ret); }'

# Stats (count, avg, total)
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @bytes[comm] = stats(args.ret); }'

Filters and Formatting

# Filter by PID
sudo bpftrace -e 'tracepoint:syscalls:sys_enter_write /pid == 1234/ { @bytes = hist(args.count); }'

# Filter by process name
sudo bpftrace -e 'kprobe:vfs_read /comm == "nginx"/ { @[kstack] = count(); }'

# Ternary operator
sudo bpftrace -e 'tracepoint:syscalls:sys_exit_read { @result[args.ret >= 0 ? "success" : "error"] = count(); }'

# Timestamp formatting
sudo bpftrace -e 'kprobe:do_nanosleep { printf("%s PID:%d sleeping\n", strftime("%H:%M:%S", nsecs), pid); }'

# String comparison with wildcards
sudo bpftrace -e 'tracepoint:syscalls:sys_enter_openat /str(args.filename) == "/etc/passwd"/ { printf("%s opened /etc/passwd\n", comm); }'

Scripts (Multi-line)

# Save as latency.bt and run with: sudo bpftrace latency.bt
# Measure read latency per-process
sudo bpftrace -e '
kprobe:vfs_read {
    @start[tid] = nsecs;
}

kretprobe:vfs_read /@start[tid]/ {
    $duration_us = (nsecs - @start[tid]) / 1000;
    @latency_us[comm] = hist($duration_us);
    if ($duration_us > 10000) {
        printf("SLOW READ: %s pid=%d %d us\n", comm, pid, $duration_us);
    }
    delete(@start[tid]);
}

interval:s:5 {
    print(@latency_us);
    clear(@latency_us);
}

END {
    clear(@start);
}
'

Listing and Exploring Probes

# List all tracepoints
sudo bpftrace -l 'tracepoint:*'

# List probes matching a pattern
sudo bpftrace -l 'kprobe:tcp_*'

# List tracepoint arguments
sudo bpftrace -lv 'tracepoint:syscalls:sys_enter_openat'

# List USDT probes in a binary
sudo bpftrace -l 'usdt:/usr/sbin/mysqld:*'

# Count available probe types
sudo bpftrace -l | awk -F: '{print $1}' | sort | uniq -c | sort -rn

User-Space Tracing

# Trace malloc calls in a running process
sudo bpftrace -e 'uprobe:/lib/x86_64-linux-gnu/libc.so.6:malloc { @size = hist(arg0); }' -p 1234

# Trace function entry/return in a binary
sudo bpftrace -e 'uprobe:/usr/local/bin/myapp:process_request { @start[tid] = nsecs; } uretprobe:/usr/local/bin/myapp:process_request /@start[tid]/ { @latency = hist((nsecs - @start[tid]) / 1000); delete(@start[tid]); }'

# Trace Python function calls via USDT
sudo bpftrace -e 'usdt:/usr/bin/python3:function__entry { printf("%s:%s\n", str(arg0), str(arg1)); }'

Useful Patterns

# Top processes by syscall count (with interval output)
sudo bpftrace -e 'tracepoint:raw_syscalls:sys_enter { @[comm] = count(); } interval:s:5 { print(@, 10); clear(@); }'

# Trace DNS lookups
sudo bpftrace -e 'uprobe:/lib/x86_64-linux-gnu/libc.so.6:getaddrinfo { printf("%s is resolving: %s\n", comm, str(arg0)); }'

# Page fault tracking
sudo bpftrace -e 'software:page-faults:1 { @[comm, kstack] = count(); }'

# CPU scheduler tracing
sudo bpftrace -e 'tracepoint:sched:sched_switch { @[args.prev_comm] = count(); }'