gf (Gf patterns) Cheat Sheet
Overview
gf is a wrapper around grep that allows you to quickly search for patterns in files using predefined pattern sets. It's particularly useful for vulnerability hunting, security research, and code analysis. The tool comes with built-in patterns for common vulnerabilities and security issues, and supports custom pattern creation for specialized hunting scenarios.
💡 Key Features: Predefined vulnerability patterns, custom pattern support, fast grep-based searching, integration with other security tools, extensible pattern library, and automation-friendly output.
Installation and Setup
Go Installation
# Install gf via Go
go install github.com/tomnomnom/gf@latest
# Verify installation
gf --help
# Check version
gf --version
# Install to specific location
GOBIN=/usr/local/bin go install github.com/tomnomnom/gf@latest
# Update to latest version
go install github.com/tomnomnom/gf@latest
Binary Installation
# Download latest release for Linux
wget https://github.com/tomnomnom/gf/releases/latest/download/gf-linux-amd64.tgz
tar -xzf gf-linux-amd64.tgz
sudo mv gf /usr/local/bin/
# Download for macOS
wget https://github.com/tomnomnom/gf/releases/latest/download/gf-darwin-amd64.tgz
tar -xzf gf-darwin-amd64.tgz
sudo mv gf /usr/local/bin/
# Download for Windows
wget https://github.com/tomnomnom/gf/releases/latest/download/gf-windows-amd64.zip
unzip gf-windows-amd64.zip
# Move gf.exe to PATH
# Verify installation
gf --help
Package Manager Installation
# Homebrew (macOS/Linux)
brew install gf
# Arch Linux (AUR)
yay -S gf-bin
# Snap package
sudo snap install gf
# Build from source
git clone https://github.com/tomnomnom/gf.git
cd gf
go build -o gf main.go
sudo mv gf /usr/local/bin/
Pattern Installation and Setup
# Create gf configuration directory
mkdir -p ~/.gf
# Install default patterns
git clone https://github.com/tomnomnom/gf.git /tmp/gf-repo
cp /tmp/gf-repo/examples/*.json ~/.gf/
# Install community patterns
git clone https://github.com/1ndianl33t/Gf-Patterns.git /tmp/gf-patterns
cp /tmp/gf-patterns/*.json ~/.gf/
# Install additional pattern collections
git clone https://github.com/dwisiswant0/gf-secrets.git /tmp/gf-secrets
cp /tmp/gf-secrets/*.json ~/.gf/
# List available patterns
gf -list
# Verify pattern installation
ls -la ~/.gf/
# Set up environment variables
echo 'export GF_PATTERNS_PATH=~/.gf' >> ~/.bashrc
source ~/.bashrc
Custom Pattern Creation
# Create custom pattern directory
mkdir -p ~/.gf/custom
# Create basic custom pattern
cat > ~/.gf/custom/api-keys.json << 'EOF'
{
"flags": "-HnriE",
"patterns": [
"api[_-]?key['\"]?\\s*[:=]\\s*['\"]?[a-zA-Z0-9]{20,}",
"secret[_-]?key['\"]?\\s*[:=]\\s*['\"]?[a-zA-Z0-9]{20,}",
"access[_-]?token['\"]?\\s*[:=]\\s*['\"]?[a-zA-Z0-9]{20,}",
"auth[_-]?token['\"]?\\s*[:=]\\s*['\"]?[a-zA-Z0-9]{20,}"
]
}
EOF
# Create advanced custom pattern with multiple flags
cat > ~/.gf/custom/sql-injection.json << 'EOF'
{
"flags": "-HnriE",
"patterns": [
"\\b(union|select|insert|update|delete|drop|create|alter|exec|execute)\\b.*\\b(from|where|order|group|having)\\b",
"'\\s*(or|and)\\s*'?\\d+'?\\s*=\\s*'?\\d+'?",
"\\b(sleep|benchmark|waitfor)\\s*\\(",
"\\b(load_file|into\\s+outfile|into\\s+dumpfile)\\b",
"\\b(information_schema|mysql|sys|performance_schema)\\.",
"\\b(concat|group_concat|char|ascii|substring|mid|left|right)\\s*\\("
]
}
EOF
# Create XSS detection pattern
cat > ~/.gf/custom/xss-advanced.json << 'EOF'
{
"flags": "-HnriE",
"patterns": [
"<\\s*script[^>]*>.*?</\\s*script\\s*>",
"javascript\\s*:",
"on(load|error|click|mouseover|focus|blur|change|submit)\\s*=",
"<\\s*(img|iframe|object|embed|applet|meta|link|style)\\s+[^>]*>",
"\\b(alert|confirm|prompt|eval|setTimeout|setInterval)\\s*\\(",
"document\\.(write|writeln|cookie|location|referrer)",
"window\\.(location|open|close|focus|blur)",
"\\b(innerHTML|outerHTML|insertAdjacentHTML)\\s*="
]
}
EOF
# Create file inclusion pattern
cat > ~/.gf/custom/file-inclusion.json << 'EOF'
{
"flags": "-HnriE",
"patterns": [
"\\b(include|require|include_once|require_once)\\s*\\([^)]*\\$",
"\\b(file_get_contents|readfile|fopen|file)\\s*\\([^)]*\\$",
"\\.\\.[\\/\\\\]",
"\\b(etc\\/passwd|etc\\/shadow|boot\\.ini|win\\.ini)",
"\\b(proc\\/self\\/environ|proc\\/version|proc\\/cmdline)",
"\\b(WEB-INF|META-INF)\\/",
"\\b(php:\\/\\/|file:\\/\\/|data:\\/\\/|expect:\\/\\/)"
]
}
EOF
# Create command injection pattern
cat > ~/.gf/custom/command-injection.json << 'EOF'
{
"flags": "-HnriE",
"patterns": [
"\\b(system|exec|shell_exec|passthru|eval|popen|proc_open)\\s*\\(",
"\\b(cmd|command|sh|bash|powershell|pwsh)\\s*\\(",
"[;&|`$(){}\\[\\]]",
"\\b(cat|ls|dir|type|echo|whoami|id|uname|pwd)\\b",
"\\b(nc|netcat|telnet|wget|curl|ping|nslookup)\\b",
"\\\\x[0-9a-fA-F]{2}",
"\\b(base64|hex|url)\\s*(encode|decode)"
]
}
EOF
# Verify custom patterns
gf -list | grep custom
Basic Usage and Commands
Simple Pattern Searching
# Search for basic patterns in files
gf xss file.txt
gf sqli *.php
gf secrets config.json
# Search in multiple files
gf api-keys *.js *.json *.yaml
# Search recursively in directories
gf passwords -r /path/to/directory
# Search with case sensitivity
gf -i secrets file.txt
# Search and show line numbers
gf -n xss index.html
# Search and show file names only
gf -l sqli *.php
# Search and show context lines
gf -A 3 -B 3 secrets config.js
# Search with custom flags
gf -E -i -n passwords *.txt
Advanced Pattern Searching
# Combine multiple patterns
gf xss file.html && gf sqli file.php
# Search with output redirection
gf secrets *.js > found_secrets.txt
# Search and count matches
gf api-keys *.json | wc -l
# Search with grep options
gf -v false-positive *.log
# Search with custom delimiters
gf --delimiter=":" secrets file.txt
# Search with pattern exclusions
gf secrets *.js | grep -v "test\|example"
# Search with multiple file types
find . -name "*.js" -o -name "*.json" | xargs gf api-keys
# Search with time-based filtering
find . -mtime -7 -name "*.log" | xargs gf errors
Pipeline Integration
# Use with other tools
cat urls.txt | gf xss
echo "test.php?id=1" | gf sqli
# Chain with curl
curl -s https://example.com | gf secrets
# Use with subfinder and httpx
subfinder -d example.com | httpx | gf endpoints
# Combine with waybackurls
echo "example.com" | waybackurls | gf xss
# Use with paramspider
python3 paramspider.py -d example.com | gf sqli
# Chain with nuclei
gf secrets *.js | nuclei -t exposures/
# Use with meg
meg / domains.txt | gf secrets
# Combine with hakrawler
echo "https://example.com" | hakrawler | gf endpoints
Advanced Vulnerability Hunting
Custom Hunting Workflows
#!/usr/bin/env python3
# Advanced vulnerability hunting with gf patterns
import subprocess
import json
import os
import threading
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
class GfVulnerabilityHunter:
def __init__(self, max_workers=10):
self.max_workers = max_workers
self.results = []
self.lock = threading.Lock()
self.patterns_dir = os.path.expanduser("~/.gf")
def get_available_patterns(self):
"""Get list of available gf patterns"""
try:
result = subprocess.run(
['gf', '-list'],
capture_output=True, text=True
)
if result.returncode == 0:
patterns = result.stdout.strip().split('\n')
return [p.strip() for p in patterns if p.strip()]
else:
return []
except Exception as e:
print(f"Error getting patterns: {e}")
return []
def search_pattern(self, pattern, target_files, options=None):
"""Search for specific pattern in target files"""
if options is None:
options = []
try:
# Build gf command
cmd = ['gf'] + options + [pattern] + target_files
result = subprocess.run(
cmd,
capture_output=True, text=True,
timeout=300
)
search_result = {
'pattern': pattern,
'target_files': target_files,
'matches': [],
'match_count': 0,
'success': result.returncode == 0,
'error': result.stderr if result.returncode != 0 else None
}
if result.stdout:
matches = result.stdout.strip().split('\n')
search_result['matches'] = [m for m in matches if m.strip()]
search_result['match_count'] = len(search_result['matches'])
return search_result
except subprocess.TimeoutExpired:
return {
'pattern': pattern,
'target_files': target_files,
'matches': [],
'match_count': 0,
'success': False,
'error': 'Search timeout'
}
except Exception as e:
return {
'pattern': pattern,
'target_files': target_files,
'matches': [],
'match_count': 0,
'success': False,
'error': str(e)
}
def hunt_vulnerabilities(self, target_directory, patterns=None, file_extensions=None):
"""Hunt for vulnerabilities using multiple patterns"""
if patterns is None:
patterns = self.get_available_patterns()
if file_extensions is None:
file_extensions = ['*.js', '*.php', '*.py', '*.java', '*.jsp', '*.asp', '*.aspx', '*.json', '*.xml', '*.yaml', '*.yml', '*.conf', '*.config', '*.ini', '*.env']
# Find target files
target_files = []
for ext in file_extensions:
try:
result = subprocess.run(
['find', target_directory, '-name', ext, '-type', 'f'],
capture_output=True, text=True
)
if result.returncode == 0:
files = result.stdout.strip().split('\n')
target_files.extend([f for f in files if f.strip()])
except Exception as e:
print(f"Error finding files with extension {ext}: {e}")
if not target_files:
print("No target files found")
return []
print(f"Found {len(target_files)} target files")
print(f"Hunting with {len(patterns)} patterns")
# Search patterns concurrently
with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
# Submit search tasks
future_to_pattern = {}
for pattern in patterns:
future = executor.submit(
self.search_pattern,
pattern,
target_files,
['-H', '-n', '-r']
)
future_to_pattern[future] = pattern
# Process results
for future in as_completed(future_to_pattern):
pattern = future_to_pattern[future]
try:
result = future.result()
with self.lock:
self.results.append(result)
if result['success'] and result['match_count'] > 0:
print(f"✓ {pattern}: {result['match_count']} matches")
else:
print(f"- {pattern}: No matches")
except Exception as e:
print(f"✗ Error with pattern {pattern}: {e}")
return self.results
def analyze_results(self):
"""Analyze hunting results and categorize findings"""
analysis = {
'total_patterns': len(self.results),
'successful_patterns': 0,
'total_matches': 0,
'high_risk_findings': [],
'medium_risk_findings': [],
'low_risk_findings': [],
'pattern_summary': {}
}
# Risk categorization
high_risk_patterns = [
'sqli', 'xss', 'rce', 'lfi', 'rfi', 'xxe', 'ssti',
'secrets', 'api-keys', 'passwords', 'tokens'
]
medium_risk_patterns = [
'debug', 'errors', 'endpoints', 'upload', 'redirect',
'cors', 'csrf', 'idor', 'path-traversal'
]
for result in self.results:
if result['success']:
analysis['successful_patterns'] += 1
analysis['total_matches'] += result['match_count']
# Categorize by risk
pattern = result['pattern'].lower()
risk_level = 'low'
if any(hrp in pattern for hrp in high_risk_patterns):
risk_level = 'high'
analysis['high_risk_findings'].append(result)
elif any(mrp in pattern for mrp in medium_risk_patterns):
risk_level = 'medium'
analysis['medium_risk_findings'].append(result)
else:
analysis['low_risk_findings'].append(result)
# Pattern summary
analysis['pattern_summary'][result['pattern']] = {
'matches': result['match_count'],
'risk_level': risk_level,
'success': result['success']
}
return analysis
def generate_report(self, output_file='gf_vulnerability_report.json'):
"""Generate comprehensive vulnerability hunting report"""
analysis = self.analyze_results()
report = {
'scan_info': {
'timestamp': time.strftime('%Y-%m-%d %H:%M:%S'),
'tool': 'gf (Gf patterns)',
'total_patterns_used': analysis['total_patterns'],
'successful_patterns': analysis['successful_patterns'],
'total_matches': analysis['total_matches']
},
'risk_summary': {
'high_risk_findings': len(analysis['high_risk_findings']),
'medium_risk_findings': len(analysis['medium_risk_findings']),
'low_risk_findings': len(analysis['low_risk_findings'])
},
'detailed_findings': {
'high_risk': analysis['high_risk_findings'],
'medium_risk': analysis['medium_risk_findings'],
'low_risk': analysis['low_risk_findings']
},
'pattern_summary': analysis['pattern_summary'],
'raw_results': self.results
}
# Save report
with open(output_file, 'w') as f:
json.dump(report, f, indent=2)
print(f"\nVulnerability Hunting Report:")
print(f"Total patterns used: {analysis['total_patterns']}")
print(f"Successful patterns: {analysis['successful_patterns']}")
print(f"Total matches found: {analysis['total_matches']}")
print(f"High risk findings: {len(analysis['high_risk_findings'])}")
print(f"Medium risk findings: {len(analysis['medium_risk_findings'])}")
print(f"Low risk findings: {len(analysis['low_risk_findings'])}")
print(f"Report saved to: {output_file}")
return report
def create_custom_pattern(self, name, patterns, flags="-HnriE", description=""):
"""Create custom gf pattern"""
pattern_data = {
"flags": flags,
"patterns": patterns
}
if description:
pattern_data["description"] = description
pattern_file = os.path.join(self.patterns_dir, f"{name}.json")
try:
with open(pattern_file, 'w') as f:
json.dump(pattern_data, f, indent=2)
print(f"Custom pattern '{name}' created: {pattern_file}")
return True
except Exception as e:
print(f"Error creating custom pattern: {e}")
return False
# Usage example
if __name__ == "__main__":
# Create hunter instance
hunter = GfVulnerabilityHunter(max_workers=5)
# Create custom patterns
hunter.create_custom_pattern(
"crypto-keys",
[
"-----BEGIN [A-Z ]+-----",
"-----END [A-Z ]+-----",
"ssh-rsa [A-Za-z0-9+/=]+",
"ssh-ed25519 [A-Za-z0-9+/=]+",
"ecdsa-sha2-nistp[0-9]+ [A-Za-z0-9+/=]+"
],
description="Detect cryptographic keys and certificates"
)
hunter.create_custom_pattern(
"cloud-secrets",
[
"AKIA[0-9A-Z]{16}", # AWS Access Key
"AIza[0-9A-Za-z\\-_]{35}", # Google API Key
"sk_live_[0-9a-zA-Z]{24}", # Stripe Live Key
"sk_test_[0-9a-zA-Z]{24}", # Stripe Test Key
"xox[baprs]-[0-9]{12}-[0-9]{12}-[0-9a-zA-Z]{24}" # Slack Token
],
description="Detect cloud service secrets and API keys"
)
# Hunt for vulnerabilities
target_dir = "/path/to/target/directory"
results = hunter.hunt_vulnerabilities(
target_dir,
patterns=['secrets', 'xss', 'sqli', 'api-keys', 'crypto-keys', 'cloud-secrets'],
file_extensions=['*.js', '*.php', '*.py', '*.json', '*.yaml']
)
# Generate report
report = hunter.generate_report('vulnerability_hunting_report.json')
Automated Pattern Management
#!/bin/bash
# Automated gf pattern management and updates
GF_PATTERNS_DIR="$HOME/.gf"
BACKUP_DIR="$HOME/.gf-backup"
TEMP_DIR="/tmp/gf-patterns-update"
# Function to backup existing patterns
backup_patterns() {
echo "Backing up existing patterns..."
mkdir -p "$BACKUP_DIR"
cp -r "$GF_PATTERNS_DIR"/* "$BACKUP_DIR/" 2>/dev/null || true
echo "Backup completed: $BACKUP_DIR"
}
# Function to update patterns from repositories
update_patterns() {
echo "Updating gf patterns from repositories..."
mkdir -p "$TEMP_DIR"
cd "$TEMP_DIR"
# Update from main repository
echo "Updating from tomnomnom/gf..."
git clone https://github.com/tomnomnom/gf.git main-repo
cp main-repo/examples/*.json "$GF_PATTERNS_DIR/" 2>/dev/null || true
# Update from community patterns
echo "Updating from 1ndianl33t/Gf-Patterns..."
git clone https://github.com/1ndianl33t/Gf-Patterns.git community-patterns
cp community-patterns/*.json "$GF_PATTERNS_DIR/" 2>/dev/null || true
# Update from secrets patterns
echo "Updating from dwisiswant0/gf-secrets..."
git clone https://github.com/dwisiswant0/gf-secrets.git secrets-patterns
cp secrets-patterns/*.json "$GF_PATTERNS_DIR/" 2>/dev/null || true
# Update from additional sources
echo "Updating from additional pattern sources..."
# Nuclei templates patterns
git clone https://github.com/projectdiscovery/nuclei-templates.git nuclei-templates
find nuclei-templates -name "*.yaml" -exec grep -l "gf:" {} \; | head -10 | while read template; do
pattern_name=$(basename "$template" .yaml)
echo "Extracting pattern from $template..."
# Extract gf patterns from nuclei templates (simplified)
grep -A 5 "gf:" "$template" | grep -E "pattern|regex" | sed 's/.*: //' > "$GF_PATTERNS_DIR/nuclei-$pattern_name.txt" 2>/dev/null || true
done
# Clean up
cd "$HOME"
rm -rf "$TEMP_DIR"
echo "Pattern update completed"
}
# Function to validate patterns
validate_patterns() {
echo "Validating gf patterns..."
local invalid_patterns=()
for pattern_file in "$GF_PATTERNS_DIR"/*.json; do
if [ -f "$pattern_file" ]; then
pattern_name=$(basename "$pattern_file" .json)
# Check JSON syntax
if ! jq . "$pattern_file" > /dev/null 2>&1; then
echo "❌ Invalid JSON syntax: $pattern_name"
invalid_patterns+=("$pattern_name")
continue
fi
# Check required fields
if ! jq -e '.patterns' "$pattern_file" > /dev/null 2>&1; then
echo "❌ Missing 'patterns' field: $pattern_name"
invalid_patterns+=("$pattern_name")
continue
fi
# Test pattern with gf
if ! gf -list | grep -q "^$pattern_name$"; then
echo "❌ Pattern not recognized by gf: $pattern_name"
invalid_patterns+=("$pattern_name")
continue
fi
echo "✅ Valid pattern: $pattern_name"
fi
done
if [ ${#invalid_patterns[@]} -gt 0 ]; then
echo "Found ${#invalid_patterns[@]} invalid patterns"
return 1
else
echo "All patterns are valid"
return 0
fi
}
# Function to create comprehensive pattern collection
create_comprehensive_patterns() {
echo "Creating comprehensive pattern collection..."
# Web vulnerabilities
cat > "$GF_PATTERNS_DIR/web-vulns-comprehensive.json" << 'EOF'
{
"flags": "-HnriE",
"patterns": [
"\\b(union|select|insert|update|delete|drop|create|alter|exec|execute)\\b.*\\b(from|where|order|group|having)\\b",
"<\\s*script[^>]*>.*?</\\s*script\\s*>",
"javascript\\s*:",
"on(load|error|click|mouseover|focus|blur|change|submit)\\s*=",
"\\b(system|exec|shell_exec|passthru|eval|popen|proc_open)\\s*\\(",
"\\b(include|require|include_once|require_once)\\s*\\([^)]*\\$",
"\\.\\.[\\/\\\\]",
"\\b(etc\\/passwd|etc\\/shadow|boot\\.ini|win\\.ini)",
"\\b(proc\\/self\\/environ|proc\\/version|proc\\/cmdline)",
"\\b(WEB-INF|META-INF)\\/",
"\\b(php:\\/\\/|file:\\/\\/|data:\\/\\/|expect:\\/\\/)"
]
}
EOF
# API and secrets
cat > "$GF_PATTERNS_DIR/api-secrets-comprehensive.json" << 'EOF'
{
"flags": "-HnriE",
"patterns": [
"api[_-]?key['\"]?\\s*[:=]\\s*['\"]?[a-zA-Z0-9]{20,}",
"secret[_-]?key['\"]?\\s*[:=]\\s*['\"]?[a-zA-Z0-9]{20,}",
"access[_-]?token['\"]?\\s*[:=]\\s*['\"]?[a-zA-Z0-9]{20,}",
"auth[_-]?token['\"]?\\s*[:=]\\s*['\"]?[a-zA-Z0-9]{20,}",
"bearer['\"]?\\s*[:=]\\s*['\"]?[a-zA-Z0-9]{20,}",
"password['\"]?\\s*[:=]\\s*['\"]?[^'\"\\s]{8,}",
"passwd['\"]?\\s*[:=]\\s*['\"]?[^'\"\\s]{8,}",
"pwd['\"]?\\s*[:=]\\s*['\"]?[^'\"\\s]{8,}",
"AKIA[0-9A-Z]{16}",
"AIza[0-9A-Za-z\\-_]{35}",
"sk_live_[0-9a-zA-Z]{24}",
"sk_test_[0-9a-zA-Z]{24}",
"xox[baprs]-[0-9]{12}-[0-9]{12}-[0-9a-zA-Z]{24}",
"-----BEGIN [A-Z ]+-----",
"-----END [A-Z ]+-----"
]
}
EOF
# Endpoints and parameters
cat > "$GF_PATTERNS_DIR/endpoints-comprehensive.json" << 'EOF'
{
"flags": "-HnriE",
"patterns": [
"\\/[a-zA-Z0-9_-]+\\.(php|asp|aspx|jsp|do|action)",
"\\/api\\/[a-zA-Z0-9_\\/-]+",
"\\/v[0-9]+\\/[a-zA-Z0-9_\\/-]+",
"\\?[a-zA-Z0-9_-]+=[a-zA-Z0-9_-]+",
"&[a-zA-Z0-9_-]+=[a-zA-Z0-9_-]+",
"\\/(admin|administrator|management|manager|login|signin|auth|dashboard|panel)",
"\\/(upload|download|file|files|document|documents|attachment|attachments)",
"\\/(backup|bak|old|tmp|temp|test|dev|staging|prod|production)",
"\\/(config|configuration|settings|setup|install|installation)",
"\\/(debug|error|exception|log|logs|trace|stack)"
]
}
EOF
# Configuration and sensitive files
cat > "$GF_PATTERNS_DIR/sensitive-files.json" << 'EOF'
{
"flags": "-HnriE",
"patterns": [
"\\.(env|config|conf|ini|yaml|yml|json|xml|properties|cfg)$",
"\\.(key|pem|p12|pfx|jks|keystore|crt|cer|der)$",
"\\.(sql|db|sqlite|mdb|bak|backup|dump)$",
"\\.(log|logs|trace|debug|error)$",
"\\.(git|svn|hg|bzr)\\/",
"\\.(DS_Store|thumbs\\.db|desktop\\.ini)$",
"\\b(web\\.config|app\\.config|machine\\.config)$",
"\\b(htaccess|\\.htpasswd|httpd\\.conf|nginx\\.conf)$",
"\\b(id_rsa|id_dsa|id_ecdsa|id_ed25519)$",
"\\b(known_hosts|authorized_keys|ssh_config)$"
]
}
EOF
echo "Comprehensive patterns created"
}
# Function to test patterns
test_patterns() {
echo "Testing gf patterns..."
# Create test files
mkdir -p /tmp/gf-test
cat > /tmp/gf-test/test.php << 'EOF'
<?php
$api_key = "sk_live_abcdef123456789012345678";
$password = "secretpassword123";
$sql = "SELECT * FROM users WHERE id = " . $_GET['id'];
echo "<script>alert('xss')</script>";
include($_GET['file']);
system($_POST['cmd']);
?>
EOF
cat > /tmp/gf-test/test.js << 'EOF'
const apiKey = "AIzaSyAbCdEfGhIjKlMnOpQrStUvWxYz12345678";
const token = "xoxb-123456789012-123456789012-abcdefghijklmnopqrstuvwx";
document.innerHTML = userInput;
eval(userCode);
window.location = userUrl;
EOF
cat > /tmp/gf-test/config.json << 'EOF'
{
"database": {
"password": "dbpassword123",
"connection": "mysql://user:pass@localhost/db"
},
"aws": {
"access_key": "AKIAIOSFODNN7EXAMPLE",
"secret_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
}
}
EOF
# Test patterns
local test_patterns=("secrets" "xss" "sqli" "api-keys" "endpoints")
for pattern in "${test_patterns[@]}"; do
echo "Testing pattern: $pattern"
if gf -list | grep -q "^$pattern$"; then
result=$(gf "$pattern" /tmp/gf-test/* 2>/dev/null | wc -l)
echo " Found $result matches"
else
echo " Pattern not found"
fi
done
# Clean up
rm -rf /tmp/gf-test
echo "Pattern testing completed"
}
# Function to show pattern statistics
show_pattern_stats() {
echo "Gf Pattern Statistics"
echo "===================="
local total_patterns=$(gf -list | wc -l)
echo "Total patterns: $total_patterns"
local json_patterns=$(find "$GF_PATTERNS_DIR" -name "*.json" | wc -l)
echo "JSON patterns: $json_patterns"
local txt_patterns=$(find "$GF_PATTERNS_DIR" -name "*.txt" | wc -l)
echo "TXT patterns: $txt_patterns"
echo ""
echo "Pattern categories:"
gf -list | grep -E "(xss|sqli|rce|lfi|rfi)" | wc -l | xargs echo " Web vulnerabilities:"
gf -list | grep -E "(secret|key|token|password)" | wc -l | xargs echo " Secrets and credentials:"
gf -list | grep -E "(endpoint|url|param)" | wc -l | xargs echo " Endpoints and parameters:"
gf -list | grep -E "(debug|error|log)" | wc -l | xargs echo " Debug and logging:"
echo ""
echo "Recent patterns (last 7 days):"
find "$GF_PATTERNS_DIR" -name "*.json" -mtime -7 -exec basename {} .json \; | sort
}
# Main execution
case "${1:-help}" in
"backup")
backup_patterns
;;
"update")
backup_patterns
update_patterns
validate_patterns
;;
"validate")
validate_patterns
;;
"create")
create_comprehensive_patterns
;;
"test")
test_patterns
;;
"stats")
show_pattern_stats
;;
"help"|*)
echo "Gf Pattern Management Script"
echo "Usage: $0 {backup|update|validate|create|test|stats|help}"
echo ""
echo "Commands:"
echo " backup - Backup existing patterns"
echo " update - Update patterns from repositories"
echo " validate - Validate pattern syntax and functionality"
echo " create - Create comprehensive pattern collection"
echo " test - Test patterns with sample data"
echo " stats - Show pattern statistics"
echo " help - Show this help message"
;;
esac
Integration with Security Tools
Bug Bounty Automation
#!/bin/bash
# Bug bounty automation with gf patterns
DOMAIN="$1"
OUTPUT_DIR="bug-bounty-$(date +%s)"
if [ -z "$DOMAIN" ]; then
echo "Usage: $0 <domain>"
exit 1
fi
echo "Starting bug bounty automation for: $DOMAIN"
echo "Output directory: $OUTPUT_DIR"
mkdir -p "$OUTPUT_DIR"
# Subdomain discovery
echo "Discovering subdomains..."
subfinder -d "$DOMAIN" -silent > "$OUTPUT_DIR/subdomains.txt"
amass enum -d "$DOMAIN" -passive >> "$OUTPUT_DIR/subdomains.txt"
assetfinder --subs-only "$DOMAIN" >> "$OUTPUT_DIR/subdomains.txt"
# Remove duplicates
sort "$OUTPUT_DIR/subdomains.txt" | uniq > "$OUTPUT_DIR/unique_subdomains.txt"
SUBDOMAIN_COUNT=$(wc -l < "$OUTPUT_DIR/unique_subdomains.txt")
echo "Found $SUBDOMAIN_COUNT unique subdomains"
# Probe for live hosts
echo "Probing for live hosts..."
cat "$OUTPUT_DIR/unique_subdomains.txt" | httpx -silent > "$OUTPUT_DIR/live_hosts.txt"
LIVE_COUNT=$(wc -l < "$OUTPUT_DIR/live_hosts.txt")
echo "Found $LIVE_COUNT live hosts"
# Crawl and gather URLs
echo "Crawling and gathering URLs..."
cat "$OUTPUT_DIR/live_hosts.txt" | hakrawler -depth 3 -plain > "$OUTPUT_DIR/crawled_urls.txt"
cat "$OUTPUT_DIR/live_hosts.txt" | waybackurls >> "$OUTPUT_DIR/crawled_urls.txt"
cat "$OUTPUT_DIR/live_hosts.txt" | gau >> "$OUTPUT_DIR/crawled_urls.txt"
# Remove duplicates and filter
sort "$OUTPUT_DIR/crawled_urls.txt" | uniq | grep -E "\\.(php|asp|aspx|jsp|do|action)" > "$OUTPUT_DIR/interesting_urls.txt"
URL_COUNT=$(wc -l < "$OUTPUT_DIR/interesting_urls.txt")
echo "Found $URL_COUNT interesting URLs"
# Extract parameters
echo "Extracting parameters..."
cat "$OUTPUT_DIR/crawled_urls.txt" | unfurl keys | sort | uniq > "$OUTPUT_DIR/parameters.txt"
PARAM_COUNT=$(wc -l < "$OUTPUT_DIR/parameters.txt")
echo "Found $PARAM_COUNT unique parameters"
# Hunt for vulnerabilities with gf
echo "Hunting for vulnerabilities with gf patterns..."
# XSS hunting
echo "Hunting for XSS..."
cat "$OUTPUT_DIR/crawled_urls.txt" | gf xss > "$OUTPUT_DIR/xss_candidates.txt"
XSS_COUNT=$(wc -l < "$OUTPUT_DIR/xss_candidates.txt")
echo "Found $XSS_COUNT XSS candidates"
# SQL injection hunting
echo "Hunting for SQL injection..."
cat "$OUTPUT_DIR/crawled_urls.txt" | gf sqli > "$OUTPUT_DIR/sqli_candidates.txt"
SQLI_COUNT=$(wc -l < "$OUTPUT_DIR/sqli_candidates.txt")
echo "Found $SQLI_COUNT SQL injection candidates"
# LFI hunting
echo "Hunting for LFI..."
cat "$OUTPUT_DIR/crawled_urls.txt" | gf lfi > "$OUTPUT_DIR/lfi_candidates.txt"
LFI_COUNT=$(wc -l < "$OUTPUT_DIR/lfi_candidates.txt")
echo "Found $LFI_COUNT LFI candidates"
# RCE hunting
echo "Hunting for RCE..."
cat "$OUTPUT_DIR/crawled_urls.txt" | gf rce > "$OUTPUT_DIR/rce_candidates.txt"
RCE_COUNT=$(wc -l < "$OUTPUT_DIR/rce_candidates.txt")
echo "Found $RCE_COUNT RCE candidates"
# SSRF hunting
echo "Hunting for SSRF..."
cat "$OUTPUT_DIR/crawled_urls.txt" | gf ssrf > "$OUTPUT_DIR/ssrf_candidates.txt"
SSRF_COUNT=$(wc -l < "$OUTPUT_DIR/ssrf_candidates.txt")
echo "Found $SSRF_COUNT SSRF candidates"
# Secrets hunting
echo "Hunting for secrets..."
cat "$OUTPUT_DIR/crawled_urls.txt" | gf secrets > "$OUTPUT_DIR/secrets_candidates.txt"
SECRETS_COUNT=$(wc -l < "$OUTPUT_DIR/secrets_candidates.txt")
echo "Found $SECRETS_COUNT secrets candidates"
# API keys hunting
echo "Hunting for API keys..."
cat "$OUTPUT_DIR/crawled_urls.txt" | gf api-keys > "$OUTPUT_DIR/api_keys_candidates.txt"
API_KEYS_COUNT=$(wc -l < "$OUTPUT_DIR/api_keys_candidates.txt")
echo "Found $API_KEYS_COUNT API keys candidates"
# Generate summary report
cat > "$OUTPUT_DIR/summary.txt" << EOF
Bug Bounty Automation Summary
============================
Domain: $DOMAIN
Date: $(date)
Discovery Results:
- Subdomains: $SUBDOMAIN_COUNT
- Live hosts: $LIVE_COUNT
- URLs crawled: $(wc -l < "$OUTPUT_DIR/crawled_urls.txt")
- Interesting URLs: $URL_COUNT
- Parameters: $PARAM_COUNT
Vulnerability Candidates:
- XSS: $XSS_COUNT
- SQL Injection: $SQLI_COUNT
- LFI: $LFI_COUNT
- RCE: $RCE_COUNT
- SSRF: $SSRF_COUNT
- Secrets: $SECRETS_COUNT
- API Keys: $API_KEYS_COUNT
Next Steps:
1. Manual verification of vulnerability candidates
2. Automated testing with nuclei or other tools
3. Report writing and submission
EOF
echo ""
echo "Bug bounty automation completed!"
echo "Results saved in: $OUTPUT_DIR"
echo "Summary:"
cat "$OUTPUT_DIR/summary.txt"
CI/CD Integration
# .github/workflows/gf-security-scan.yml
name: Gf Pattern Security Scan
on:
push:
branches: [ main, develop ]
pull_request:
branches: [ main ]
schedule:
- cron: '0 2 * * *' # Daily scan at 2 AM
jobs:
gf-security-scan:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Setup Go
uses: actions/setup-go@v4
with:
go-version: '1.19'
- name: Install gf
run: |
go install github.com/tomnomnom/gf@latest
# Install patterns
mkdir -p ~/.gf
git clone https://github.com/tomnomnom/gf.git /tmp/gf-repo
cp /tmp/gf-repo/examples/*.json ~/.gf/
git clone https://github.com/1ndianl33t/Gf-Patterns.git /tmp/gf-patterns
cp /tmp/gf-patterns/*.json ~/.gf/
# Verify installation
gf -list
- name: Scan for secrets
run: |
mkdir -p scan-results
# Scan for various types of secrets
gf secrets . > scan-results/secrets.txt 2>/dev/null || touch scan-results/secrets.txt
gf api-keys . > scan-results/api-keys.txt 2>/dev/null || touch scan-results/api-keys.txt
gf passwords . > scan-results/passwords.txt 2>/dev/null || touch scan-results/passwords.txt
# Count findings
SECRETS_COUNT=$(cat scan-results/secrets.txt | wc -l)
API_KEYS_COUNT=$(cat scan-results/api-keys.txt | wc -l)
PASSWORDS_COUNT=$(cat scan-results/passwords.txt | wc -l)
echo "SECRETS_COUNT=$SECRETS_COUNT" >> $GITHUB_ENV
echo "API_KEYS_COUNT=$API_KEYS_COUNT" >> $GITHUB_ENV
echo "PASSWORDS_COUNT=$PASSWORDS_COUNT" >> $GITHUB_ENV
# Generate summary
echo "Secrets found: $SECRETS_COUNT" > scan-results/summary.txt
echo "API keys found: $API_KEYS_COUNT" >> scan-results/summary.txt
echo "Passwords found: $PASSWORDS_COUNT" >> scan-results/summary.txt
- name: Scan for vulnerabilities
run: |
# Scan for common web vulnerabilities in code
gf xss . > scan-results/xss.txt 2>/dev/null || touch scan-results/xss.txt
gf sqli . > scan-results/sqli.txt 2>/dev/null || touch scan-results/sqli.txt
gf lfi . > scan-results/lfi.txt 2>/dev/null || touch scan-results/lfi.txt
gf rce . > scan-results/rce.txt 2>/dev/null || touch scan-results/rce.txt
# Count findings
XSS_COUNT=$(cat scan-results/xss.txt | wc -l)
SQLI_COUNT=$(cat scan-results/sqli.txt | wc -l)
LFI_COUNT=$(cat scan-results/lfi.txt | wc -l)
RCE_COUNT=$(cat scan-results/rce.txt | wc -l)
echo "XSS_COUNT=$XSS_COUNT" >> $GITHUB_ENV
echo "SQLI_COUNT=$SQLI_COUNT" >> $GITHUB_ENV
echo "LFI_COUNT=$LFI_COUNT" >> $GITHUB_ENV
echo "RCE_COUNT=$RCE_COUNT" >> $GITHUB_ENV
# Add to summary
echo "XSS patterns found: $XSS_COUNT" >> scan-results/summary.txt
echo "SQL injection patterns found: $SQLI_COUNT" >> scan-results/summary.txt
echo "LFI patterns found: $LFI_COUNT" >> scan-results/summary.txt
echo "RCE patterns found: $RCE_COUNT" >> scan-results/summary.txt
- name: Upload scan results
uses: actions/upload-artifact@v3
with:
name: gf-scan-results
path: scan-results/
- name: Comment PR with results
if: github.event_name == 'pull_request'
uses: actions/github-script@v6
with:
script: |
const fs = require('fs');
const summary = fs.readFileSync('scan-results/summary.txt', 'utf8');
github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: `## Gf Security Scan Results\n\n\`\`\`\n${summary}\n\`\`\``
});
- name: Fail if critical findings
run: |
TOTAL_CRITICAL=$((SECRETS_COUNT + API_KEYS_COUNT + PASSWORDS_COUNT))
if [ "$TOTAL_CRITICAL" -gt "0" ]; then
echo "Critical security findings detected!"
echo "Secrets: $SECRETS_COUNT"
echo "API Keys: $API_KEYS_COUNT"
echo "Passwords: $PASSWORDS_COUNT"
exit 1
fi
Performance Optimization and Troubleshooting
Performance Tuning
# Optimize gf for different scenarios
# Fast scanning with basic patterns
gf -l secrets *.js
# Thorough scanning with context
gf -A 3 -B 3 api-keys *.json
# Memory-efficient scanning for large files
find . -name "*.log" -exec gf -l errors {} \;
# Parallel processing for multiple patterns
patterns=("secrets" "xss" "sqli" "api-keys")
for pattern in "${patterns[@]}"; do
gf "$pattern" *.php > "results_$pattern.txt" &
done
wait
# Performance monitoring
time gf secrets large_file.txt
# Resource usage monitoring
#!/bin/bash
monitor_gf_performance() {
local pattern="$1"
local files="$2"
local output_file="gf-performance-$(date +%s).log"
echo "Monitoring gf performance for pattern: $pattern"
# Start monitoring
{
echo "Timestamp,CPU%,Memory(MB),Files_processed"
while true; do
if pgrep -f "gf.*$pattern" > /dev/null; then
local cpu=$(ps -p $(pgrep -f "gf.*$pattern") -o %cpu --no-headers)
local mem=$(ps -p $(pgrep -f "gf.*$pattern") -o rss --no-headers | awk '{print $1/1024}')
echo "$(date +%s),$cpu,$mem,processing"
fi
sleep 1
done
} > "$output_file" &
local monitor_pid=$!
# Run gf
gf "$pattern" $files
# Stop monitoring
kill $monitor_pid 2>/dev/null
echo "Performance log saved: $output_file"
}
# Usage
monitor_gf_performance "secrets" "*.js *.json *.yaml"
Troubleshooting Common Issues
# Troubleshooting script for gf
troubleshoot_gf() {
echo "Gf Troubleshooting Guide"
echo "======================="
# Check if gf is installed
if ! command -v gf &> /dev/null; then
echo "❌ gf not found in PATH"
echo "Solution: Install gf using 'go install github.com/tomnomnom/gf@latest'"
return 1
fi
echo "✅ gf found: $(which gf)"
# Check patterns directory
if [ ! -d "$HOME/.gf" ]; then
echo "❌ Patterns directory not found: $HOME/.gf"
echo "Solution: Create directory and install patterns"
echo "mkdir -p ~/.gf"
echo "git clone https://github.com/tomnomnom/gf.git /tmp/gf && cp /tmp/gf/examples/*.json ~/.gf/"
return 1
fi
echo "✅ Patterns directory found: $HOME/.gf"
# Check available patterns
local pattern_count=$(gf -list 2>/dev/null | wc -l)
if [ "$pattern_count" -eq 0 ]; then
echo "❌ No patterns available"
echo "Solution: Install pattern files in ~/.gf/"
return 1
fi
echo "✅ Found $pattern_count patterns"
# Test basic functionality
echo "Testing basic gf functionality..."
echo "test api_key = 'abc123'" > /tmp/gf_test.txt
if gf secrets /tmp/gf_test.txt > /dev/null 2>&1; then
echo "✅ Basic functionality test passed"
else
echo "❌ Basic functionality test failed"
echo "Solution: Check pattern files and gf installation"
rm -f /tmp/gf_test.txt
return 1
fi
rm -f /tmp/gf_test.txt
echo "Troubleshooting completed successfully"
}
# Common error solutions
fix_common_gf_errors() {
echo "Common Gf Error Solutions"
echo "========================"
echo "1. 'pattern not found'"
echo " Solution: Check available patterns with 'gf -list'"
echo " Install missing patterns in ~/.gf/"
echo ""
echo "2. 'no such file or directory'"
echo " Solution: Check if target files exist"
echo " Use absolute paths or verify current directory"
echo ""
echo "3. 'permission denied'"
echo " Solution: Check file permissions"
echo " Use 'chmod +r filename' to make files readable"
echo ""
echo "4. 'grep: invalid option'"
echo " Solution: Check pattern file syntax"
echo " Verify JSON format with 'jq . pattern.json'"
echo ""
echo "5. 'no matches found' (potential false negatives)"
echo " Solution: Try different patterns or create custom ones"
echo " Use verbose mode with grep flags: gf -v pattern files"
echo ""
echo "6. 'too many matches' (performance issues)"
echo " Solution: Use more specific patterns"
echo " Filter results with additional grep: gf pattern files | grep specific"
}
# Run troubleshooting
troubleshoot_gf
fix_common_gf_errors
Resources and Documentation
Official Resources
- gf GitHub Repository - Main repository and documentation
- Pattern Examples - Official pattern collection
- Release Notes - Latest updates and changes
- Bug Reports - Issue tracking and bug reports
Community Resources
- Gf-Patterns Collection - Community pattern repository
- Gf-Secrets Patterns - Specialized secret detection patterns
- Bug Bounty Patterns - Bug bounty hunting patterns
- Security Research Patterns - Nuclei template patterns
Integration Examples
- Automation Scripts - Tom Hudson's security automation
- Bug Bounty Workflows - The Bug Hunter's Methodology
- Security Testing Pipelines - DevSecOps integration
- Pattern Development Guide - Creating custom patterns