Incident Response Playbook

The playbook monitors /var/log/auth.log for repeated failed password attempts from the same source IP. When threshold exceeded, the IP is immediately blocked with iptables and evidence is collected.

#!/usr/bin/env bash
# /usr/local/ir/playbooks/ssh_bruteforce.sh
set -euo pipefail
THRESHOLD=10
TIMEWINDOW=300  # seconds
LOGFILE="/var/log/auth.log"
EVIDENCE_DIR="/var/ir/evidence/$(date +%Y%m%d_%H%M%S)_ssh_bruteforce"
ALERT_EMAIL="[email protected]"

collect_failed_ips() {
    # Count failures per IP in the last $TIMEWINDOW seconds
    local cutoff=$(date -d "@$(($(date +%s) - TIMEWINDOW))" '+%b %_d %H:%M:%S')
    awk -v cutoff="$cutoff" -v threshold="$THRESHOLD" '
        /Failed password/ {
            match($0, /from ([0-9.]+)/, arr)
            count[arr[1]]++
        }
        END {
            for (ip in count)
                if (count[ip] >= threshold)
                    print ip
        }
    ' "$LOGFILE"
}

block_ip() {
    local ip="$1"
    # Check if already blocked
    iptables -C INPUT -s "$ip" -j DROP 2>/dev/null && return
    iptables -I INPUT 1 -s "$ip" -j DROP
    logger -t IR-PLAYBOOK "CONTAINED: Blocked $ip via iptables (SSH brute force)"
}

collect_evidence() {
    local ip="$1"
    mkdir -p "$EVIDENCE_DIR"
    # Auth log excerpts
    grep "$ip" "$LOGFILE" > "$EVIDENCE_DIR/auth_log_excerpt.txt"
    # Current connections from that IP
    ss -tnp | grep "$ip" > "$EVIDENCE_DIR/active_connections.txt" 2>/dev/null || true
    # System state
    ps aux > "$EVIDENCE_DIR/process_list.txt"
    who > "$EVIDENCE_DIR/logged_in_users.txt"
    # Package evidence with hash
    tar czf "${EVIDENCE_DIR}.tar.gz" "$EVIDENCE_DIR"
    sha256sum "${EVIDENCE_DIR}.tar.gz" > "${EVIDENCE_DIR}.sha256"
}

notify() {
    local ip="$1" count="$2"
    mail -s "[IR] SSH Brute Force Blocked: $ip ($count attempts)" "$ALERT_EMAIL" <

Monitors Apache access logs for SQL injection signatures using regex pattern matching. Automatically extracts the attacking IP, logs the attack signature, and adds a temporary firewall block.

#!/usr/bin/env bash
# /usr/local/ir/playbooks/web_anomaly.sh
APACHE_LOG="/var/log/apache2/access.log"
SQLI_PATTERNS='(union|select|insert|drop|exec|xp_|0x[0-9a-f]+|--|\/\*|\*\/|char\(|concat\()'
EVIDENCE_DIR="/var/ir/evidence/$(date +%Y%m%d_%H%M%S)_web_anomaly"

detect_sqli() {
    grep -iE "$SQLI_PATTERNS" "$APACHE_LOG" | \
        awk '{print $1}' | sort | uniq -c | sort -rn | \
        awk '$1 > 5 {print $2}'  # IPs with >5 suspicious requests
}

while IFS= read -r ip; do
    mkdir -p "$EVIDENCE_DIR"
    # Extract matching log lines
    grep "$ip" "$APACHE_LOG" | grep -iE "$SQLI_PATTERNS" \
        > "$EVIDENCE_DIR/sqli_attempts_${ip//./_}.txt"
    # Count attack requests
    count=$(wc -l < "$EVIDENCE_DIR/sqli_attempts_${ip//./_}.txt")
    # Block IP
    iptables -I INPUT 1 -s "$ip" -j DROP
    logger -t IR-PLAYBOOK "CONTAINED: Web anomaly from $ip ($count requests) — IP blocked"
    echo "Blocked $ip — $count suspicious web requests"
done < <(detect_sqli)

Uses ausearch to detect unusual sudo usage by non-privileged accounts or failed su attempts. When triggered, the account is locked and a full audit trail is extracted.

#!/usr/bin/env bash
# /usr/local/ir/playbooks/privesc.sh
EVIDENCE_DIR="/var/ir/evidence/$(date +%Y%m%d_%H%M%S)_privesc"
ADMIN_USERS=("labadmin" "ops")

detect_privesc() {
    # Find users who ran sudo and are NOT in the admin list
    ausearch -m USER_CMD -ts today 2>/dev/null | \
        grep -oP 'acct="[^"]+"' | \
        cut -d'"' -f2 | sort -u | while read -r user; do
            if [[ ! " ${ADMIN_USERS[*]} " =~ " $user " ]]; then
                echo "$user"
            fi
        done
}

respond_to_privesc() {
    local user="$1"
    mkdir -p "$EVIDENCE_DIR"
    # Collect audit events for this user
    ausearch -ua "$user" -ts today > "$EVIDENCE_DIR/audit_${user}.txt" 2>/dev/null
    # Get current sessions
    who | grep "^$user" > "$EVIDENCE_DIR/sessions_${user}.txt" || true
    # Lock the account
    usermod -L "$user"
    passwd -e "$user"  # force password change on next login
    logger -t IR-PLAYBOOK "CONTAINED: Account $user locked (privilege escalation detected)"
    # Kill active sessions
    pkill -u "$user" -KILL || true
    echo "Account $user locked — sessions terminated"
}

while IFS= read -r user; do
    respond_to_privesc "$user"
done < <(detect_privesc)

Created a persistent watchdog service using inotifywait and a systemd service to trigger the appropriate playbook when log files change and thresholds are met.

#!/usr/bin/env bash
# /usr/local/ir/watchdog.sh — monitors logs and triggers playbooks
PLAYBOOK_DIR="/usr/local/ir/playbooks"

# Watch auth.log for SSH events
inotifywait -m -e modify /var/log/auth.log 2>/dev/null | while read -r; do
    bash "$PLAYBOOK_DIR/ssh_bruteforce.sh" &
done &

# Watch apache log for web events
inotifywait -m -e modify /var/log/apache2/access.log 2>/dev/null | while read -r; do
    bash "$PLAYBOOK_DIR/web_anomaly.sh" &
done &

wait

# /etc/systemd/system/ir-watchdog.service
[Unit]
Description=Incident Response Watchdog
After=network.target auditd.service

[Service]
Type=simple
ExecStart=/usr/local/ir/watchdog.sh
Restart=always
RestartSec=5

[Install]
WantedBy=multi-user.target

Simulated each incident scenario in a controlled lab environment to verify detection, containment, and evidence collection worked correctly without false positives on legitimate traffic.

# Simulate SSH brute force (from test machine)
for i in $(seq 1 15); do
    ssh -o StrictHostKeyChecking=no [email protected] 2>/dev/null || true
done

# Verify IP was blocked
sudo iptables -L INPUT -n | grep "$(hostname -I | awk '{print $1}')"

# Simulate SQLi web request
curl "http://192.168.56.20/?id=1%20UNION%20SELECT%201,2,3--"

# Trigger privilege escalation check
sudo ausearch -m USER_CMD -ts today | tail -20

# Review IR evidence directory
ls -lh /var/ir/evidence/
tar -tzvf /var/ir/evidence/*.tar.gz | head -20