fix: Red-team attack vector promotion: tmp/ → git via PR (#974)
Add scripts/harb-evaluator/promote-attacks.sh which: - Reads tmp/red-team-attacks.jsonl after a successful red-team run - Deduplicates by op-type fingerprint against all existing attack files - Classifies attack type (staking, il-crystallization, fee-drain-oscillation, floor-ratchet, lp-manipulation, floor-attack) from the op sequence - Creates an isolated git worktree branch from origin/master - Commits the attack file to onchain/script/backtesting/attacks/<type>-<candidate>.jsonl - Opens a Codeberg PR with attack type, ETH extracted, and optimizer profile Integrate into red-team.sh: when the floor breaks (ETH extracted) and an attack export exists, promote-attacks.sh is called automatically (non-fatal). Gracefully no-ops when CODEBERG_TOKEN / ~/.netrc are absent. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
openhands
parent
e105d91818
commit
12940f1201
2 changed files with 337 additions and 0 deletions
323
scripts/harb-evaluator/promote-attacks.sh
Executable file
323
scripts/harb-evaluator/promote-attacks.sh
Executable file
|
|
@ -0,0 +1,323 @@
|
|||
#!/usr/bin/env bash
|
||||
# promote-attacks.sh — Promote red-team attack vectors to onchain/script/backtesting/attacks/ via PR.
|
||||
#
|
||||
# After a red-team run that extracted ETH, this script:
|
||||
# 1. Reads the discovered attack JSONL
|
||||
# 2. Deduplicates against existing files (by op-type fingerprint)
|
||||
# 3. Classifies the attack type from the op sequence
|
||||
# 4. Creates a git branch, commits the file, pushes, and opens a Codeberg PR
|
||||
#
|
||||
# Usage:
|
||||
# promote-attacks.sh [OPTIONS]
|
||||
#
|
||||
# Options:
|
||||
# --attacks FILE Path to attack JSONL (default: <repo>/tmp/red-team-attacks.jsonl)
|
||||
# --candidate NAME Optimizer candidate name (default: $CANDIDATE_NAME or "unknown")
|
||||
# --profile PROFILE Optimizer profile string (default: $OPTIMIZER_PROFILE or "unknown")
|
||||
# --eth-extracted DELTA ETH extracted in wei (default: 0)
|
||||
# --eth-before AMOUNT LM ETH before attack in wei (default: 0)
|
||||
#
|
||||
# Env (all optional):
|
||||
# CODEBERG_TOKEN Codeberg API token. If absent, ~/.netrc is tried.
|
||||
# If neither is present, PR creation is skipped with exit 0.
|
||||
# CANDIDATE_NAME Fallback for --candidate
|
||||
# OPTIMIZER_PROFILE Fallback for --profile
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 PR created, or gracefully skipped (no novel attacks, no token, etc.)
|
||||
# 1 Hard failure (git or API error)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
ATTACKS_DIR="$REPO_ROOT/onchain/script/backtesting/attacks"
|
||||
CODEBERG_REPO="johba/harb"
|
||||
CODEBERG_API="https://codeberg.org/api/v1"
|
||||
|
||||
log() { echo "[promote-attacks] $*"; }
|
||||
warn() { echo "[promote-attacks] WARNING: $*" >&2; }
|
||||
die() { echo "[promote-attacks] ERROR: $*" >&2; exit 1; }
|
||||
|
||||
# ── Parse arguments ──────────────────────────────────────────────────────────
|
||||
ATTACKS_FILE="$REPO_ROOT/tmp/red-team-attacks.jsonl"
|
||||
CANDIDATE="${CANDIDATE_NAME:-unknown}"
|
||||
PROFILE="${OPTIMIZER_PROFILE:-unknown}"
|
||||
ETH_EXTRACTED="0"
|
||||
ETH_BEFORE="0"
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--attacks) ATTACKS_FILE="$2"; shift 2 ;;
|
||||
--candidate) CANDIDATE="$2"; shift 2 ;;
|
||||
--profile) PROFILE="$2"; shift 2 ;;
|
||||
--eth-extracted) ETH_EXTRACTED="$2"; shift 2 ;;
|
||||
--eth-before) ETH_BEFORE="$2"; shift 2 ;;
|
||||
*) die "Unknown argument: $1" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# ── Guard: file must exist and be non-empty ──────────────────────────────────
|
||||
if [[ ! -f "$ATTACKS_FILE" ]]; then
|
||||
log "Attack file not found: $ATTACKS_FILE — nothing to promote"
|
||||
exit 0
|
||||
fi
|
||||
if [[ ! -s "$ATTACKS_FILE" ]]; then
|
||||
log "Attack file is empty — nothing to promote"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
OP_COUNT=$(wc -l < "$ATTACKS_FILE")
|
||||
log "Processing $OP_COUNT ops from $ATTACKS_FILE"
|
||||
log " candidate : $CANDIDATE"
|
||||
log " profile : $PROFILE"
|
||||
log " extracted : $ETH_EXTRACTED wei"
|
||||
|
||||
# ── Resolve Codeberg API token ───────────────────────────────────────────────
|
||||
API_TOKEN="${CODEBERG_TOKEN:-}"
|
||||
if [[ -z "$API_TOKEN" ]] && [[ -f "${HOME:-/home/debian}/.netrc" ]]; then
|
||||
API_TOKEN=$(awk '/codeberg.org/{getline;getline;print $2}' \
|
||||
"${HOME:-/home/debian}/.netrc" 2>/dev/null || true)
|
||||
fi
|
||||
if [[ -z "$API_TOKEN" ]]; then
|
||||
warn "No Codeberg token found (set CODEBERG_TOKEN or configure ~/.netrc) — skipping PR"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Classify attack type and deduplicate ─────────────────────────────────────
|
||||
CLASSIFY_OUT=$(python3 - "$ATTACKS_FILE" "$ATTACKS_DIR" <<'PYEOF'
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def load_ops(path):
|
||||
ops = []
|
||||
with open(path) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
try:
|
||||
ops.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return ops
|
||||
|
||||
|
||||
def fingerprint(ops):
|
||||
"""Ordered tuple of op types — used for deduplication (ignores amounts)."""
|
||||
return tuple(op.get("op", "") for op in ops)
|
||||
|
||||
|
||||
def classify(ops):
|
||||
"""Classify attack type from the operation sequence."""
|
||||
types = [op.get("op", "") for op in ops]
|
||||
has_stake = "stake" in types
|
||||
has_unstake = "unstake" in types
|
||||
has_mint_lp = "mint_lp" in types
|
||||
has_burn_lp = "burn_lp" in types
|
||||
has_loop = "buy_recenter_loop" in types
|
||||
buys = types.count("buy")
|
||||
sells = types.count("sell")
|
||||
recenters = types.count("recenter")
|
||||
|
||||
if has_stake and has_unstake:
|
||||
return "staking"
|
||||
if has_mint_lp or has_burn_lp:
|
||||
return "lp-manipulation"
|
||||
# Compound loop op — il-crystallization with large count
|
||||
if has_loop:
|
||||
return "il-crystallization"
|
||||
# Oscillation: many buys and sells with frequent switching
|
||||
if buys >= 3 and sells >= 3:
|
||||
non_rc = [t for t in types if t not in ("recenter", "mine")]
|
||||
alternations = sum(
|
||||
1 for i in range(len(non_rc) - 1) if non_rc[i] != non_rc[i + 1]
|
||||
)
|
||||
if alternations >= 4:
|
||||
return "fee-drain-oscillation"
|
||||
# IL crystallisation: multiple buys then one final sell
|
||||
if buys >= 3 and sells == 1:
|
||||
return "il-crystallization"
|
||||
# Floor ratchet: many recenters triggered by buys
|
||||
if recenters >= 5 and buys >= 2:
|
||||
return "floor-ratchet"
|
||||
return "floor-attack"
|
||||
|
||||
|
||||
attacks_file = sys.argv[1]
|
||||
attacks_dir = sys.argv[2]
|
||||
|
||||
new_ops = load_ops(attacks_file)
|
||||
if not new_ops:
|
||||
print("EMPTY")
|
||||
sys.exit(0)
|
||||
|
||||
new_fp = fingerprint(new_ops)
|
||||
|
||||
# Deduplication: compare op-type fingerprint against every existing file
|
||||
existing_dir = Path(attacks_dir)
|
||||
if existing_dir.is_dir():
|
||||
for existing_file in sorted(existing_dir.glob("*.jsonl")):
|
||||
try:
|
||||
existing_ops = load_ops(existing_file)
|
||||
if fingerprint(existing_ops) == new_fp:
|
||||
print(f"DUPLICATE:{existing_file.name}")
|
||||
sys.exit(0)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
print(f"NOVEL:{classify(new_ops)}")
|
||||
PYEOF
|
||||
)
|
||||
|
||||
log "Classifier: $CLASSIFY_OUT"
|
||||
|
||||
case "$CLASSIFY_OUT" in
|
||||
EMPTY)
|
||||
log "No ops found in attack file — skipping"
|
||||
exit 0
|
||||
;;
|
||||
DUPLICATE:*)
|
||||
log "Op sequence matches existing file: ${CLASSIFY_OUT#DUPLICATE:} — skipping"
|
||||
exit 0
|
||||
;;
|
||||
NOVEL:*)
|
||||
ATTACK_TYPE="${CLASSIFY_OUT#NOVEL:}"
|
||||
;;
|
||||
*)
|
||||
warn "Unexpected classifier output: $CLASSIFY_OUT"
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
|
||||
log "Novel attack type: $ATTACK_TYPE"
|
||||
|
||||
# ── Determine destination filename ───────────────────────────────────────────
|
||||
# Slug: lowercase, alphanumeric + hyphens, max 30 chars
|
||||
CANDIDATE_SLUG=$(printf '%s' "$CANDIDATE" \
|
||||
| tr '[:upper:]' '[:lower:]' \
|
||||
| sed 's/[^a-z0-9-]/-/g' \
|
||||
| sed 's/-\+/-/g;s/^-//;s/-$//' \
|
||||
| cut -c1-30)
|
||||
|
||||
BASE_NAME="${ATTACK_TYPE}-${CANDIDATE_SLUG}"
|
||||
|
||||
# Avoid collisions with existing files by appending -v2, -v3, ...
|
||||
SUFFIX=""
|
||||
V=2
|
||||
while [[ -f "$ATTACKS_DIR/${BASE_NAME}${SUFFIX}.jsonl" ]]; do
|
||||
SUFFIX="-v${V}"
|
||||
(( V++ ))
|
||||
done
|
||||
BASE_NAME="${BASE_NAME}${SUFFIX}"
|
||||
DEST_RELPATH="onchain/script/backtesting/attacks/${BASE_NAME}.jsonl"
|
||||
|
||||
log "Destination: $DEST_RELPATH"
|
||||
|
||||
# ── Format ETH values for human-readable output ──────────────────────────────
|
||||
ETH_X=$(python3 -c "print(f'{int(\"$ETH_EXTRACTED\") / 1e18:.4f}')" 2>/dev/null \
|
||||
|| echo "$ETH_EXTRACTED wei")
|
||||
ETH_B=$(python3 -c "print(f'{int(\"$ETH_BEFORE\") / 1e18:.4f}')" 2>/dev/null \
|
||||
|| echo "$ETH_BEFORE wei")
|
||||
|
||||
# ── Git: create branch + commit in a temporary worktree ──────────────────────
|
||||
DATE_TAG=$(date -u +%Y%m%d-%H%M%S)
|
||||
BRANCH="red-team/${ATTACK_TYPE}-${CANDIDATE_SLUG}-${DATE_TAG}"
|
||||
TMPWT=$(mktemp -d)
|
||||
|
||||
cleanup_worktree() {
|
||||
local rc=$?
|
||||
cd "$REPO_ROOT" 2>/dev/null || true
|
||||
git worktree remove --force "$TMPWT" 2>/dev/null || true
|
||||
git worktree prune --quiet 2>/dev/null || true
|
||||
rm -rf "$TMPWT" 2>/dev/null || true
|
||||
exit $rc
|
||||
}
|
||||
trap cleanup_worktree EXIT
|
||||
|
||||
log "Fetching origin/master ..."
|
||||
git -C "$REPO_ROOT" fetch origin master --quiet 2>/dev/null \
|
||||
|| warn "git fetch failed — using local origin/master state"
|
||||
|
||||
log "Creating worktree branch: $BRANCH ..."
|
||||
git -C "$REPO_ROOT" worktree add -b "$BRANCH" "$TMPWT" "origin/master" --quiet
|
||||
|
||||
# Copy attack file into the isolated worktree
|
||||
cp "$ATTACKS_FILE" "$TMPWT/$DEST_RELPATH"
|
||||
|
||||
cd "$TMPWT"
|
||||
git add "$DEST_RELPATH"
|
||||
git commit --quiet -m "$(cat <<EOF
|
||||
red-team: add ${ATTACK_TYPE} attack vector (${CANDIDATE})
|
||||
|
||||
Attack type : ${ATTACK_TYPE}
|
||||
Optimizer : ${CANDIDATE}
|
||||
Profile : ${PROFILE}
|
||||
ETH extracted : ${ETH_X} ETH
|
||||
LM ETH before : ${ETH_B} ETH
|
||||
Source file : ${DEST_RELPATH}
|
||||
EOF
|
||||
)"
|
||||
|
||||
log "Pushing branch: $BRANCH ..."
|
||||
git push origin "$BRANCH" --quiet
|
||||
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
# ── Codeberg: create PR ──────────────────────────────────────────────────────
|
||||
PR_TITLE="red-team: ${ATTACK_TYPE} attack via ${CANDIDATE} (${ETH_X} ETH extracted)"
|
||||
|
||||
PR_BODY=$(cat <<EOF
|
||||
## Red-team Attack Discovery
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| Attack type | \`${ATTACK_TYPE}\` |
|
||||
| Optimizer tested | \`${CANDIDATE}\` |
|
||||
| Optimizer profile | \`${PROFILE}\` |
|
||||
| ETH extracted | **${ETH_X} ETH** |
|
||||
| LM ETH before | ${ETH_B} ETH |
|
||||
| Attack file | \`${DEST_RELPATH}\` |
|
||||
|
||||
## What this means
|
||||
|
||||
This attack vector successfully extracted ETH from the LiquidityManager when tested
|
||||
against the \`${CANDIDATE}\` optimizer. Adding it to the attack suite raises the fitness
|
||||
bar for evolution — future optimizer candidates must survive this attack to pass.
|
||||
|
||||
## Review checklist
|
||||
|
||||
- [ ] Attack operations are valid (no malformed ops)
|
||||
- [ ] Attack type classification (\`${ATTACK_TYPE}\`) is accurate
|
||||
- [ ] Not a duplicate of an existing attack (deduplication checked by promote-attacks.sh)
|
||||
- [ ] Appropriate to add as a permanent regression test
|
||||
|
||||
---
|
||||
🤖 Auto-generated by \`scripts/harb-evaluator/promote-attacks.sh\`
|
||||
EOF
|
||||
)
|
||||
|
||||
log "Creating Codeberg PR ..."
|
||||
PR_JSON=$(jq -n \
|
||||
--arg title "$PR_TITLE" \
|
||||
--arg body "$PR_BODY" \
|
||||
--arg head "$BRANCH" \
|
||||
--arg base "master" \
|
||||
'{title: $title, body: $body, head: $head, base: $base}')
|
||||
|
||||
PR_RESPONSE=$(curl -sf \
|
||||
-H "Authorization: token ${API_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
"${CODEBERG_API}/repos/${CODEBERG_REPO}/pulls" \
|
||||
-d "$PR_JSON" 2>&1) || die "curl failed when creating PR"
|
||||
|
||||
PR_NUMBER=$(printf '%s' "$PR_RESPONSE" | jq -r '.number // empty' 2>/dev/null || true)
|
||||
PR_URL=$(printf '%s' "$PR_RESPONSE" | jq -r '.html_url // empty' 2>/dev/null || true)
|
||||
|
||||
if [[ -n "$PR_NUMBER" && "$PR_NUMBER" != "null" ]]; then
|
||||
log "PR #${PR_NUMBER} created: ${PR_URL}"
|
||||
else
|
||||
warn "PR creation returned unexpected response:"
|
||||
printf '%s' "$PR_RESPONSE" | head -c 400 >&2
|
||||
die "PR creation failed"
|
||||
fi
|
||||
|
|
@ -767,6 +767,20 @@ lm_eth_after : $LM_ETH_AFTER
|
|||
delta : -$DELTA
|
||||
verdict : ETH_EXTRACTED
|
||||
SUMMARY_EOF
|
||||
|
||||
# ── 9a. Promote attack vector to git via Codeberg PR (non-fatal) ──────────
|
||||
if [[ -f "$ATTACK_EXPORT" && -s "$ATTACK_EXPORT" ]]; then
|
||||
log "Promoting attack vector to git via PR ..."
|
||||
set +e
|
||||
bash "$SCRIPT_DIR/promote-attacks.sh" \
|
||||
--attacks "$ATTACK_EXPORT" \
|
||||
--candidate "$CANDIDATE_NAME" \
|
||||
--profile "$OPTIMIZER_PROFILE" \
|
||||
--eth-extracted "$DELTA" \
|
||||
--eth-before "$LM_ETH_BEFORE" 2>&1 | while IFS= read -r line; do log " $line"; done
|
||||
set -e
|
||||
fi
|
||||
|
||||
exit 1
|
||||
else
|
||||
log " RESULT: ETH SAFE ✅"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue