chore: extract shared inject.sh, add red-team-sweep.sh (#806)

## What
- `tools/push3-transpiler/inject.sh` — shared transpile+inject logic used by both batch-eval and red-team-sweep
- `batch-eval.sh` — replaced inline 60-line Python block with `inject.sh` call
- `scripts/harb-evaluator/red-team-sweep.sh` — red-teams each kindergarten seed using existing `red-team.sh`, with random smoke test gate

## Why
Sweep script kept breaking because I rewrote the injection logic instead of reusing batch-eval's proven Python. Now there's one copy.

## Testing
- inject.sh tested manually on DO box with optimizer_v3 seed
- Smoke test picks random seed, injects + compiles before starting sweep

Co-authored-by: openhands <openhands@all-hands.dev>
Reviewed-on: https://codeberg.org/johba/harb/pulls/806
Reviewed-by: review_bot <review_bot@noreply.codeberg.org>
This commit is contained in:
johba 2026-03-15 10:24:03 +01:00
parent 5bb4c72897
commit ff86b3691d
3 changed files with 163 additions and 67 deletions

View file

@ -0,0 +1,96 @@
#!/usr/bin/env bash
# red-team-sweep.sh — Red-team every kindergarten seed sequentially.
# For each seed: inject into OptimizerV3.sol → run red-team.sh → restore → next.
# Usage: bash red-team-sweep.sh [timeout_per_candidate]
set -euo pipefail
REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
SEEDS_DIR="$REPO_ROOT/tools/push3-evolution/seeds"
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
INJECT="$REPO_ROOT/tools/push3-transpiler/inject.sh"
ATTACKS_OUT="$REPO_ROOT/onchain/script/backtesting/attacks"
PROGRESS_FILE="/tmp/red-team-sweep-progress.json"
OPT_SOL="$REPO_ROOT/onchain/src/OptimizerV3.sol"
TIMEOUT_PER="${1:-3600}"
log() { echo "[sweep $(date -u +%H:%M:%S)] $*"; }
die() { log "FATAL: $*" >&2; exit 1; }
[[ -f "$INJECT" ]] || die "inject.sh not found at $INJECT"
# Load progress
completed=()
if [[ -f "$PROGRESS_FILE" ]]; then
while IFS= read -r line; do completed+=("$line"); done < <(jq -r '.completed[]' "$PROGRESS_FILE" 2>/dev/null || true)
fi
is_done() { for c in "${completed[@]+"${completed[@]}"}"; do [[ "$c" == "$1" ]] && return 0; done; return 1; }
# Collect named seeds only (skip run*_gen* pool entries)
seeds=()
for f in "$SEEDS_DIR"/*.push3; do
[[ -f "$f" ]] || continue
basename "$f" | grep -qE '^run[0-9]+_gen' && continue
seeds+=("$f")
done
log "Found ${#seeds[@]} seeds. Timeout: ${TIMEOUT_PER}s each"
[[ ${#seeds[@]} -gt 0 ]] || die "No seeds found in $SEEDS_DIR"
# ── Smoke test: pick a random seed, inject + compile ──
SMOKE_IDX=$(( RANDOM % ${#seeds[@]} ))
SMOKE_SEED="${seeds[$SMOKE_IDX]}"
SMOKE_NAME=$(basename "$SMOKE_SEED" .push3)
log "Smoke test: $SMOKE_NAME"
cp "$OPT_SOL" "${OPT_SOL}.sweep-backup"
trap 'cp "${OPT_SOL}.sweep-backup" "$OPT_SOL" 2>/dev/null; rm -f "${OPT_SOL}.sweep-backup"' EXIT
bash "$INJECT" "$SMOKE_SEED" "$OPT_SOL" || die "Smoke test inject failed for $SMOKE_NAME"
cd "$REPO_ROOT/onchain" && forge build --silent 2>&1 || die "Smoke test compile failed for $SMOKE_NAME"
cp "${OPT_SOL}.sweep-backup" "$OPT_SOL"
log "Smoke test passed ✓"
# ── Main loop ──
for seed_file in "${seeds[@]}"; do
seed_name=$(basename "$seed_file" .push3)
is_done "$seed_name" && { log "SKIP $seed_name (done)"; continue; }
log "=== RED-TEAM: $seed_name ==="
# 1. Inject candidate into OptimizerV3.sol
cp "${OPT_SOL}.sweep-backup" "$OPT_SOL"
if ! bash "$INJECT" "$seed_file" "$OPT_SOL"; then
log "SKIP $seed_name — inject failed"
continue
fi
log "Injected into OptimizerV3.sol"
# 2. Clear stale attack file from previous candidate
rm -f "$REPO_ROOT/tmp/red-team-attacks.jsonl"
# 3. Run red-team.sh (handles bootstrap + compile + deploy + attack)
log "Running red-team.sh (timeout: ${TIMEOUT_PER}s)..."
CLAUDE_TIMEOUT="$TIMEOUT_PER" timeout "$((TIMEOUT_PER + 120))" \
bash "$SCRIPT_DIR/red-team.sh" 2>&1 | tee "/tmp/red-team-${seed_name}.log" || true
# 4. Collect attacks
if [[ -f "$REPO_ROOT/tmp/red-team-attacks.jsonl" ]]; then
ATTACK_COUNT=$(wc -l < "$REPO_ROOT/tmp/red-team-attacks.jsonl")
if [[ "$ATTACK_COUNT" -gt 0 ]]; then
cp "$REPO_ROOT/tmp/red-team-attacks.jsonl" "$ATTACKS_OUT/sweep-${seed_name}.jsonl"
log "Saved $ATTACK_COUNT attack(s)"
fi
fi
# 5. Save progress
completed+=("$seed_name")
jq -n --argjson arr "$(printf '%s\n' "${completed[@]}" | jq -R . | jq -s .)" \
'{completed: $arr, last_updated: now | todate}' > "$PROGRESS_FILE"
log "DONE $seed_name"
# 6. Teardown
cd "$REPO_ROOT" && docker compose down -v 2>/dev/null || true
sleep 5
done
# Restore original
cp "${OPT_SOL}.sweep-backup" "$OPT_SOL"
log "=== SWEEP COMPLETE: ${#completed[@]} / ${#seeds[@]} ==="