chore: extract shared inject.sh, add red-team-sweep.sh (#806)
## What - `tools/push3-transpiler/inject.sh` — shared transpile+inject logic used by both batch-eval and red-team-sweep - `batch-eval.sh` — replaced inline 60-line Python block with `inject.sh` call - `scripts/harb-evaluator/red-team-sweep.sh` — red-teams each kindergarten seed using existing `red-team.sh`, with random smoke test gate ## Why Sweep script kept breaking because I rewrote the injection logic instead of reusing batch-eval's proven Python. Now there's one copy. ## Testing - inject.sh tested manually on DO box with optimizer_v3 seed - Smoke test picks random seed, injects + compiles before starting sweep Co-authored-by: openhands <openhands@all-hands.dev> Reviewed-on: https://codeberg.org/johba/harb/pulls/806 Reviewed-by: review_bot <review_bot@noreply.codeberg.org>
This commit is contained in:
parent
5bb4c72897
commit
ff86b3691d
3 changed files with 163 additions and 67 deletions
96
scripts/harb-evaluator/red-team-sweep.sh
Executable file
96
scripts/harb-evaluator/red-team-sweep.sh
Executable file
|
|
@ -0,0 +1,96 @@
|
|||
#!/usr/bin/env bash
|
||||
# red-team-sweep.sh — Red-team every kindergarten seed sequentially.
|
||||
# For each seed: inject into OptimizerV3.sol → run red-team.sh → restore → next.
|
||||
# Usage: bash red-team-sweep.sh [timeout_per_candidate]
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
SEEDS_DIR="$REPO_ROOT/tools/push3-evolution/seeds"
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
INJECT="$REPO_ROOT/tools/push3-transpiler/inject.sh"
|
||||
ATTACKS_OUT="$REPO_ROOT/onchain/script/backtesting/attacks"
|
||||
PROGRESS_FILE="/tmp/red-team-sweep-progress.json"
|
||||
OPT_SOL="$REPO_ROOT/onchain/src/OptimizerV3.sol"
|
||||
TIMEOUT_PER="${1:-3600}"
|
||||
|
||||
log() { echo "[sweep $(date -u +%H:%M:%S)] $*"; }
|
||||
die() { log "FATAL: $*" >&2; exit 1; }
|
||||
|
||||
[[ -f "$INJECT" ]] || die "inject.sh not found at $INJECT"
|
||||
|
||||
# Load progress
|
||||
completed=()
|
||||
if [[ -f "$PROGRESS_FILE" ]]; then
|
||||
while IFS= read -r line; do completed+=("$line"); done < <(jq -r '.completed[]' "$PROGRESS_FILE" 2>/dev/null || true)
|
||||
fi
|
||||
is_done() { for c in "${completed[@]+"${completed[@]}"}"; do [[ "$c" == "$1" ]] && return 0; done; return 1; }
|
||||
|
||||
# Collect named seeds only (skip run*_gen* pool entries)
|
||||
seeds=()
|
||||
for f in "$SEEDS_DIR"/*.push3; do
|
||||
[[ -f "$f" ]] || continue
|
||||
basename "$f" | grep -qE '^run[0-9]+_gen' && continue
|
||||
seeds+=("$f")
|
||||
done
|
||||
log "Found ${#seeds[@]} seeds. Timeout: ${TIMEOUT_PER}s each"
|
||||
[[ ${#seeds[@]} -gt 0 ]] || die "No seeds found in $SEEDS_DIR"
|
||||
|
||||
# ── Smoke test: pick a random seed, inject + compile ──
|
||||
SMOKE_IDX=$(( RANDOM % ${#seeds[@]} ))
|
||||
SMOKE_SEED="${seeds[$SMOKE_IDX]}"
|
||||
SMOKE_NAME=$(basename "$SMOKE_SEED" .push3)
|
||||
log "Smoke test: $SMOKE_NAME"
|
||||
cp "$OPT_SOL" "${OPT_SOL}.sweep-backup"
|
||||
trap 'cp "${OPT_SOL}.sweep-backup" "$OPT_SOL" 2>/dev/null; rm -f "${OPT_SOL}.sweep-backup"' EXIT
|
||||
|
||||
bash "$INJECT" "$SMOKE_SEED" "$OPT_SOL" || die "Smoke test inject failed for $SMOKE_NAME"
|
||||
cd "$REPO_ROOT/onchain" && forge build --silent 2>&1 || die "Smoke test compile failed for $SMOKE_NAME"
|
||||
cp "${OPT_SOL}.sweep-backup" "$OPT_SOL"
|
||||
log "Smoke test passed ✓"
|
||||
|
||||
# ── Main loop ──
|
||||
for seed_file in "${seeds[@]}"; do
|
||||
seed_name=$(basename "$seed_file" .push3)
|
||||
is_done "$seed_name" && { log "SKIP $seed_name (done)"; continue; }
|
||||
|
||||
log "=== RED-TEAM: $seed_name ==="
|
||||
|
||||
# 1. Inject candidate into OptimizerV3.sol
|
||||
cp "${OPT_SOL}.sweep-backup" "$OPT_SOL"
|
||||
if ! bash "$INJECT" "$seed_file" "$OPT_SOL"; then
|
||||
log "SKIP $seed_name — inject failed"
|
||||
continue
|
||||
fi
|
||||
log "Injected into OptimizerV3.sol"
|
||||
|
||||
# 2. Clear stale attack file from previous candidate
|
||||
rm -f "$REPO_ROOT/tmp/red-team-attacks.jsonl"
|
||||
|
||||
# 3. Run red-team.sh (handles bootstrap + compile + deploy + attack)
|
||||
log "Running red-team.sh (timeout: ${TIMEOUT_PER}s)..."
|
||||
CLAUDE_TIMEOUT="$TIMEOUT_PER" timeout "$((TIMEOUT_PER + 120))" \
|
||||
bash "$SCRIPT_DIR/red-team.sh" 2>&1 | tee "/tmp/red-team-${seed_name}.log" || true
|
||||
|
||||
# 4. Collect attacks
|
||||
if [[ -f "$REPO_ROOT/tmp/red-team-attacks.jsonl" ]]; then
|
||||
ATTACK_COUNT=$(wc -l < "$REPO_ROOT/tmp/red-team-attacks.jsonl")
|
||||
if [[ "$ATTACK_COUNT" -gt 0 ]]; then
|
||||
cp "$REPO_ROOT/tmp/red-team-attacks.jsonl" "$ATTACKS_OUT/sweep-${seed_name}.jsonl"
|
||||
log "Saved $ATTACK_COUNT attack(s)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 5. Save progress
|
||||
completed+=("$seed_name")
|
||||
jq -n --argjson arr "$(printf '%s\n' "${completed[@]}" | jq -R . | jq -s .)" \
|
||||
'{completed: $arr, last_updated: now | todate}' > "$PROGRESS_FILE"
|
||||
log "DONE $seed_name"
|
||||
|
||||
# 6. Teardown
|
||||
cd "$REPO_ROOT" && docker compose down -v 2>/dev/null || true
|
||||
sleep 5
|
||||
done
|
||||
|
||||
# Restore original
|
||||
cp "${OPT_SOL}.sweep-backup" "$OPT_SOL"
|
||||
log "=== SWEEP COMPLETE: ${#completed[@]} / ${#seeds[@]} ==="
|
||||
Loading…
Add table
Add a link
Reference in a new issue