From ff86b3691d050230bf21e745570252d220b9ba80 Mon Sep 17 00:00:00 2001 From: johba Date: Sun, 15 Mar 2026 10:24:03 +0100 Subject: [PATCH] chore: extract shared inject.sh, add red-team-sweep.sh (#806) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What - `tools/push3-transpiler/inject.sh` — shared transpile+inject logic used by both batch-eval and red-team-sweep - `batch-eval.sh` — replaced inline 60-line Python block with `inject.sh` call - `scripts/harb-evaluator/red-team-sweep.sh` — red-teams each kindergarten seed using existing `red-team.sh`, with random smoke test gate ## Why Sweep script kept breaking because I rewrote the injection logic instead of reusing batch-eval's proven Python. Now there's one copy. ## Testing - inject.sh tested manually on DO box with optimizer_v3 seed - Smoke test picks random seed, injects + compiles before starting sweep Co-authored-by: openhands Reviewed-on: https://codeberg.org/johba/harb/pulls/806 Reviewed-by: review_bot --- scripts/harb-evaluator/red-team-sweep.sh | 96 +++++++++++++++++++ .../revm-evaluator/batch-eval.sh | 71 +------------- tools/push3-transpiler/inject.sh | 63 ++++++++++++ 3 files changed, 163 insertions(+), 67 deletions(-) create mode 100755 scripts/harb-evaluator/red-team-sweep.sh create mode 100755 tools/push3-transpiler/inject.sh diff --git a/scripts/harb-evaluator/red-team-sweep.sh b/scripts/harb-evaluator/red-team-sweep.sh new file mode 100755 index 0000000..6459875 --- /dev/null +++ b/scripts/harb-evaluator/red-team-sweep.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# red-team-sweep.sh — Red-team every kindergarten seed sequentially. +# For each seed: inject into OptimizerV3.sol → run red-team.sh → restore → next. +# Usage: bash red-team-sweep.sh [timeout_per_candidate] +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +SEEDS_DIR="$REPO_ROOT/tools/push3-evolution/seeds" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +INJECT="$REPO_ROOT/tools/push3-transpiler/inject.sh" +ATTACKS_OUT="$REPO_ROOT/onchain/script/backtesting/attacks" +PROGRESS_FILE="/tmp/red-team-sweep-progress.json" +OPT_SOL="$REPO_ROOT/onchain/src/OptimizerV3.sol" +TIMEOUT_PER="${1:-3600}" + +log() { echo "[sweep $(date -u +%H:%M:%S)] $*"; } +die() { log "FATAL: $*" >&2; exit 1; } + +[[ -f "$INJECT" ]] || die "inject.sh not found at $INJECT" + +# Load progress +completed=() +if [[ -f "$PROGRESS_FILE" ]]; then + while IFS= read -r line; do completed+=("$line"); done < <(jq -r '.completed[]' "$PROGRESS_FILE" 2>/dev/null || true) +fi +is_done() { for c in "${completed[@]+"${completed[@]}"}"; do [[ "$c" == "$1" ]] && return 0; done; return 1; } + +# Collect named seeds only (skip run*_gen* pool entries) +seeds=() +for f in "$SEEDS_DIR"/*.push3; do + [[ -f "$f" ]] || continue + basename "$f" | grep -qE '^run[0-9]+_gen' && continue + seeds+=("$f") +done +log "Found ${#seeds[@]} seeds. Timeout: ${TIMEOUT_PER}s each" +[[ ${#seeds[@]} -gt 0 ]] || die "No seeds found in $SEEDS_DIR" + +# ── Smoke test: pick a random seed, inject + compile ── +SMOKE_IDX=$(( RANDOM % ${#seeds[@]} )) +SMOKE_SEED="${seeds[$SMOKE_IDX]}" +SMOKE_NAME=$(basename "$SMOKE_SEED" .push3) +log "Smoke test: $SMOKE_NAME" +cp "$OPT_SOL" "${OPT_SOL}.sweep-backup" +trap 'cp "${OPT_SOL}.sweep-backup" "$OPT_SOL" 2>/dev/null; rm -f "${OPT_SOL}.sweep-backup"' EXIT + +bash "$INJECT" "$SMOKE_SEED" "$OPT_SOL" || die "Smoke test inject failed for $SMOKE_NAME" +cd "$REPO_ROOT/onchain" && forge build --silent 2>&1 || die "Smoke test compile failed for $SMOKE_NAME" +cp "${OPT_SOL}.sweep-backup" "$OPT_SOL" +log "Smoke test passed ✓" + +# ── Main loop ── +for seed_file in "${seeds[@]}"; do + seed_name=$(basename "$seed_file" .push3) + is_done "$seed_name" && { log "SKIP $seed_name (done)"; continue; } + + log "=== RED-TEAM: $seed_name ===" + + # 1. Inject candidate into OptimizerV3.sol + cp "${OPT_SOL}.sweep-backup" "$OPT_SOL" + if ! bash "$INJECT" "$seed_file" "$OPT_SOL"; then + log "SKIP $seed_name — inject failed" + continue + fi + log "Injected into OptimizerV3.sol" + + # 2. Clear stale attack file from previous candidate + rm -f "$REPO_ROOT/tmp/red-team-attacks.jsonl" + + # 3. Run red-team.sh (handles bootstrap + compile + deploy + attack) + log "Running red-team.sh (timeout: ${TIMEOUT_PER}s)..." + CLAUDE_TIMEOUT="$TIMEOUT_PER" timeout "$((TIMEOUT_PER + 120))" \ + bash "$SCRIPT_DIR/red-team.sh" 2>&1 | tee "/tmp/red-team-${seed_name}.log" || true + + # 4. Collect attacks + if [[ -f "$REPO_ROOT/tmp/red-team-attacks.jsonl" ]]; then + ATTACK_COUNT=$(wc -l < "$REPO_ROOT/tmp/red-team-attacks.jsonl") + if [[ "$ATTACK_COUNT" -gt 0 ]]; then + cp "$REPO_ROOT/tmp/red-team-attacks.jsonl" "$ATTACKS_OUT/sweep-${seed_name}.jsonl" + log "Saved $ATTACK_COUNT attack(s)" + fi + fi + + # 5. Save progress + completed+=("$seed_name") + jq -n --argjson arr "$(printf '%s\n' "${completed[@]}" | jq -R . | jq -s .)" \ + '{completed: $arr, last_updated: now | todate}' > "$PROGRESS_FILE" + log "DONE $seed_name" + + # 6. Teardown + cd "$REPO_ROOT" && docker compose down -v 2>/dev/null || true + sleep 5 +done + +# Restore original +cp "${OPT_SOL}.sweep-backup" "$OPT_SOL" +log "=== SWEEP COMPLETE: ${#completed[@]} / ${#seeds[@]} ===" diff --git a/tools/push3-evolution/revm-evaluator/batch-eval.sh b/tools/push3-evolution/revm-evaluator/batch-eval.sh index e6c68e1..14e3c48 100755 --- a/tools/push3-evolution/revm-evaluator/batch-eval.sh +++ b/tools/push3-evolution/revm-evaluator/batch-eval.sh @@ -122,77 +122,14 @@ for PUSH3_FILE in "${PUSH3_FILES[@]}"; do PUSH3_FILE="$(cd "$(dirname "$PUSH3_FILE")" && pwd)/$(basename "$PUSH3_FILE")" CANDIDATE_ID="$(basename "$PUSH3_FILE" .push3)" - # Transpile Push3 → OptimizerV3Push3.sol - TRANSPILE_EC=0 - ( - cd "$TRANSPILER_DIR" - npx ts-node src/index.ts "$PUSH3_FILE" "$TRANSPILER_OUT" - ) >/dev/null 2>&1 || TRANSPILE_EC=$? - - if [ "$TRANSPILE_EC" -ne 0 ]; then - log "WARNING: transpile failed for $CANDIDATE_ID (exit $TRANSPILE_EC) — skipping" + # Transpile Push3 → Solidity, extract function body, inject into OptimizerV3.sol + INJECT_SCRIPT="$REPO_ROOT/tools/push3-transpiler/inject.sh" + if ! bash "$INJECT_SCRIPT" "$PUSH3_FILE" "$OPTIMIZERV3_SOL" >/dev/null 2>&1; then + log "WARNING: transpile/inject failed for $CANDIDATE_ID — skipping" FAILED_IDS="$FAILED_IDS $CANDIDATE_ID" continue fi - # Inject transpiled calculateParams body into OptimizerV3.sol (UUPS-compatible). - # Extract function body from OptimizerV3Push3.sol and replace content between - # BEGIN/END markers in OptimizerV3.sol. - python3 - "$TRANSPILER_OUT" "$OPTIMIZERV3_SOL" <<'PYEOF' || { -import sys - -push3_path = sys.argv[1] -v3_path = sys.argv[2] - -# Extract function body from OptimizerV3Push3.sol -# Find "function calculateParams" then extract everything between the opening { and -# the matching closing } at the same indent level (4 spaces / function level) -with open(push3_path) as f: - push3 = f.read() - -# Find body start: line after "function calculateParams...{" -fn_start = push3.find("function calculateParams") -if fn_start == -1: - sys.exit("calculateParams not found in OptimizerV3Push3") -brace_start = push3.find("{", fn_start) -body_start = push3.index("\n", brace_start) + 1 - -# Find body end: the closing " }" of the function (4-space indent, before contract close) -# Walk backwards from end to find the function-level closing brace -lines = push3[body_start:].split("\n") -body_lines = [] -for line in lines: - if line.strip() == "}" and (line.startswith(" }") or line == "}"): - # This is the function-closing brace - break - body_lines.append(line) - -body = "\n".join(body_lines) - -# Now inject into OptimizerV3.sol between markers -with open(v3_path) as f: - v3 = f.read() - -begin_marker = "// ── BEGIN TRANSPILER OUTPUT" -end_marker = "// ── END TRANSPILER OUTPUT" -begin_idx = v3.find(begin_marker) -end_idx = v3.find(end_marker) -if begin_idx == -1 or end_idx == -1: - sys.exit("markers not found in OptimizerV3.sol") - -begin_line_end = v3.index("\n", begin_idx) + 1 -# Keep the end marker line intact -with open(v3_path, "w") as f: - f.write(v3[:begin_line_end]) - f.write(body + "\n") - f.write(v3[end_idx:]) - -PYEOF - log "WARNING: failed to inject calculateParams into OptimizerV3.sol for $CANDIDATE_ID — skipping" - FAILED_IDS="$FAILED_IDS $CANDIDATE_ID" - continue - } - # Compile (forge's incremental build skips unchanged files quickly) FORGE_EC=0 (cd "$ONCHAIN_DIR" && forge build --silent) >/dev/null 2>&1 || FORGE_EC=$? diff --git a/tools/push3-transpiler/inject.sh b/tools/push3-transpiler/inject.sh new file mode 100755 index 0000000..de9a5c9 --- /dev/null +++ b/tools/push3-transpiler/inject.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# inject.sh — Transpile a Push3 file and inject into OptimizerV3.sol +# Usage: bash inject.sh [optimizer_v3_sol] +# Default OptimizerV3.sol: onchain/src/OptimizerV3.sol (relative to repo root) +# Exit codes: 0=success, 1=transpile failed, 2=inject failed +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +PUSH3_FILE="${1:?Usage: inject.sh [optimizer_v3_sol]}" +OPTIMIZERV3_SOL="${2:-$REPO_ROOT/onchain/src/OptimizerV3.sol}" +TRANSPILER_OUT="$REPO_ROOT/onchain/src/OptimizerV3Push3.sol" + +# Ensure transpiler deps +if [ ! -d "$SCRIPT_DIR/node_modules" ]; then + (cd "$SCRIPT_DIR" && npm install --silent) || exit 1 +fi + +# 1. Transpile Push3 → OptimizerV3Push3.sol (full contract) +(cd "$SCRIPT_DIR" && npx ts-node src/index.ts "$PUSH3_FILE" "$TRANSPILER_OUT") || exit 1 + +# 2. Extract function body and inject between BEGIN/END markers in OptimizerV3.sol +python3 - "$TRANSPILER_OUT" "$OPTIMIZERV3_SOL" <<'PYEOF' || exit 2 +import sys + +push3_path = sys.argv[1] +v3_path = sys.argv[2] + +with open(push3_path) as f: + push3 = f.read() + +fn_start = push3.find("function calculateParams") +if fn_start == -1: + sys.exit("calculateParams not found in OptimizerV3Push3") +brace_start = push3.find("{", fn_start) +body_start = push3.index("\n", brace_start) + 1 + +lines = push3[body_start:].split("\n") +body_lines = [] +for line in lines: + if line.strip() == "}" and (line.startswith(" }") or line == "}"): + break + body_lines.append(line) + +body = "\n".join(body_lines) + +with open(v3_path) as f: + v3 = f.read() + +begin_marker = "// ── BEGIN TRANSPILER OUTPUT" +end_marker = "// ── END TRANSPILER OUTPUT" +begin_idx = v3.find(begin_marker) +end_idx = v3.find(end_marker) +if begin_idx == -1 or end_idx == -1: + sys.exit("markers not found in OptimizerV3.sol") + +begin_line_end = v3.index("\n", begin_idx) + 1 +with open(v3_path, "w") as f: + f.write(v3[:begin_line_end]) + f.write(body + "\n") + f.write(v3[end_idx:]) +PYEOF