chore: extract shared inject.sh, add red-team-sweep.sh (#806)
## What - `tools/push3-transpiler/inject.sh` — shared transpile+inject logic used by both batch-eval and red-team-sweep - `batch-eval.sh` — replaced inline 60-line Python block with `inject.sh` call - `scripts/harb-evaluator/red-team-sweep.sh` — red-teams each kindergarten seed using existing `red-team.sh`, with random smoke test gate ## Why Sweep script kept breaking because I rewrote the injection logic instead of reusing batch-eval's proven Python. Now there's one copy. ## Testing - inject.sh tested manually on DO box with optimizer_v3 seed - Smoke test picks random seed, injects + compiles before starting sweep Co-authored-by: openhands <openhands@all-hands.dev> Reviewed-on: https://codeberg.org/johba/harb/pulls/806 Reviewed-by: review_bot <review_bot@noreply.codeberg.org>
This commit is contained in:
johba
parent
5bb4c72897
commit
ff86b3691d
3 changed files with 163 additions and 67 deletions
96
scripts/harb-evaluator/red-team-sweep.sh
Executable file
96
scripts/harb-evaluator/red-team-sweep.sh
Executable file
|
|
@ -0,0 +1,96 @@
|
|||
#!/usr/bin/env bash
|
||||
# red-team-sweep.sh — Red-team every kindergarten seed sequentially.
|
||||
# For each seed: inject into OptimizerV3.sol → run red-team.sh → restore → next.
|
||||
# Usage: bash red-team-sweep.sh [timeout_per_candidate]
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
SEEDS_DIR="$REPO_ROOT/tools/push3-evolution/seeds"
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
INJECT="$REPO_ROOT/tools/push3-transpiler/inject.sh"
|
||||
ATTACKS_OUT="$REPO_ROOT/onchain/script/backtesting/attacks"
|
||||
PROGRESS_FILE="/tmp/red-team-sweep-progress.json"
|
||||
OPT_SOL="$REPO_ROOT/onchain/src/OptimizerV3.sol"
|
||||
TIMEOUT_PER="${1:-3600}"
|
||||
|
||||
log() { echo "[sweep $(date -u +%H:%M:%S)] $*"; }
|
||||
die() { log "FATAL: $*" >&2; exit 1; }
|
||||
|
||||
[[ -f "$INJECT" ]] || die "inject.sh not found at $INJECT"
|
||||
|
||||
# Load progress
|
||||
completed=()
|
||||
if [[ -f "$PROGRESS_FILE" ]]; then
|
||||
while IFS= read -r line; do completed+=("$line"); done < <(jq -r '.completed[]' "$PROGRESS_FILE" 2>/dev/null || true)
|
||||
fi
|
||||
is_done() { for c in "${completed[@]+"${completed[@]}"}"; do [[ "$c" == "$1" ]] && return 0; done; return 1; }
|
||||
|
||||
# Collect named seeds only (skip run*_gen* pool entries)
|
||||
seeds=()
|
||||
for f in "$SEEDS_DIR"/*.push3; do
|
||||
[[ -f "$f" ]] || continue
|
||||
basename "$f" | grep -qE '^run[0-9]+_gen' && continue
|
||||
seeds+=("$f")
|
||||
done
|
||||
log "Found ${#seeds[@]} seeds. Timeout: ${TIMEOUT_PER}s each"
|
||||
[[ ${#seeds[@]} -gt 0 ]] || die "No seeds found in $SEEDS_DIR"
|
||||
|
||||
# ── Smoke test: pick a random seed, inject + compile ──
|
||||
SMOKE_IDX=$(( RANDOM % ${#seeds[@]} ))
|
||||
SMOKE_SEED="${seeds[$SMOKE_IDX]}"
|
||||
SMOKE_NAME=$(basename "$SMOKE_SEED" .push3)
|
||||
log "Smoke test: $SMOKE_NAME"
|
||||
cp "$OPT_SOL" "${OPT_SOL}.sweep-backup"
|
||||
trap 'cp "${OPT_SOL}.sweep-backup" "$OPT_SOL" 2>/dev/null; rm -f "${OPT_SOL}.sweep-backup"' EXIT
|
||||
|
||||
bash "$INJECT" "$SMOKE_SEED" "$OPT_SOL" || die "Smoke test inject failed for $SMOKE_NAME"
|
||||
cd "$REPO_ROOT/onchain" && forge build --silent 2>&1 || die "Smoke test compile failed for $SMOKE_NAME"
|
||||
cp "${OPT_SOL}.sweep-backup" "$OPT_SOL"
|
||||
log "Smoke test passed ✓"
|
||||
|
||||
# ── Main loop ──
|
||||
for seed_file in "${seeds[@]}"; do
|
||||
seed_name=$(basename "$seed_file" .push3)
|
||||
is_done "$seed_name" && { log "SKIP $seed_name (done)"; continue; }
|
||||
|
||||
log "=== RED-TEAM: $seed_name ==="
|
||||
|
||||
# 1. Inject candidate into OptimizerV3.sol
|
||||
cp "${OPT_SOL}.sweep-backup" "$OPT_SOL"
|
||||
if ! bash "$INJECT" "$seed_file" "$OPT_SOL"; then
|
||||
log "SKIP $seed_name — inject failed"
|
||||
continue
|
||||
fi
|
||||
log "Injected into OptimizerV3.sol"
|
||||
|
||||
# 2. Clear stale attack file from previous candidate
|
||||
rm -f "$REPO_ROOT/tmp/red-team-attacks.jsonl"
|
||||
|
||||
# 3. Run red-team.sh (handles bootstrap + compile + deploy + attack)
|
||||
log "Running red-team.sh (timeout: ${TIMEOUT_PER}s)..."
|
||||
CLAUDE_TIMEOUT="$TIMEOUT_PER" timeout "$((TIMEOUT_PER + 120))" \
|
||||
bash "$SCRIPT_DIR/red-team.sh" 2>&1 | tee "/tmp/red-team-${seed_name}.log" || true
|
||||
|
||||
# 4. Collect attacks
|
||||
if [[ -f "$REPO_ROOT/tmp/red-team-attacks.jsonl" ]]; then
|
||||
ATTACK_COUNT=$(wc -l < "$REPO_ROOT/tmp/red-team-attacks.jsonl")
|
||||
if [[ "$ATTACK_COUNT" -gt 0 ]]; then
|
||||
cp "$REPO_ROOT/tmp/red-team-attacks.jsonl" "$ATTACKS_OUT/sweep-${seed_name}.jsonl"
|
||||
log "Saved $ATTACK_COUNT attack(s)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 5. Save progress
|
||||
completed+=("$seed_name")
|
||||
jq -n --argjson arr "$(printf '%s\n' "${completed[@]}" | jq -R . | jq -s .)" \
|
||||
'{completed: $arr, last_updated: now | todate}' > "$PROGRESS_FILE"
|
||||
log "DONE $seed_name"
|
||||
|
||||
# 6. Teardown
|
||||
cd "$REPO_ROOT" && docker compose down -v 2>/dev/null || true
|
||||
sleep 5
|
||||
done
|
||||
|
||||
# Restore original
|
||||
cp "${OPT_SOL}.sweep-backup" "$OPT_SOL"
|
||||
log "=== SWEEP COMPLETE: ${#completed[@]} / ${#seeds[@]} ==="
|
||||
|
|
@ -122,77 +122,14 @@ for PUSH3_FILE in "${PUSH3_FILES[@]}"; do
|
|||
PUSH3_FILE="$(cd "$(dirname "$PUSH3_FILE")" && pwd)/$(basename "$PUSH3_FILE")"
|
||||
CANDIDATE_ID="$(basename "$PUSH3_FILE" .push3)"
|
||||
|
||||
# Transpile Push3 → OptimizerV3Push3.sol
|
||||
TRANSPILE_EC=0
|
||||
(
|
||||
cd "$TRANSPILER_DIR"
|
||||
npx ts-node src/index.ts "$PUSH3_FILE" "$TRANSPILER_OUT"
|
||||
) >/dev/null 2>&1 || TRANSPILE_EC=$?
|
||||
|
||||
if [ "$TRANSPILE_EC" -ne 0 ]; then
|
||||
log "WARNING: transpile failed for $CANDIDATE_ID (exit $TRANSPILE_EC) — skipping"
|
||||
# Transpile Push3 → Solidity, extract function body, inject into OptimizerV3.sol
|
||||
INJECT_SCRIPT="$REPO_ROOT/tools/push3-transpiler/inject.sh"
|
||||
if ! bash "$INJECT_SCRIPT" "$PUSH3_FILE" "$OPTIMIZERV3_SOL" >/dev/null 2>&1; then
|
||||
log "WARNING: transpile/inject failed for $CANDIDATE_ID — skipping"
|
||||
FAILED_IDS="$FAILED_IDS $CANDIDATE_ID"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Inject transpiled calculateParams body into OptimizerV3.sol (UUPS-compatible).
|
||||
# Extract function body from OptimizerV3Push3.sol and replace content between
|
||||
# BEGIN/END markers in OptimizerV3.sol.
|
||||
python3 - "$TRANSPILER_OUT" "$OPTIMIZERV3_SOL" <<'PYEOF' || {
|
||||
import sys
|
||||
|
||||
push3_path = sys.argv[1]
|
||||
v3_path = sys.argv[2]
|
||||
|
||||
# Extract function body from OptimizerV3Push3.sol
|
||||
# Find "function calculateParams" then extract everything between the opening { and
|
||||
# the matching closing } at the same indent level (4 spaces / function level)
|
||||
with open(push3_path) as f:
|
||||
push3 = f.read()
|
||||
|
||||
# Find body start: line after "function calculateParams...{"
|
||||
fn_start = push3.find("function calculateParams")
|
||||
if fn_start == -1:
|
||||
sys.exit("calculateParams not found in OptimizerV3Push3")
|
||||
brace_start = push3.find("{", fn_start)
|
||||
body_start = push3.index("\n", brace_start) + 1
|
||||
|
||||
# Find body end: the closing " }" of the function (4-space indent, before contract close)
|
||||
# Walk backwards from end to find the function-level closing brace
|
||||
lines = push3[body_start:].split("\n")
|
||||
body_lines = []
|
||||
for line in lines:
|
||||
if line.strip() == "}" and (line.startswith(" }") or line == "}"):
|
||||
# This is the function-closing brace
|
||||
break
|
||||
body_lines.append(line)
|
||||
|
||||
body = "\n".join(body_lines)
|
||||
|
||||
# Now inject into OptimizerV3.sol between markers
|
||||
with open(v3_path) as f:
|
||||
v3 = f.read()
|
||||
|
||||
begin_marker = "// ── BEGIN TRANSPILER OUTPUT"
|
||||
end_marker = "// ── END TRANSPILER OUTPUT"
|
||||
begin_idx = v3.find(begin_marker)
|
||||
end_idx = v3.find(end_marker)
|
||||
if begin_idx == -1 or end_idx == -1:
|
||||
sys.exit("markers not found in OptimizerV3.sol")
|
||||
|
||||
begin_line_end = v3.index("\n", begin_idx) + 1
|
||||
# Keep the end marker line intact
|
||||
with open(v3_path, "w") as f:
|
||||
f.write(v3[:begin_line_end])
|
||||
f.write(body + "\n")
|
||||
f.write(v3[end_idx:])
|
||||
|
||||
PYEOF
|
||||
log "WARNING: failed to inject calculateParams into OptimizerV3.sol for $CANDIDATE_ID — skipping"
|
||||
FAILED_IDS="$FAILED_IDS $CANDIDATE_ID"
|
||||
continue
|
||||
}
|
||||
|
||||
# Compile (forge's incremental build skips unchanged files quickly)
|
||||
FORGE_EC=0
|
||||
(cd "$ONCHAIN_DIR" && forge build --silent) >/dev/null 2>&1 || FORGE_EC=$?
|
||||
|
|
|
|||
63
tools/push3-transpiler/inject.sh
Executable file
63
tools/push3-transpiler/inject.sh
Executable file
|
|
@ -0,0 +1,63 @@
|
|||
#!/usr/bin/env bash
|
||||
# inject.sh — Transpile a Push3 file and inject into OptimizerV3.sol
|
||||
# Usage: bash inject.sh <push3_file> [optimizer_v3_sol]
|
||||
# Default OptimizerV3.sol: onchain/src/OptimizerV3.sol (relative to repo root)
|
||||
# Exit codes: 0=success, 1=transpile failed, 2=inject failed
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
|
||||
PUSH3_FILE="${1:?Usage: inject.sh <push3_file> [optimizer_v3_sol]}"
|
||||
OPTIMIZERV3_SOL="${2:-$REPO_ROOT/onchain/src/OptimizerV3.sol}"
|
||||
TRANSPILER_OUT="$REPO_ROOT/onchain/src/OptimizerV3Push3.sol"
|
||||
|
||||
# Ensure transpiler deps
|
||||
if [ ! -d "$SCRIPT_DIR/node_modules" ]; then
|
||||
(cd "$SCRIPT_DIR" && npm install --silent) || exit 1
|
||||
fi
|
||||
|
||||
# 1. Transpile Push3 → OptimizerV3Push3.sol (full contract)
|
||||
(cd "$SCRIPT_DIR" && npx ts-node src/index.ts "$PUSH3_FILE" "$TRANSPILER_OUT") || exit 1
|
||||
|
||||
# 2. Extract function body and inject between BEGIN/END markers in OptimizerV3.sol
|
||||
python3 - "$TRANSPILER_OUT" "$OPTIMIZERV3_SOL" <<'PYEOF' || exit 2
|
||||
import sys
|
||||
|
||||
push3_path = sys.argv[1]
|
||||
v3_path = sys.argv[2]
|
||||
|
||||
with open(push3_path) as f:
|
||||
push3 = f.read()
|
||||
|
||||
fn_start = push3.find("function calculateParams")
|
||||
if fn_start == -1:
|
||||
sys.exit("calculateParams not found in OptimizerV3Push3")
|
||||
brace_start = push3.find("{", fn_start)
|
||||
body_start = push3.index("\n", brace_start) + 1
|
||||
|
||||
lines = push3[body_start:].split("\n")
|
||||
body_lines = []
|
||||
for line in lines:
|
||||
if line.strip() == "}" and (line.startswith(" }") or line == "}"):
|
||||
break
|
||||
body_lines.append(line)
|
||||
|
||||
body = "\n".join(body_lines)
|
||||
|
||||
with open(v3_path) as f:
|
||||
v3 = f.read()
|
||||
|
||||
begin_marker = "// ── BEGIN TRANSPILER OUTPUT"
|
||||
end_marker = "// ── END TRANSPILER OUTPUT"
|
||||
begin_idx = v3.find(begin_marker)
|
||||
end_idx = v3.find(end_marker)
|
||||
if begin_idx == -1 or end_idx == -1:
|
||||
sys.exit("markers not found in OptimizerV3.sol")
|
||||
|
||||
begin_line_end = v3.index("\n", begin_idx) + 1
|
||||
with open(v3_path, "w") as f:
|
||||
f.write(v3[:begin_line_end])
|
||||
f.write(body + "\n")
|
||||
f.write(v3[end_idx:])
|
||||
PYEOF
|
||||
Loading…
Add table
Add a link
Reference in a new issue