a23064f576
- Add skip_candidate() helper that emits fitness=0 JSON to stdout and tracks the failed score for the output-dir file, satisfying the downstream scorer's expectation of one JSON line per candidate. - Unify all failure paths (transpile, forge build, bytecode extract, empty bytecode) through skip_candidate() with a distinct error key. - Log message now reads "WARNING: <id> compile failed — scoring as 0" as required by the acceptance criteria. - Output-dir scores.jsonl now merges successful + failed scores so the file is complete even when some candidates fail to compile. - All-candidates-fail path (COMPILED_COUNT=0) still exits 2 (no viable population); true infra errors (missing tool, bad RPC) unchanged. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
241 lines
8.6 KiB
Bash
Executable file
241 lines
8.6 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# =============================================================================
|
|
# batch-eval.sh — revm-based batch fitness evaluator
|
|
#
|
|
# Replaces the per-candidate Anvil+forge-script pipeline with in-process EVM
|
|
# execution via Foundry's native revm backend (FitnessEvaluator.t.sol).
|
|
#
|
|
# Speedup: compiles each candidate once (unavoidable — different Solidity per
|
|
# candidate), then runs ALL attack sequences in a single in-process forge test
|
|
# with O(1) memory snapshot/revert instead of RPC calls per attack.
|
|
#
|
|
# Usage:
|
|
# ./tools/push3-evolution/revm-evaluator/batch-eval.sh \
|
|
# [--output-dir /tmp/scores] \
|
|
# candidate0.push3 candidate1.push3 ...
|
|
#
|
|
# Output (stdout):
|
|
# One JSON object per candidate:
|
|
# {"candidate_id":"candidate_000","fitness":123456789,"gas_used":15432}
|
|
# Over-gas-limit candidates emit fitness:0 with "error":"gas_over_limit".
|
|
# Downstream parsers use the "fitness" key; extra fields are ignored.
|
|
#
|
|
# Exit codes:
|
|
# 0 Success.
|
|
# 1 Candidate-level error (transpile/compile failed for at least one candidate).
|
|
# 2 Infrastructure error (missing tool, BASE_RPC_URL not set, forge test failed).
|
|
#
|
|
# Environment:
|
|
# BASE_RPC_URL Required. Base network RPC endpoint for forking.
|
|
# ATTACKS_DIR Optional. Path to *.jsonl attack files.
|
|
# (default: <repo>/onchain/script/backtesting/attacks)
|
|
# OUTPUT_DIR Optional. Directory to copy scores.jsonl into (--output-dir overrides).
|
|
# =============================================================================
|
|
|
|
set -euo pipefail
|
|
|
|
export PATH="${HOME}/.foundry/bin:${PATH}"
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
|
ONCHAIN_DIR="$REPO_ROOT/onchain"
|
|
TRANSPILER_DIR="$REPO_ROOT/tools/push3-transpiler"
|
|
TRANSPILER_OUT="$ONCHAIN_DIR/src/OptimizerV3Push3.sol"
|
|
# Use OptimizerV3 (inherits Optimizer → UUPS compatible, has getLiquidityParams)
|
|
# instead of standalone OptimizerV3Push3 which lacks UUPS hooks.
|
|
OPTIMIZERV3_SOL="$ONCHAIN_DIR/src/OptimizerV3.sol"
|
|
ARTIFACT_PATH="$ONCHAIN_DIR/out/OptimizerV3.sol/OptimizerV3.json"
|
|
DEFAULT_ATTACKS_DIR="$ONCHAIN_DIR/script/backtesting/attacks"
|
|
|
|
# =============================================================================
|
|
# Argument parsing
|
|
# =============================================================================
|
|
|
|
OUTPUT_DIR="${OUTPUT_DIR:-}"
|
|
|
|
declare -a PUSH3_FILES=()
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case $1 in
|
|
--output-dir) OUTPUT_DIR="$2"; shift 2 ;;
|
|
--*) echo "Unknown option: $1" >&2; exit 2 ;;
|
|
*) PUSH3_FILES+=("$1"); shift ;;
|
|
esac
|
|
done
|
|
|
|
if [ "${#PUSH3_FILES[@]}" -eq 0 ]; then
|
|
echo "Usage: $0 [--output-dir DIR] candidate1.push3 ..." >&2
|
|
exit 2
|
|
fi
|
|
|
|
# =============================================================================
|
|
# Environment checks
|
|
# =============================================================================
|
|
|
|
BASE_RPC_URL="${BASE_RPC_URL:-}"
|
|
if [ -z "$BASE_RPC_URL" ]; then
|
|
echo " [batch-eval] ERROR: BASE_RPC_URL env var required for Base network fork" >&2
|
|
exit 2
|
|
fi
|
|
|
|
for _tool in forge node python3; do
|
|
command -v "$_tool" &>/dev/null || { echo " [batch-eval] ERROR: $_tool not found in PATH" >&2; exit 2; }
|
|
done
|
|
|
|
# =============================================================================
|
|
# Helpers
|
|
# =============================================================================
|
|
|
|
log() { echo " [batch-eval] $*" >&2; }
|
|
fail2() { echo " [batch-eval] ERROR: $*" >&2; exit 2; }
|
|
|
|
# =============================================================================
|
|
# Step 1 — Ensure transpiler dependencies are installed
|
|
# =============================================================================
|
|
|
|
if [ ! -d "$TRANSPILER_DIR/node_modules" ]; then
|
|
log "Installing transpiler dependencies…"
|
|
(cd "$TRANSPILER_DIR" && npm install --silent) || fail2 "npm install in push3-transpiler failed"
|
|
fi
|
|
|
|
# =============================================================================
|
|
# Step 2 — Transpile + compile each candidate, extract bytecodes into manifest
|
|
# =============================================================================
|
|
|
|
MANIFEST_DIR="$(mktemp -d)"
|
|
|
|
cleanup() {
|
|
[ -d "${MANIFEST_DIR:-}" ] && rm -rf "$MANIFEST_DIR"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
IDS_FILE="$MANIFEST_DIR/ids.txt"
|
|
BYTECODES_FILE="$MANIFEST_DIR/bytecodes.txt"
|
|
|
|
: > "$IDS_FILE"
|
|
: > "$BYTECODES_FILE"
|
|
|
|
COMPILED_COUNT=0
|
|
FAILED_IDS=""
|
|
FAILED_SCORES=""
|
|
|
|
# Emit a fitness=0 JSON line for a candidate that failed to compile, and track it.
|
|
skip_candidate() {
|
|
local cid="$1" reason="$2"
|
|
log "WARNING: $cid compile failed — scoring as 0"
|
|
local line='{"candidate_id":"'"$cid"'","fitness":0,"error":"'"$reason"'"}'
|
|
printf '%s\n' "$line"
|
|
FAILED_SCORES="${FAILED_SCORES:+$FAILED_SCORES$'\n'}$line"
|
|
FAILED_IDS="$FAILED_IDS $cid"
|
|
}
|
|
|
|
for PUSH3_FILE in "${PUSH3_FILES[@]}"; do
|
|
PUSH3_FILE="$(cd "$(dirname "$PUSH3_FILE")" && pwd)/$(basename "$PUSH3_FILE")"
|
|
CANDIDATE_ID="$(basename "$PUSH3_FILE" .push3)"
|
|
|
|
# Transpile Push3 → Solidity, extract function body, inject into OptimizerV3.sol
|
|
INJECT_SCRIPT="$REPO_ROOT/tools/push3-transpiler/inject.sh"
|
|
if ! bash "$INJECT_SCRIPT" "$PUSH3_FILE" "$OPTIMIZERV3_SOL" >/dev/null 2>&1; then
|
|
skip_candidate "$CANDIDATE_ID" "transpile_failed"
|
|
continue
|
|
fi
|
|
|
|
# Compile (forge's incremental build skips unchanged files quickly)
|
|
FORGE_EC=0
|
|
(cd "$ONCHAIN_DIR" && forge build --silent) >/dev/null 2>&1 || FORGE_EC=$?
|
|
|
|
if [ "$FORGE_EC" -ne 0 ]; then
|
|
skip_candidate "$CANDIDATE_ID" "compile_failed"
|
|
continue
|
|
fi
|
|
|
|
# Extract bytecode from artifact (strip leading 0x if present)
|
|
BYTECODE_HEX="$(python3 - "$ARTIFACT_PATH" <<'PYEOF'
|
|
import json, sys
|
|
with open(sys.argv[1]) as f:
|
|
d = json.load(f)
|
|
bytecode = d["deployedBytecode"]["object"]
|
|
# Ensure 0x prefix
|
|
if not bytecode.startswith("0x"):
|
|
bytecode = "0x" + bytecode
|
|
print(bytecode)
|
|
PYEOF
|
|
)" || { skip_candidate "$CANDIDATE_ID" "bytecode_extract_failed"; continue; }
|
|
|
|
if [ -z "$BYTECODE_HEX" ] || [ "$BYTECODE_HEX" = "0x" ]; then
|
|
skip_candidate "$CANDIDATE_ID" "empty_bytecode"
|
|
continue
|
|
fi
|
|
|
|
printf '%s\n' "$CANDIDATE_ID" >> "$IDS_FILE"
|
|
printf '%s\n' "$BYTECODE_HEX" >> "$BYTECODES_FILE"
|
|
COMPILED_COUNT=$((COMPILED_COUNT + 1))
|
|
log "Compiled $CANDIDATE_ID"
|
|
done
|
|
|
|
if [ "$COMPILED_COUNT" -eq 0 ]; then
|
|
fail2 "No candidates compiled successfully — aborting"
|
|
fi
|
|
|
|
log "Compiled $COMPILED_COUNT / ${#PUSH3_FILES[@]} candidates"
|
|
|
|
# =============================================================================
|
|
# Step 3 — Run FitnessEvaluator.t.sol (in-process revm, all candidates at once)
|
|
# =============================================================================
|
|
|
|
ATTACKS_DIR="${ATTACKS_DIR:-$DEFAULT_ATTACKS_DIR}"
|
|
|
|
log "Running FitnessEvaluator.t.sol (in-process revm, fork: $BASE_RPC_URL)…"
|
|
|
|
FORGE_TEST_EC=0
|
|
FORGE_OUTPUT="$(
|
|
cd "$ONCHAIN_DIR"
|
|
BASE_RPC_URL="$BASE_RPC_URL" \
|
|
FITNESS_MANIFEST_DIR="$MANIFEST_DIR" \
|
|
ATTACKS_DIR="$ATTACKS_DIR" \
|
|
forge test \
|
|
--match-contract FitnessEvaluator \
|
|
--match-test testBatchEvaluate \
|
|
-vv \
|
|
--no-match-path "NOT_A_REAL_PATH" \
|
|
2>&1
|
|
)" || FORGE_TEST_EC=$?
|
|
|
|
if [ "$FORGE_TEST_EC" -ne 0 ]; then
|
|
# Surface forge output on failure for diagnosis
|
|
printf '%s\n' "$FORGE_OUTPUT" >&2
|
|
fail2 "forge test failed (exit $FORGE_TEST_EC)"
|
|
fi
|
|
|
|
# =============================================================================
|
|
# Step 4 — Extract and emit score JSON lines
|
|
#
|
|
# forge test -vv wraps console.log output with leading spaces and a "Logs:" header.
|
|
# We grep for lines containing the score JSON pattern and strip the indentation.
|
|
# =============================================================================
|
|
|
|
SCORES_JSONL="$(printf '%s\n' "$FORGE_OUTPUT" | grep -E '"candidate_id"' | sed 's/^[[:space:]]*//' || true)"
|
|
|
|
if [ -z "$SCORES_JSONL" ]; then
|
|
printf '%s\n' "$FORGE_OUTPUT" >&2
|
|
fail2 "No score lines found in forge test output"
|
|
fi
|
|
|
|
# Emit scores to stdout (failed candidates already emitted individually above)
|
|
printf '%s\n' "$SCORES_JSONL"
|
|
|
|
# Optionally write to output directory (merge successful + failed scores)
|
|
if [ -n "$OUTPUT_DIR" ]; then
|
|
mkdir -p "$OUTPUT_DIR"
|
|
ALL_SCORES="${SCORES_JSONL}${FAILED_SCORES:+$'\n'$FAILED_SCORES}"
|
|
printf '%s\n' "$ALL_SCORES" > "$OUTPUT_DIR/scores.jsonl"
|
|
log "Scores written to $OUTPUT_DIR/scores.jsonl"
|
|
fi
|
|
|
|
# Warn summary if any candidates were skipped (compile failures)
|
|
if [ -n "$FAILED_IDS" ]; then
|
|
log "WARNING: the following candidates were skipped (compile failed): $FAILED_IDS"
|
|
exit 1
|
|
fi
|
|
|
|
log "Done — scored $COMPILED_COUNT candidates"
|