#!/usr/bin/env bash # ============================================================================= # batch-eval.sh — revm-based batch fitness evaluator # # Replaces the per-candidate Anvil+forge-script pipeline with in-process EVM # execution via Foundry's native revm backend (FitnessEvaluator.t.sol). # # Speedup: compiles each candidate once (unavoidable — different Solidity per # candidate), then runs ALL attack sequences in a single in-process forge test # with O(1) memory snapshot/revert instead of RPC calls per attack. # # Usage: # ./tools/push3-evolution/revm-evaluator/batch-eval.sh \ # [--output-dir /tmp/scores] \ # candidate0.push3 candidate1.push3 ... # # Output (stdout): # One JSON object per candidate: # {"candidate_id":"candidate_000","fitness":123456789,"gas_used":15432} # Over-gas-limit candidates emit fitness:0 with "error":"gas_over_limit". # Downstream parsers use the "fitness" key; extra fields are ignored. # # Exit codes: # 0 Success. # 1 Candidate-level error (transpile/compile failed for at least one candidate). # 2 Infrastructure error (missing tool, BASE_RPC_URL not set, forge test failed). # # Environment: # BASE_RPC_URL Required. Base network RPC endpoint for forking. # ATTACKS_DIR Optional. Path to *.jsonl attack files. # (default: /onchain/script/backtesting/attacks) # OUTPUT_DIR Optional. Directory to copy scores.jsonl into (--output-dir overrides). # ============================================================================= set -euo pipefail export PATH="${HOME}/.foundry/bin:${PATH}" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" ONCHAIN_DIR="$REPO_ROOT/onchain" TRANSPILER_DIR="$REPO_ROOT/tools/push3-transpiler" TRANSPILER_OUT="$ONCHAIN_DIR/src/OptimizerV3Push3.sol" # Use OptimizerV3 (inherits Optimizer → UUPS compatible, has getLiquidityParams) # instead of standalone OptimizerV3Push3 which lacks UUPS hooks. OPTIMIZERV3_SOL="$ONCHAIN_DIR/src/OptimizerV3.sol" ARTIFACT_PATH="$ONCHAIN_DIR/out/OptimizerV3.sol/OptimizerV3.json" DEFAULT_ATTACKS_DIR="$ONCHAIN_DIR/script/backtesting/attacks" # ============================================================================= # Argument parsing # ============================================================================= OUTPUT_DIR="${OUTPUT_DIR:-}" declare -a PUSH3_FILES=() while [[ $# -gt 0 ]]; do case $1 in --output-dir) OUTPUT_DIR="$2"; shift 2 ;; --*) echo "Unknown option: $1" >&2; exit 2 ;; *) PUSH3_FILES+=("$1"); shift ;; esac done if [ "${#PUSH3_FILES[@]}" -eq 0 ]; then echo "Usage: $0 [--output-dir DIR] candidate1.push3 ..." >&2 exit 2 fi # ============================================================================= # Environment checks # ============================================================================= BASE_RPC_URL="${BASE_RPC_URL:-}" if [ -z "$BASE_RPC_URL" ]; then echo " [batch-eval] ERROR: BASE_RPC_URL env var required for Base network fork" >&2 exit 2 fi for _tool in forge node python3; do command -v "$_tool" &>/dev/null || { echo " [batch-eval] ERROR: $_tool not found in PATH" >&2; exit 2; } done # ============================================================================= # Helpers # ============================================================================= log() { echo " [batch-eval] $*" >&2; } fail2() { echo " [batch-eval] ERROR: $*" >&2; exit 2; } # ============================================================================= # Step 1 — Ensure transpiler dependencies are installed # ============================================================================= if [ ! -d "$TRANSPILER_DIR/node_modules" ]; then log "Installing transpiler dependencies…" (cd "$TRANSPILER_DIR" && npm install --silent) || fail2 "npm install in push3-transpiler failed" fi # ============================================================================= # Step 2 — Transpile + compile each candidate, extract bytecodes into manifest # ============================================================================= MANIFEST_DIR="$(mktemp -d)" cleanup() { [ -d "${MANIFEST_DIR:-}" ] && rm -rf "$MANIFEST_DIR" } trap cleanup EXIT IDS_FILE="$MANIFEST_DIR/ids.txt" BYTECODES_FILE="$MANIFEST_DIR/bytecodes.txt" : > "$IDS_FILE" : > "$BYTECODES_FILE" COMPILED_COUNT=0 FAILED_IDS="" FAILED_SCORES="" # Emit a fitness=0 JSON line for a candidate that failed to compile, and track it. skip_candidate() { local cid="$1" reason="$2" log "WARNING: $cid compile failed — scoring as 0" local line='{"candidate_id":"'"$cid"'","fitness":0,"error":"'"$reason"'"}' printf '%s\n' "$line" FAILED_SCORES="${FAILED_SCORES:+$FAILED_SCORES$'\n'}$line" FAILED_IDS="$FAILED_IDS $cid" } for PUSH3_FILE in "${PUSH3_FILES[@]}"; do PUSH3_FILE="$(cd "$(dirname "$PUSH3_FILE")" && pwd)/$(basename "$PUSH3_FILE")" CANDIDATE_ID="$(basename "$PUSH3_FILE" .push3)" # Transpile Push3 → Solidity, extract function body, inject into OptimizerV3.sol INJECT_SCRIPT="$REPO_ROOT/tools/push3-transpiler/inject.sh" if ! bash "$INJECT_SCRIPT" "$PUSH3_FILE" "$OPTIMIZERV3_SOL" >/dev/null 2>&1; then skip_candidate "$CANDIDATE_ID" "transpile_failed" continue fi # Compile (forge's incremental build skips unchanged files quickly) FORGE_EC=0 (cd "$ONCHAIN_DIR" && forge build --silent) >/dev/null 2>&1 || FORGE_EC=$? if [ "$FORGE_EC" -ne 0 ]; then skip_candidate "$CANDIDATE_ID" "compile_failed" continue fi # Extract bytecode from artifact (strip leading 0x if present) BYTECODE_HEX="$(python3 - "$ARTIFACT_PATH" <<'PYEOF' import json, sys with open(sys.argv[1]) as f: d = json.load(f) bytecode = d["deployedBytecode"]["object"] # Ensure 0x prefix if not bytecode.startswith("0x"): bytecode = "0x" + bytecode print(bytecode) PYEOF )" || { skip_candidate "$CANDIDATE_ID" "bytecode_extract_failed"; continue; } if [ -z "$BYTECODE_HEX" ] || [ "$BYTECODE_HEX" = "0x" ]; then skip_candidate "$CANDIDATE_ID" "empty_bytecode" continue fi printf '%s\n' "$CANDIDATE_ID" >> "$IDS_FILE" printf '%s\n' "$BYTECODE_HEX" >> "$BYTECODES_FILE" COMPILED_COUNT=$((COMPILED_COUNT + 1)) log "Compiled $CANDIDATE_ID" done if [ "$COMPILED_COUNT" -eq 0 ]; then fail2 "No candidates compiled successfully — aborting" fi log "Compiled $COMPILED_COUNT / ${#PUSH3_FILES[@]} candidates" # ============================================================================= # Step 3 — Run FitnessEvaluator.t.sol (in-process revm, all candidates at once) # ============================================================================= ATTACKS_DIR="${ATTACKS_DIR:-$DEFAULT_ATTACKS_DIR}" log "Running FitnessEvaluator.t.sol (in-process revm, fork: $BASE_RPC_URL)…" FORGE_TEST_EC=0 FORGE_OUTPUT="$( cd "$ONCHAIN_DIR" BASE_RPC_URL="$BASE_RPC_URL" \ FITNESS_MANIFEST_DIR="$MANIFEST_DIR" \ ATTACKS_DIR="$ATTACKS_DIR" \ forge test \ --match-contract FitnessEvaluator \ --match-test testBatchEvaluate \ -vv \ --no-match-path "NOT_A_REAL_PATH" \ 2>&1 )" || FORGE_TEST_EC=$? if [ "$FORGE_TEST_EC" -ne 0 ]; then # Surface forge output on failure for diagnosis printf '%s\n' "$FORGE_OUTPUT" >&2 fail2 "forge test failed (exit $FORGE_TEST_EC)" fi # ============================================================================= # Step 4 — Extract and emit score JSON lines # # forge test -vv wraps console.log output with leading spaces and a "Logs:" header. # We grep for lines containing the score JSON pattern and strip the indentation. # ============================================================================= SCORES_JSONL="$(printf '%s\n' "$FORGE_OUTPUT" | grep -E '"candidate_id"' | sed 's/^[[:space:]]*//' || true)" if [ -z "$SCORES_JSONL" ]; then printf '%s\n' "$FORGE_OUTPUT" >&2 fail2 "No score lines found in forge test output" fi # Emit scores to stdout (failed candidates already emitted individually above) printf '%s\n' "$SCORES_JSONL" # Optionally write to output directory (merge successful + failed scores) if [ -n "$OUTPUT_DIR" ]; then mkdir -p "$OUTPUT_DIR" ALL_SCORES="${SCORES_JSONL}${FAILED_SCORES:+$'\n'$FAILED_SCORES}" printf '%s\n' "$ALL_SCORES" > "$OUTPUT_DIR/scores.jsonl" log "Scores written to $OUTPUT_DIR/scores.jsonl" fi # Warn summary if any candidates were skipped (compile failures) if [ -n "$FAILED_IDS" ]; then log "WARNING: the following candidates were skipped (compile failed): $FAILED_IDS" exit 1 fi log "Done — scored $COMPILED_COUNT candidates"