diff --git a/tools/push3-evolution/revm-evaluator/batch-eval.sh b/tools/push3-evolution/revm-evaluator/batch-eval.sh index 14e3c48..6fa0a94 100755 --- a/tools/push3-evolution/revm-evaluator/batch-eval.sh +++ b/tools/push3-evolution/revm-evaluator/batch-eval.sh @@ -117,6 +117,17 @@ BYTECODES_FILE="$MANIFEST_DIR/bytecodes.txt" COMPILED_COUNT=0 FAILED_IDS="" +FAILED_SCORES="" + +# Emit a fitness=0 JSON line for a candidate that failed to compile, and track it. +skip_candidate() { + local cid="$1" reason="$2" + log "WARNING: $cid compile failed — scoring as 0" + local line='{"candidate_id":"'"$cid"'","fitness":0,"error":"'"$reason"'"}' + printf '%s\n' "$line" + FAILED_SCORES="${FAILED_SCORES:+$FAILED_SCORES$'\n'}$line" + FAILED_IDS="$FAILED_IDS $cid" +} for PUSH3_FILE in "${PUSH3_FILES[@]}"; do PUSH3_FILE="$(cd "$(dirname "$PUSH3_FILE")" && pwd)/$(basename "$PUSH3_FILE")" @@ -125,8 +136,7 @@ for PUSH3_FILE in "${PUSH3_FILES[@]}"; do # Transpile Push3 → Solidity, extract function body, inject into OptimizerV3.sol INJECT_SCRIPT="$REPO_ROOT/tools/push3-transpiler/inject.sh" if ! bash "$INJECT_SCRIPT" "$PUSH3_FILE" "$OPTIMIZERV3_SOL" >/dev/null 2>&1; then - log "WARNING: transpile/inject failed for $CANDIDATE_ID — skipping" - FAILED_IDS="$FAILED_IDS $CANDIDATE_ID" + skip_candidate "$CANDIDATE_ID" "transpile_failed" continue fi @@ -135,8 +145,7 @@ for PUSH3_FILE in "${PUSH3_FILES[@]}"; do (cd "$ONCHAIN_DIR" && forge build --silent) >/dev/null 2>&1 || FORGE_EC=$? if [ "$FORGE_EC" -ne 0 ]; then - log "WARNING: forge build failed for $CANDIDATE_ID (exit $FORGE_EC) — skipping" - FAILED_IDS="$FAILED_IDS $CANDIDATE_ID" + skip_candidate "$CANDIDATE_ID" "compile_failed" continue fi @@ -151,11 +160,10 @@ if not bytecode.startswith("0x"): bytecode = "0x" + bytecode print(bytecode) PYEOF -)" || { log "WARNING: failed to extract bytecode for $CANDIDATE_ID — skipping"; FAILED_IDS="$FAILED_IDS $CANDIDATE_ID"; continue; } +)" || { skip_candidate "$CANDIDATE_ID" "bytecode_extract_failed"; continue; } if [ -z "$BYTECODE_HEX" ] || [ "$BYTECODE_HEX" = "0x" ]; then - log "WARNING: empty bytecode for $CANDIDATE_ID — skipping" - FAILED_IDS="$FAILED_IDS $CANDIDATE_ID" + skip_candidate "$CANDIDATE_ID" "empty_bytecode" continue fi @@ -213,17 +221,18 @@ if [ -z "$SCORES_JSONL" ]; then fail2 "No score lines found in forge test output" fi -# Emit scores to stdout +# Emit scores to stdout (failed candidates already emitted individually above) printf '%s\n' "$SCORES_JSONL" -# Optionally write to output directory +# Optionally write to output directory (merge successful + failed scores) if [ -n "$OUTPUT_DIR" ]; then mkdir -p "$OUTPUT_DIR" - printf '%s\n' "$SCORES_JSONL" > "$OUTPUT_DIR/scores.jsonl" + ALL_SCORES="${SCORES_JSONL}${FAILED_SCORES:+$'\n'$FAILED_SCORES}" + printf '%s\n' "$ALL_SCORES" > "$OUTPUT_DIR/scores.jsonl" log "Scores written to $OUTPUT_DIR/scores.jsonl" fi -# Warn about any candidates that were skipped (compile failures) +# Warn summary if any candidates were skipped (compile failures) if [ -n "$FAILED_IDS" ]; then log "WARNING: the following candidates were skipped (compile failed): $FAILED_IDS" exit 1