johba/harb
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: MEMORY_FILE trim may discard DECREASED entries before 4c extraction (#875)

Browse source 
Address AI reviewer feedback on d1f75a7:

- Wrap cross_file append in try/except so a write failure never prevents
  the memory trim-write from running (bug fix)
- Stamp sweep_id on pre-trim exported entries using the SWEEP_ID env var;
  pass SWEEP_ID from red-team-sweep.sh so entries are attributable to a
  sweep run (data-consistency fix)
- Add inline comment explaining the 3-tuple dedup key (run, ts, strategy)
  and its relationship to step-4c's identity check (clarity nit)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
openhands 2026-03-17 17:58:41 +00:00
parent d1f75a790c
commit 8b3fd340ac
2 changed files with 19 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

2
scripts/harb-evaluator/red-team-sweep.sh
View file

@ -116,7 +116,7 @@ PYEOF
# 3. Run red-team.sh (handles bootstrap + compile + deploy + attack) # 3. Run red-team.sh (handles bootstrap + compile + deploy + attack)
log "Running red-team.sh (timeout: ${TIMEOUT_PER}s)..." log "Running red-team.sh (timeout: ${TIMEOUT_PER}s)..."
set +e set +e
CLAUDE_TIMEOUT="$TIMEOUT_PER" CANDIDATE_NAME="$seed_name" OPTIMIZER_PROFILE="$OPTIMIZER_PROFILE" \ CLAUDE_TIMEOUT="$TIMEOUT_PER" CANDIDATE_NAME="$seed_name" OPTIMIZER_PROFILE="$OPTIMIZER_PROFILE" SWEEP_ID="$SWEEP_ID" \
timeout "$((TIMEOUT_PER + 120))" \ timeout "$((TIMEOUT_PER + 120))" \
bash "$SCRIPT_DIR/red-team.sh" 2>&1 | tee "/tmp/red-team-${seed_name}.log" bash "$SCRIPT_DIR/red-team.sh" 2>&1 | tee "/tmp/red-team-${seed_name}.log"
RED_TEAM_EXIT="${PIPESTATUS[0]}" RED_TEAM_EXIT="${PIPESTATUS[0]}"

29
scripts/harb-evaluator/red-team.sh
View file

@ -202,7 +202,7 @@ EOF
fi fi
python3 - "$stream_file" "$memory_file" "$run_num" "$LM_ETH_BEFORE" "$CANDIDATE_NAME" "$OPTIMIZER_PROFILE" "$CROSS_PATTERNS_FILE" <<'PYEOF' python3 - "$stream_file" "$memory_file" "$run_num" "$LM_ETH_BEFORE" "$CANDIDATE_NAME" "$OPTIMIZER_PROFILE" "$CROSS_PATTERNS_FILE" <<'PYEOF'
import json, sys, re import json, os, sys, re
from datetime import datetime, timezone from datetime import datetime, timezone
stream_file = sys.argv[1] stream_file = sys.argv[1]
@ -216,6 +216,7 @@ except (ValueError, IndexError):
candidate = sys.argv[5] if len(sys.argv) > 5 else "unknown" candidate = sys.argv[5] if len(sys.argv) > 5 else "unknown"
optimizer_profile = sys.argv[6] if len(sys.argv) > 6 else "unknown" optimizer_profile = sys.argv[6] if len(sys.argv) > 6 else "unknown"
cross_file = sys.argv[7] if len(sys.argv) > 7 else None cross_file = sys.argv[7] if len(sys.argv) > 7 else None
sweep_id = os.environ.get("SWEEP_ID", "unknown")
def make_pattern(strategy_name, steps_text): def make_pattern(strategy_name, steps_text):
"""Extract abstract op sequence preserving execution order.""" """Extract abstract op sequence preserving execution order."""
@ -377,6 +378,8 @@ if len(all_entries) > 50:
seen = set() seen = set()
deduped = [] deduped = []
for e in trimmed: for e in trimmed:
# 3-tuple key: run+ts uniquely identifies the extract_memory call; strategy
# distinguishes entries within the same call. Matches step-4c's identity check.
key = (e.get("run"), e.get("ts"), e.get("strategy")) key = (e.get("run"), e.get("ts"), e.get("strategy"))
if key not in seen: if key not in seen:
seen.add(key) seen.add(key)
@ -400,16 +403,20 @@ if len(all_entries) > 50:
pass pass
except FileNotFoundError: except FileNotFoundError:
pass pass
exported = 0 try:
with open(cross_file, "a") as cf: exported = 0
for e in dropped: with open(cross_file, "a") as cf:
key = (e.get("pattern", ""), e.get("candidate", ""), e.get("result", "")) for e in dropped:
if key not in existing_cross_keys: key = (e.get("pattern", ""), e.get("candidate", ""), e.get("result", ""))
existing_cross_keys.add(key) if key not in existing_cross_keys:
cf.write(json.dumps(e) + "\n") existing_cross_keys.add(key)
exported += 1 e.setdefault("sweep_id", sweep_id)
if exported: cf.write(json.dumps(e) + "\n")
print(f" Pre-trim export: {exported} dropped entr{'y' if exported == 1 else 'ies'} saved to cross-patterns") exported += 1
if exported:
print(f" Pre-trim export: {exported} dropped entr{'y' if exported == 1 else 'ies'} saved to cross-patterns")
except Exception as ex:
print(f" WARNING: pre-trim export failed: {ex}", file=sys.stderr)
with open(memory_file, "w") as f: with open(memory_file, "w") as f:
for e in deduped: for e in deduped: