johba/harb
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request 'fix: MEMORY_FILE trim may discard DECREASED entries before 4c extraction (#875)' (#926) from fix/issue-875 into master

Browse source
This commit is contained in:
johba 2026-03-17 19:27:09 +01:00
commit 415e94243e
2 changed files with 41 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

2
scripts/harb-evaluator/red-team-sweep.sh
View file

@ -116,7 +116,7 @@ PYEOF
# 3. Run red-team.sh (handles bootstrap + compile + deploy + attack) # 3. Run red-team.sh (handles bootstrap + compile + deploy + attack)
log "Running red-team.sh (timeout: ${TIMEOUT_PER}s)..." log "Running red-team.sh (timeout: ${TIMEOUT_PER}s)..."
set +e set +e
CLAUDE_TIMEOUT="$TIMEOUT_PER" CANDIDATE_NAME="$seed_name" OPTIMIZER_PROFILE="$OPTIMIZER_PROFILE" \ CLAUDE_TIMEOUT="$TIMEOUT_PER" CANDIDATE_NAME="$seed_name" OPTIMIZER_PROFILE="$OPTIMIZER_PROFILE" SWEEP_ID="$SWEEP_ID" \
timeout "$((TIMEOUT_PER + 120))" \ timeout "$((TIMEOUT_PER + 120))" \
bash "$SCRIPT_DIR/red-team.sh" 2>&1 | tee "/tmp/red-team-${seed_name}.log" bash "$SCRIPT_DIR/red-team.sh" 2>&1 | tee "/tmp/red-team-${seed_name}.log"
RED_TEAM_EXIT="${PIPESTATUS[0]}" RED_TEAM_EXIT="${PIPESTATUS[0]}"

42
scripts/harb-evaluator/red-team.sh
View file

@ -201,8 +201,8 @@ EOF
run_num=1 run_num=1
fi fi
python3 - "$stream_file" "$memory_file" "$run_num" "$LM_ETH_BEFORE" "$CANDIDATE_NAME" "$OPTIMIZER_PROFILE" <<'PYEOF' python3 - "$stream_file" "$memory_file" "$run_num" "$LM_ETH_BEFORE" "$CANDIDATE_NAME" "$OPTIMIZER_PROFILE" "$CROSS_PATTERNS_FILE" <<'PYEOF'
import json, sys, re import json, os, sys, re
from datetime import datetime, timezone from datetime import datetime, timezone
stream_file = sys.argv[1] stream_file = sys.argv[1]
@ -215,6 +215,8 @@ except (ValueError, IndexError):
sys.exit(0) sys.exit(0)
candidate = sys.argv[5] if len(sys.argv) > 5 else "unknown" candidate = sys.argv[5] if len(sys.argv) > 5 else "unknown"
optimizer_profile = sys.argv[6] if len(sys.argv) > 6 else "unknown" optimizer_profile = sys.argv[6] if len(sys.argv) > 6 else "unknown"
cross_file = sys.argv[7] if len(sys.argv) > 7 else None
sweep_id = os.environ.get("SWEEP_ID", "unknown")
def make_pattern(strategy_name, steps_text): def make_pattern(strategy_name, steps_text):
"""Extract abstract op sequence preserving execution order.""" """Extract abstract op sequence preserving execution order."""
@ -376,10 +378,46 @@ if len(all_entries) > 50:
seen = set() seen = set()
deduped = [] deduped = []
for e in trimmed: for e in trimmed:
# 3-tuple key: run+ts uniquely identifies the extract_memory call; strategy
# distinguishes entries within the same call. Matches step-4c's identity check.
key = (e.get("run"), e.get("ts"), e.get("strategy")) key = (e.get("run"), e.get("ts"), e.get("strategy"))
if key not in seen: if key not in seen:
seen.add(key) seen.add(key)
deduped.append(e) deduped.append(e)
# Export entries that would be dropped to cross-patterns before discarding them
if cross_file:
kept_keys = {(e.get("run"), e.get("ts"), e.get("strategy")) for e in deduped}
dropped = [e for e in all_entries if (e.get("run"), e.get("ts"), e.get("strategy")) not in kept_keys]
if dropped:
existing_cross_keys = set()
try:
with open(cross_file) as cf:
for line in cf:
line = line.strip()
if line:
try:
ce = json.loads(line)
existing_cross_keys.add((ce.get("pattern", ""), ce.get("candidate", ""), ce.get("result", "")))
except Exception:
pass
except FileNotFoundError:
pass
try:
exported = 0
with open(cross_file, "a") as cf:
for e in dropped:
key = (e.get("pattern", ""), e.get("candidate", ""), e.get("result", ""))
if key not in existing_cross_keys:
existing_cross_keys.add(key)
e.setdefault("sweep_id", sweep_id)
cf.write(json.dumps(e) + "\n")
exported += 1
if exported:
print(f" Pre-trim export: {exported} dropped entr{'y' if exported == 1 else 'ies'} saved to cross-patterns")
except Exception as ex:
print(f" WARNING: pre-trim export failed: {ex}", file=sys.stderr)
with open(memory_file, "w") as f: with open(memory_file, "w") as f:
for e in deduped: for e in deduped:
f.write(json.dumps(e) + "\n") f.write(json.dumps(e) + "\n")