johba/harb
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: address review findings for sweep-results.tsv (#818)

Browse source 
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
openhands 2026-03-15 20:48:33 +00:00
parent 3c6be7d86f
commit ae3eb14833
1 changed files with 15 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

27
scripts/harb-evaluator/red-team-sweep.sh
View file

@ -127,7 +127,8 @@ PYEOF
fi fi
fi fi
# 4c. Write one TSV row to sweep-results.tsv # 4b. Write one TSV row to sweep-results.tsv
# NOTE: intentionally runs before 4c (memory clear) so strategy data is still available.
if [[ "$RED_TEAM_EXIT" -eq 0 ]]; then if [[ "$RED_TEAM_EXIT" -eq 0 ]]; then
_sweep_status="safe" _sweep_status="safe"
elif [[ "$RED_TEAM_EXIT" -eq 1 ]]; then elif [[ "$RED_TEAM_EXIT" -eq 1 ]]; then
@ -147,17 +148,17 @@ candidate = sys.argv[3]
status = sys.argv[4] status = sys.argv[4]
tsv_file = sys.argv[5] tsv_file = sys.argv[5]
# Parse eth_before and eth_after from the candidate log # Parse eth_before (first occurrence = baseline) and eth_after (last occurrence = final state)
eth_before = "" eth_before = ""
eth_after = "" eth_after = ""
try: try:
with open(log_file) as f: with open(log_file) as f:
for line in f: for line in f:
m = re.search(r'lm_eth_before\s*[=:]\s*(\d+)', line) m = re.search(r'lm_eth_before\s*[=:]\s*(\d+)', line)
if m: if m and not eth_before: # first occurrence wins
eth_before = m.group(1) eth_before = m.group(1)
m = re.search(r'lm_eth_after\s*[=:]\s*(\d+)', line) m = re.search(r'lm_eth_after\s*[=:]\s*(\d+)', line)
if m: if m: # last occurrence wins
eth_after = m.group(1) eth_after = m.group(1)
except Exception as e: except Exception as e:
print(f" tsv: could not read log: {e}", file=sys.stderr) print(f" tsv: could not read log: {e}", file=sys.stderr)
@ -170,7 +171,7 @@ try:
with open(mem_file) as f: with open(mem_file) as f:
entries = [json.loads(l) for l in f if l.strip()] entries = [json.loads(l) for l in f if l.strip()]
cand_entries = [e for e in entries if e.get("candidate") == candidate] cand_entries = [e for e in entries if e.get("candidate") == candidate]
strategies_tried = len(set(e.get("strategy", "") for e in cand_entries)) strategies_tried = len(set(e["strategy"] for e in cand_entries if e.get("strategy")))
best_delta = 0 best_delta = 0
for e in cand_entries: for e in cand_entries:
if e.get("result") == "DECREASED" and e.get("delta_bps", 0) < best_delta: if e.get("result") == "DECREASED" and e.get("delta_bps", 0) < best_delta:
@ -180,17 +181,19 @@ try:
except Exception as e: except Exception as e:
print(f" tsv: could not read memory: {e}", file=sys.stderr) print(f" tsv: could not read memory: {e}", file=sys.stderr)
# Compute pct_extracted # Compute pct_extracted; use sentinel when ETH values are absent (crash/early-timeout)
pct_extracted = "0.00" if not eth_before and not eth_after:
try: pct_extracted = ""
if eth_before and eth_after: else:
pct_extracted = "0.00"
try:
before = int(eth_before) before = int(eth_before)
after = int(eth_after) after = int(eth_after)
if before > 0: if before > 0:
extracted = max(0, before - after) extracted = max(0, before - after)
pct_extracted = f"{extracted * 100 / before:.2f}" pct_extracted = f"{extracted * 100 / before:.2f}"
except Exception: except Exception:
pass pass
# Sanitise fields: strip tabs so the row is always valid TSV # Sanitise fields: strip tabs so the row is always valid TSV
def clean(s): def clean(s):
@ -209,7 +212,7 @@ PYEOF
set -e set -e
[[ $_py_exit -ne 0 ]] && log "WARNING: TSV row write failed (exit $_py_exit) — continuing" [[ $_py_exit -ne 0 ]] && log "WARNING: TSV row write failed (exit $_py_exit) — continuing"
# 4b. Extract abstract patterns into cross-candidate file, then clear raw memory # 4c. Extract abstract patterns into cross-candidate file, then clear raw memory
if [[ -f "$MEMORY_FILE" && -s "$MEMORY_FILE" ]]; then if [[ -f "$MEMORY_FILE" && -s "$MEMORY_FILE" ]]; then
set +e set +e
_extract_out=$(python3 - "$MEMORY_FILE" "$CROSS_PATTERNS_FILE" <<'PYEOF' _extract_out=$(python3 - "$MEMORY_FILE" "$CROSS_PATTERNS_FILE" <<'PYEOF'