# python3 score.py [--misses] [--json <out>]  Scores each run: share of the 3 planted flaws its review names, plus tokens, cost, time.
import glob, json, os, re, statistics as st, sys, collections

HERE = os.path.dirname(os.path.abspath(__file__))
PAPERS = {p["id"]: p for p in json.load(open(f"{HERE}/papers.json"))}
I = re.I | re.S
RULES = {
    "causal": r"causa|cross-sectional|associational|reverse caus|confound",
    "overgen": r"generali[sz]|external validity|representative|extrapolat|(well )?beyond (the |its |this |one )?(sample|study|single|one)|one university|single university|go(es)? (well |far )?beyond",
    "no_control": r"control group|comparison group|no control|uncontrolled|pre[- ]?post|before[- ]and[- ]after|counterfactual|without a control",
    "attrition": r"attrition|drop[- ]?outs?|dropped out|lost to follow|missing data|complete[- ]case|intention[- ]to[- ]treat|\bITT\b|survivor|missing (outcome|visit)|excluded.{0,80}(stopp|missed|non[- ]?complet)",
    "selfreport": r"self[- ]report",
    "leakage": r"leak|test (set|split|data).{0,120}(tun|select|hyperparam|optimi)|(tun|select|hyperparam|optimi).{0,120}test (set|split|data)|validation (set|split)",
    "baseline": r"baseline.{0,120}(untuned|not tuned|no tuning|default)|(untuned|default (settings|hyperparam)|no tuning|not tuned).{0,120}baseline|(un)?fair comparison|equal tuning|tuning (budget|effort)|asymmetric",
    "single_seed": r"single (random )?(seed|run)|one (random )?seed|seed 42|multiple (random )?(seeds|runs)|trained (each model )?once|variance across",
    "multiple": r"multiple (comparison|testing|test|hypothes)|bonferroni|false discovery|\bFDR\b|family[- ]?wise|p[- ]?hack|forking|correct(ion|ed|ing) for (multiple|multiplicity)|multiplicity",
    "p_misread": r"not (statistically )?significant|non[- ]?significant|(cross|span|includ)\w* (zero|0)\b|marginal|above (0\.05|alpha|the (conventional )?threshold)|does not reach significance|misinterpret",
    "regression_mean": r"regression (to|toward|towards) the mean",
    "selection": r"self[- ]select|selection bias|non[- ]?random|not random|randomi[sz]|volunteer bias",
    "pseudorep": r"pseudo[- ]?replicat|not independent|non[- ]?independen|single field|one field|\bn ?= ?1\b|unit of (analysis|replication)",
    "blinding": r"blind|masked|knew (each|the|which).{0,40}group|assessor|detection bias|observer bias|rater bias|ascertainment",
}
# Per-paper checks for numbers that do not add up.
SPECIFIC = {
    ("p01", "n_mismatch"): r"1,?240.{0,400}(984|1,?520)|(984|1,?520).{0,400}1,?240",
    ("p06", "n_mismatch"): r"3,?400.{0,400}2,?150|2,?150.{0,400}3,?400",
    ("p09", "n_mismatch"): r"\b64\b.{0,400}\b40\b|\b40\b.{0,400}\b64\b",
    ("p05", "arithmetic"): r"(62|45) ?%.{0,300}(10[0-9]|add up|sum|exceed|more than 100|over 100|impossible|total)|10[0-9] ?%",
    ("p10", "arithmetic"): r"(41|33|38) ?%.{0,300}(11[0-9]|add up|sum|exceed|more than 100|over 100|impossible|total)|11[0-9] ?%",
}


def load(path):
    L = [json.loads(l) for l in open(path) if l.strip()]
    res = next((l for l in L if l.get("type") == "result"), None)
    files = L[-1]["files"] if L and L[-1].get("type") == "files" else {}
    text = (res or {}).get("result", "") + "\n" + "\n".join(files.values())
    skill_tool = any(c.get("type") == "tool_use" and c["name"] == "Skill"
                     for l in L if l.get("type") == "assistant" for c in l["message"]["content"])
    return res, text, skill_tool, any(l.get("type") == "timeout" for l in L)


def found(pid, flaw, text):
    return bool(re.search(SPECIFIC.get((pid, flaw), RULES.get(flaw, "$^")), text, I))


rows = collections.defaultdict(list)
misses = []
for path in sorted(glob.glob(f"{HERE}/runs/*.jsonl")):
    name = os.path.basename(path)[:-6]
    arm, pid, rep = name.rsplit("-", 2)
    res, text, skill_tool, timeout = load(path)
    flaws = PAPERS[pid]["flaws"]
    hit = [f for f in flaws if found(pid, f, text)]
    misses += [(name, f) for f in flaws if f not in hit]
    u = (res or {}).get("usage", {})
    rows[arm].append(dict(run=name, found=hit, missed=[f for f in flaws if f not in hit], hit=len(hit), n=len(flaws), ok=res is not None and not res.get("is_error") and not timeout,
                          out=u.get("output_tokens", 0), cost=(res or {}).get("total_cost_usd", 0),
                          sec=(res or {}).get("duration_ms", 0) / 1000, turns=(res or {}).get("num_turns", 0),
                          words=len(text.split()), skill_tool=skill_tool, pid=pid))

for arm, R in rows.items():
    h, n = sum(r["hit"] for r in R), sum(r["n"] for r in R)
    med = lambda k: st.median(r[k] for r in R)
    print(f"{arm:11} runs {len(R):3} ok {sum(r['ok'] for r in R):3}  flaws {h}/{n} = {h/n:.0%}  "
          f"all3 {sum(r['hit']==r['n'] for r in R)}  out_tok med {med('out'):.0f}  words med {med('words'):.0f}  "
          f"cost med ${med('cost'):.4f} sum ${sum(r['cost'] for r in R):.3f}  sec med {med('sec'):.0f}  "
          f"turns med {med('turns'):.0f}  Skill tool {sum(r['skill_tool'] for r in R)}")
by = collections.defaultdict(lambda: collections.Counter())
for name, f in misses:
    by[name.rsplit("-", 2)[0]][f] += 1
print({a: dict(c) for a, c in by.items()})
if "--misses" in sys.argv:
    for m in misses:
        print("miss", *m)

if "--json" in sys.argv:
    summary = {}
    for arm, R in rows.items():
        h, n = sum(r["hit"] for r in R), sum(r["n"] for r in R)
        summary[arm] = dict(runs=len(R), flaws_found=h, flaws_total=n,
                            median_output_tokens=st.median(r["out"] for r in R),
                            mean_cost_usd=round(st.mean(r["cost"] for r in R), 5),
                            median_seconds=round(st.median(r["sec"] for r in R), 1))
    json.dump(dict(date="2026-11-14", model="claude-haiku-5-5 via claude -p, Claude Code 2.1.322",
                   prompt="see PROMPT in the run script; ars and kdense arms prefix /academic-paper-reviewer and /peer-review",
                   rules=RULES, specific={f"{k[0]}:{k[1]}": v for k, v in SPECIFIC.items()},
                   papers=list(PAPERS.values()), summary=summary,
                   runs=[r for R in rows.values() for r in R]),
              open(sys.argv[sys.argv.index("--json") + 1], "w"), indent=1)