"""Aggregate automatic scores per model and language (manual review is in results/manual_review.md)."""
import json, statistics as st, sys
from pathlib import Path
R = Path(__file__).parent / "results"
# python3 summarize.py [suffix]: only answer files ending in that set suffix, e.g. "lang2" (default: the first test).
SUF = sys.argv[1] if len(sys.argv) > 1 else ""
files = [f for f in sorted(R.glob("answers_*.json")) if (f.stem.endswith("_" + SUF) if SUF else not f.stem.endswith("_lang2"))]
for f in files:
    d = json.loads(f.read_text()); rows = d["rows"]
    print("==", d["meta"]["model"], "load_s", d["meta"]["load_s"], "peak_ram_mb", d["meta"]["peak_ram_mb_all_ollama"])
    for lang in list(dict.fromkeys(r["lang"] for r in rows)) + ["all"]:
        rs = [r for r in rows if lang == "all" or r["lang"] == lang]
        ans = [r for r in rs if "hit_top3" in r["score"]]
        oos = [r for r in rs if "hit_top3" not in r["score"]]
        c = lambda xs, k: f"{sum(x['score'][k] for x in xs)}/{len(xs)}"
        print(f" {lang}: top1 {c(ans,'hit_top1')} top3 {c(ans,'hit_top3')} | answered(not wrongly refused) {c(ans,'refusal_ok')} | "
              f"refused out-of-library {c(oos,'refusal_ok')} | citation_ok {c(ans,'citation_ok')} | must {c(ans,'must_ok')} neg {c(ans,'neg_ok')} | lang_auto {c(rs,'lang_ok')}")
    p = [r["perf"] for r in rows]
    print(f" median ttft {st.median(x['ttft_s'] for x in p)} s, median total {st.median(x['total_s'] for x in p)} s, median tok/s {st.median(x['tok_per_s'] for x in p if x['tok_per_s'])}, median prompt tok/s {st.median(x['prompt_tok_per_s'] for x in p if x['prompt_tok_per_s'])}")