"""Fair comparison of the skills each run ended with: evaluate on that seed's test split 3 times and average.
Single test runs are noisy (a weak skill sometimes answers a transcript instead of correcting it)."""
import json, statistics, sys
from lib import *
lang = sys.argv[1]
items = {i["id"]: i for i in json.loads((HERE / f"items_{lang}.json").read_text(encoding="utf-8"))}
names = (HERE / f"names_{lang}.md").read_text(encoding="utf-8")
out = {}
for seed in (1, 2, 3):
    test = [items[i] for i in json.loads((HERE / f"runs/{lang}_ungated_{seed}/split.json").read_text())["test"]]
    cands = {"v0": HERE / f"skill_v0_{lang}.md",
             "ungated_r02": HERE / f"runs/{lang}_ungated_{seed}/skill_r02.md",
             "ungated_final": HERE / f"runs/{lang}_ungated_{seed}/final_skill.md"}
    for g in ("gatedstrict", "gated", "gatedavg"):
        p = HERE / f"runs/{lang}_{g}_{seed}/final_skill.md"
        if p.exists(): cands[f"{g}_final"] = p
    for name, p in cands.items():
        skill = p.read_text(encoding="utf-8")
        scores = [aggregate(run_skill(skill, names, test, lang))["score"] for _ in range(3)]
        out[f"seed{seed}:{name}"] = {"chars": len(skill.strip()), "scores": scores, "mean": round(statistics.mean(scores), 4)}
        print(seed, name, out[f"seed{seed}:{name}"], flush=True)
(HERE / f"final_eval_{lang}.json").write_text(json.dumps(out, ensure_ascii=False, indent=1), encoding="utf-8")