"""Re-measure the ungated curve: each round's skill on the 140 unseen items (holdout + test), two more runs,
combined with the run already in log.json -> mean of three. Single runs are noisy, especially for v0."""
import json, statistics, sys
from lib import *
lang = sys.argv[1]
items = {i["id"]: i for i in json.loads((HERE / f"items_{lang}.json").read_text(encoding="utf-8"))}
names = (HERE / f"names_{lang}.md").read_text(encoding="utf-8")
out = {}
for seed in (1, 2, 3):
    d = HERE / f"runs/{lang}_ungated_{seed}"
    sp = json.loads((d / "split.json").read_text())
    unseen = [items[i] for i in sp["holdout"] + sp["test"]]
    tune = [items[i] for i in sp["tune"]]
    log = json.loads((d / "log.json").read_text())
    for e in log:
        r = e["round"]
        skill = (d / f"skill_r{r:02d}.md").read_text(encoding="utf-8")
        h, t = e["holdout"], e["test"]
        first = (h["correct"] + t["correct"] - h["inserts"] - t["inserts"]) / (h["total"] + t["total"])
        more = [aggregate(run_skill(skill, names, unseen, lang))["score"] for _ in range(2)]
        tmore = [aggregate(run_skill(skill, names, tune, lang))["score"] for _ in range(2)]
        scores = [round(first, 4)] + more
        tscores = [e["tune"]["score"]] + tmore
        out[f"seed{seed}:r{r}"] = {"unseen": scores, "unseen_mean": round(statistics.mean(scores), 4),
                                    "tune": tscores, "tune_mean": round(statistics.mean(tscores), 4), "chars": e["skill_chars"]}
        print(seed, r, out[f"seed{seed}:r{r}"], flush=True)
        (HERE / f"curve_reeval_{lang}.json").write_text(json.dumps(out, indent=1), encoding="utf-8")