"""Which items flip between identical runs, and why? Runs a skill 3 times on all items like noise.py, but keeps every
output so the flipping items can be read. Writes noise_trace_<lang>.json (scores, flipping ids, per-run output and
verdict for each flipping item) and prints a summary. A run's output is tagged "answered" when it talks about the
input instead of returning a corrected transcript (the failure mode seen in the 3.2/3.4 baseline).
Usage: noise_trace.py <lang> <skill_path>"""
import json, re, sys
from lib import *

lang, skill_path = sys.argv[1], sys.argv[2]
items = json.loads((HERE / f"items_{lang}.json").read_text(encoding="utf-8"))
names = (HERE / f"names_{lang}.md").read_text(encoding="utf-8")
skill = Path(skill_path).read_text(encoding="utf-8")
runs = [run_skill(skill, names, items, lang) for _ in range(3)]
ok = [{r["id"]: not r["misses"] and not r["extras"] for r in run} for run in runs]
flip_ids = [i for i in ok[0] if len({o[i] for o in ok}) > 1]
ANSWERED = re.compile(r"(no|n't have a|not a raw) (speech-to-text )?transcript|nothing (for me )?to (correct|edit)|your (message|question) is", re.I)
by_id = [{r["id"]: r for r in run} for run in runs]
detail = {i: [{"pass": ok[k][i], "answered": bool(ANSWERED.search(by_id[k][i]["output"])),
               "misses": by_id[k][i]["misses"], "extras": by_id[k][i]["extras"], "output": by_id[k][i]["output"]}
              for k in range(3)] for i in flip_ids}
answered_any = {k: sorted(i for i in by_id[k] if ANSWERED.search(by_id[k][i]["output"])) for k in range(3)}
out = {"skill": skill_path, "scores": [aggregate(r)["score"] for r in runs], "flipping": flip_ids,
       "answered_per_run": answered_any, "detail": detail}
(HERE / f"noise_trace_{lang}.json").write_text(json.dumps(out, ensure_ascii=False, indent=1), encoding="utf-8")
print("scores", out["scores"])
print("flipping", flip_ids)
for i in flip_ids:
    print(i, [(d["pass"], d["answered"], d["misses"], d["extras"]) for d in detail[i]])
print("answered per run", {k: v for k, v in answered_any.items()})