"""Noise floor: run the same skill 3 times on all items; how much do the score and per-item verdicts move?"""
import json, sys
from lib import *
lang, skill_path = sys.argv[1], sys.argv[2]
items = json.loads((HERE / f"items_{lang}.json").read_text(encoding="utf-8"))
names = (HERE / f"names_{lang}.md").read_text(encoding="utf-8")
skill = Path(skill_path).read_text(encoding="utf-8")
runs = [run_skill(skill, names, items, lang) for _ in range(3)]
ok = [{r["id"]: not r["misses"] and not r["extras"] for r in run} for run in runs]
flips = sum(1 for i in ok[0] if len({o[i] for o in ok}) > 1)
out = {"skill": skill_path, "scores": [aggregate(r)["score"] for r in runs], "items_flipping": flips, "n_items": len(items)}
print(json.dumps(out, ensure_ascii=False))
with open(HERE / f"noise_{lang}.jsonl", "a") as f: f.write(json.dumps(out, ensure_ascii=False) + "\n")