"""2.7: grade every raw output with the gpt-5 judge and aggregate. grade.py judge everything in raw/ not yet judged (cached in graded/), then write results.json grade.py sample N write spotcheck.json: N judgments sampled with a fixed seed for hand review grade.py agree compare spotcheck_review.json (hand labels) with the judge A deepseek answer is flagged web=true when the gateway orchestrator ran a web search/extract during that task. Accuracy is reported with and without those flagged answers.""" import hashlib, json, random, sys from collections import Counter from concurrent.futures import ThreadPoolExecutor from lib import HERE, judge, load_pool RAW, GRADED = HERE / 'raw', HERE / 'graded' GRADED.mkdir(exist_ok=True) TASKS = {t['id']: t for t in json.loads((HERE / 'tasks.json').read_text())} ORDER = ['none', 'full_1', 'full_10', 'full_30', 'full_all', 'index', 'index_rep2', 'index_fixed'] def h(s): return hashlib.sha1((s or '').encode()).hexdigest()[:12] def grade_all(): jobs = [] for f in sorted(RAW.glob('*.json')): gpath = GRADED / f.name cache = {d['id']: d for d in json.loads(gpath.read_text())} if gpath.exists() else {} for d in json.loads(f.read_text()): c = cache.get(d['id']) if c and c.get('answer_hash') == h(d.get('answer')) and c.get('correct') is not None: continue jobs.append((f.name, d)) def one(job): name, d = job if d.get('error') or d.get('answer') is None: g = {'correct': False, 'reason': f'no answer: {d.get("error")}'} else: g = judge(TASKS[d['id']], d['answer']) return name, {'id': d['id'], 'answer_hash': h(d.get('answer')), 'correct': g['correct'], 'reason': g['reason']} print('judging', len(jobs), flush=True) store = {} with ThreadPoolExecutor(12) as ex: for i, (name, g) in enumerate(ex.map(one, jobs)): store.setdefault(name, []).append(g) if i % 50 == 0: print(i, flush=True) for name, gs in store.items(): gpath = GRADED / name cache = {d['id']: d for d in json.loads(gpath.read_text())} if gpath.exists() else {} for g in gs: cache[g['id']] = g gpath.write_text(json.dumps(sorted(cache.values(), key=lambda x: x['id']), ensure_ascii=False, indent=1)) aggregate() def load_joined(): rows = [] for f in sorted(RAW.glob('*.json')): gpath = GRADED / f.name gs = {d['id']: d for d in json.loads(gpath.read_text())} if gpath.exists() else {} for d in json.loads(f.read_text()): g = gs.get(d['id'], {}) rows.append({**d, 'correct': g.get('correct'), 'judge_reason': g.get('reason'), 'web': bool(d.get('web_tools'))}) return rows def mean(xs): xs = [x for x in xs if x is not None] return sum(xs) / len(xs) if xs else None def aggregate(): pool, _ = load_pool() rows = load_joined() res = {'pool': {'n_skills': pool['n_skills'], 'total_tokens_cl100k': pool['total_tokens_cl100k']}, 'cells': {}} for model in sorted({r['model'] for r in rows}): for cond in ORDER: rs = [r for r in rows if r['model'] == model and r['condition'] == cond] if not rs: continue ok = [r for r in rs if not r.get('error') and r.get('answer') is not None] cell = {'n': len(rs), 'n_errors': len(rs) - len(ok), 'errors_sample': sorted({(r.get('error') or '')[:160] for r in rs if r.get('error')})[:3], 'correct': sum(1 for r in rs if r['correct']), 'accuracy': sum(1 for r in rs if r['correct']) / len(rs), 'mean_prompt_tokens': mean([r.get('prompt_tokens') for r in ok]), 'mean_completion_tokens': mean([r.get('completion_tokens') for r in ok]), 'mean_latency_s': mean([r.get('sec') for r in ok]), 'total_cost_usd': sum(r.get('cost') or 0 for r in rs), 'mean_cost_usd': mean([r.get('cost') for r in ok]), 'n_web_flagged': sum(r['web'] for r in rs)} clean = [r for r in rs if not r['web']] cell['accuracy_excl_web'] = (sum(1 for r in clean if r['correct']) / len(clean)) if clean else None cell['n_excl_web'] = len(clean) if cond.startswith('index'): cell['opened_target'] = sum(1 for r in rs if r.get('opened_target')) cell['mean_opens'] = mean([len(r.get('opened') or []) for r in rs]) wrong = [r for r in rs if not r['correct']] cell['wrong_breakdown'] = { 'no_skill_opened': sum(1 for r in wrong if not r.get('requested')), 'only_other_skills_opened': sum(1 for r in wrong if r.get('requested') and not r.get('opened_target')), 'target_opened_but_wrong': sum(1 for r in wrong if r.get('opened_target')), 'error_no_answer': sum(1 for r in wrong if r.get('error'))} cell['right_without_target'] = sum(1 for r in rs if r['correct'] and not r.get('opened_target')) cell['other_opened'] = Counter(n for r in rs for n in (r.get('opened') or []) if n != r['target']).most_common(15) res['cells'][f'{model}:{cond}'] = cell (HERE / 'results.json').write_text(json.dumps(res, ensure_ascii=False, indent=1)) print(f'{"cell":40s} {"acc":>6s} {"n":>3s} {"err":>3s} {"web":>3s} {"tok":>9s} {"lat":>6s} {"cost$":>8s}') for k, c in res['cells'].items(): print(f'{k:40s} {c["correct"]:>3d}/{c["n"]:<3d} {c["n_errors"]:>3d} {c["n_web_flagged"]:>3d} ' f'{(c["mean_prompt_tokens"] or 0):>9.0f} {(c["mean_latency_s"] or 0):>6.1f} {c["total_cost_usd"]:>8.3f}' + (f' opened_target={c["opened_target"]} wrong={c["wrong_breakdown"]}' if 'opened_target' in c else '')) def sample(n): rows = [r for r in load_joined() if r.get('answer')] rng = random.Random(7) pos = [r for r in rows if r['correct']] neg = [r for r in rows if r['correct'] is False] pick = rng.sample(pos, min(n // 2, len(pos))) + rng.sample(neg, min(n - n // 2, len(neg))) rng.shuffle(pick) out = [{'model': r['model'], 'condition': r['condition'], 'id': r['id'], 'question': TASKS[r['id']]['question'], 'excerpt': TASKS[r['id']]['excerpt'], 'rubric': TASKS[r['id']]['rubric'], 'answer': r['answer'], 'judge_correct': r['correct'], 'judge_reason': r['judge_reason']} for r in pick] (HERE / 'spotcheck.json').write_text(json.dumps(out, ensure_ascii=False, indent=1)) print('wrote', len(out)) def agree(): sc = json.loads((HERE / 'spotcheck.json').read_text()) rv = json.loads((HERE / 'spotcheck_review.json').read_text()) n = len(rv) same = sum(1 for s, v in zip(sc, rv) if s['judge_correct'] == v['my_correct']) print(f'agreement {same}/{n}') for s, v in zip(sc, rv): if s['judge_correct'] != v['my_correct']: print('DISAGREE', s['model'], s['condition'], s['id'], 'judge', s['judge_correct'], 'me', v['my_correct'], v.get('note')) def _main(): if len(sys.argv) > 1 and sys.argv[1] == 'analyze': analyze() elif len(sys.argv) > 1 and sys.argv[1] == 'sample': sample(int(sys.argv[2])) elif len(sys.argv) > 1 and sys.argv[1] == 'agree': agree() elif len(sys.argv) > 1 and sys.argv[1] == 'agg': aggregate() else: grade_all() def analyze(): """Extra tables for the README: index_fixed before/after, cue split, dilution by k, full_all failure positions.""" rows = load_joined() by = {(r['model'], r['condition'], r['id']): r for r in rows} models = sorted({r['model'] for r in rows}) no_cue = set(json.loads((HERE / 'task_cues.json').read_text())['no_cue']) plan = json.loads((HERE / 'index_fixed_plan.json').read_text()) out = {} for m in models: a = {} p = plan.get(m) if p: f = p['failed_not_opened'] a['index_fixed'] = { 'failed_not_opened': len(f), 'now_correct': [i for i in f if by[(m, 'index_fixed', i)]['correct']], 'now_opened_target': [i for i in f if by[(m, 'index_fixed', i)].get('opened_target')], 'still_opened_nothing': [i for i in f if not by[(m, 'index_fixed', i)].get('requested')], 'passing_sample': len(p['passing_sample']), 'passing_sample_still_correct': sum(1 for i in p['passing_sample'] if by[(m, 'index_fixed', i)]['correct']), 'passing_sample_broke': [i for i in p['passing_sample'] if not by[(m, 'index_fixed', i)]['correct']]} idx = [by[(m, 'index', t)] for t in TASKS if (m, 'index', t) in by] for lab, sel in (('cue', lambda r: r['id'] not in no_cue), ('no_cue', lambda r: r['id'] in no_cue)): rs = [r for r in idx if sel(r)] a[f'index_{lab}'] = {'n': len(rs), 'correct': sum(1 for r in rs if r['correct']), 'opened_target': sum(1 for r in rs if r.get('opened_target')), 'opened_nothing': sum(1 for r in rs if not r.get('requested'))} ok1 = [t for t in TASKS if (m, 'full_1', t) in by and by[(m, 'full_1', t)]['correct']] a['full_1_correct'] = len(ok1) a['dilution'] = {} for k in ('full_10', 'full_30', 'full_all'): if (m, k, 'T01') in by and not by[(m, k, 'T01')].get('error'): lost = [t for t in ok1 if not by[(m, k, t)]['correct']] gained = [t for t in TASKS if (m, 'full_1', t) in by and not by[(m, 'full_1', t)]['correct'] and by[(m, k, t)]['correct']] a['dilution'][k] = {'lost_vs_full_1': lost, 'gained_vs_full_1': gained, 'lost_target_pos': {t: by[(m, k, t)].get('target_pos') for t in lost}} a['index_right_without_target'] = [r['id'] for r in idx if r['correct'] and not r.get('opened_target')] a['index_opened_counts'] = Counter(len(r.get('requested') or []) for r in idx) out[m] = a res = json.loads((HERE / 'results.json').read_text()) res['analysis'] = out (HERE / 'results.json').write_text(json.dumps(res, ensure_ascii=False, indent=1)) print(json.dumps(out, ensure_ascii=False, indent=1)) if __name__ == '__main__': _main()