"""2.7: build the task set.

  make_tasks.py draft      gpt-5 reads each skill and proposes 2 candidate tasks -> candidates_raw.json
                           (excerpt must be a verbatim substring of the skill; non-verbatim candidates are dropped)
  make_tasks.py baseline   every candidate, no skill in the window, 3 models, graded by the judge -> candidates_baseline.json
  make_tasks.py finalize   apply the hand review in selection.json -> tasks.json

Twins (two near-identical files) are not used as targets, because opening either one answers the task."""
import json, re, sys, random
from concurrent.futures import ThreadPoolExecutor
from lib import HERE, call, judge, load_pool, PREAMBLE

NOT_TARGET = {'skill_download_paper', 'bestpractice_academic_paper_conversion'}
MODELS = ['deepseek-v4.1-flash', 'grok-4-fast', 'gpt-5']

DRAFT_SYS = """你在为一个对照实验出题。给你一份 skill 文档（给 AI agent 看的操作说明）。请从中找出 2 条“具体、非显而易见”的规则或事实，
比如一个确切的命令行参数、默认值、上限、禁止做的步骤、容易踩的坑、必须先做的检查。然后为每条写一道题。

要求：
1. 题目用中文，写成一个真实用户会提的请求或问题（1 到 3 句），可以提到产品或领域名（比如“我的 Narwal 扫地机”），
   但不能提 skill 的文件名，也不能照抄文档里有辨识度的原句或术语组合。
2. 只有知道这条规则才能答对；一个懂行但没看过这份文档的工程师大概率会答错或答不具体。不要考常识，不要考作者机器上的私有路径。
3. 规则必须在文档里写得明确，答案唯一，不存在两种都说得通的答法。
4. excerpt 必须是文档原文里连续的一段（逐字复制，不改一个字符，30 到 400 字符），它单独就能支撑标准答案。
5. rubric.must_include 是 1 到 3 个判分要点（答案必须包含的具体内容），reference_answer 是一句话标准答案。

只输出 JSON：{"tasks":[{"question":"...","excerpt":"...","fact":"一句话概括规则","rubric":{"must_include":["..."],"reference_answer":"..."}}]}"""


def norm(s):
    return re.sub(r'\s+', ' ', s).strip()


def draft():
    pool, texts = load_pool()
    names = [s['name'] for s in pool['skills'] if s['name'] not in NOT_TARGET]
    out_path = HERE / 'candidates_raw.json'
    done = json.loads(out_path.read_text()) if out_path.exists() else {}

    def one(n):
        if n in done and done[n].get('tasks') is not None:
            return n, done[n]
        r = call('gpt-5', [{'role': 'system', 'content': DRAFT_SYS},
                           {'role': 'user', 'content': f'skill 名：{n}\n\n<document>\n{texts[n]}\n</document>'}],
                 max_tokens=16000)
        if 'error' in r:
            return n, {'error': r['error']}
        txt = r['message'].get('content') or ''
        try:
            d = json.loads(txt[txt.find('{'):txt.rfind('}') + 1])
        except Exception:
            return n, {'error': 'bad json', 'raw': txt}
        kept = []
        for t in d.get('tasks', []):
            ok = norm(t.get('excerpt', '')) in norm(texts[n])
            t['excerpt_verbatim'] = ok
            kept.append(t)
        print(n, len(kept), sum(t['excerpt_verbatim'] for t in kept), flush=True)
        return n, {'tasks': kept, 'cost': r['cost']}

    with ThreadPoolExecutor(8) as ex:
        for n, v in ex.map(one, names):
            done[n] = v
            out_path.write_text(json.dumps(done, ensure_ascii=False, indent=1))
    cands = []
    for n in names:
        for i, t in enumerate(done.get(n, {}).get('tasks') or []):
            if t['excerpt_verbatim']:
                cands.append({'cid': f'{n}#{i}', 'target': n, **{k: t[k] for k in ('question', 'excerpt', 'fact', 'rubric')}})
    (HERE / 'candidates.json').write_text(json.dumps(cands, ensure_ascii=False, indent=1))
    print('candidates with verbatim excerpt:', len(cands))


def other_skills_with(excerpt, target, texts):
    """Which other skills contain the core of the excerpt (longest line, >=25 chars) verbatim?"""
    lines = sorted((norm(l) for l in excerpt.splitlines()), key=len, reverse=True)
    core = lines[0] if lines else ''
    if len(core) < 25:
        return []
    return [n for n, t in texts.items() if n != target and core in norm(t)]


def baseline():
    _, texts = load_pool()
    cands = json.loads((HERE / 'candidates.json').read_text())
    out_path = HERE / 'candidates_baseline.json'
    done = {(d['cid'], d['model']): d for d in json.loads(out_path.read_text())} if out_path.exists() else {}
    jobs = [(c, m) for c in cands for m in MODELS if (c['cid'], m) not in done]

    def one(job):
        c, m = job
        r = call(m, [{'role': 'system', 'content': PREAMBLE}, {'role': 'user', 'content': c['question']}],
                 max_tokens=8000 if m == 'gpt-5' else 3000)
        ans = None if 'error' in r else (r['message'].get('content') or '')
        g = judge(c, ans) if ans is not None else {'correct': None, 'reason': r.get('error')}
        return {'cid': c['cid'], 'model': m, 'answer': ans, 'web_tools': r.get('web_tools', []), 'error': r.get('error'),
                'correct': g['correct'], 'reason': g['reason']}

    res = list(done.values())
    with ThreadPoolExecutor(12) as ex:
        for i, d in enumerate(ex.map(one, jobs)):
            res.append(d)
            if i % 20 == 0:
                out_path.write_text(json.dumps(res, ensure_ascii=False, indent=1))
                print(i, '/', len(jobs), flush=True)
    out_path.write_text(json.dumps(res, ensure_ascii=False, indent=1))
    # summary per candidate
    by = {}
    for d in res:
        by.setdefault(d['cid'], {})[d['model']] = d
    summ = []
    for c in cands:
        b = by.get(c['cid'], {})
        n_right = sum(1 for m in MODELS if b.get(m, {}).get('correct'))
        web = any(b.get(m, {}).get('web_tools') for m in MODELS)
        summ.append({'cid': c['cid'], 'target': c['target'], 'baseline_correct': n_right, 'deepseek_used_web': web,
                     'excerpt_also_in': other_skills_with(c['excerpt'], c['target'], texts)})
    (HERE / 'candidates_summary.json').write_text(json.dumps(summ, ensure_ascii=False, indent=1))
    print('kept (baseline <2/3):', sum(s['baseline_correct'] < 2 for s in summ), 'of', len(summ))


def finalize():
    cands = {c['cid']: c for c in json.loads((HERE / 'candidates.json').read_text())}
    summ = {s['cid']: s for s in json.loads((HERE / 'candidates_summary.json').read_text())}
    sel = json.loads((HERE / 'selection.json').read_text())
    tasks = []
    for i, s in enumerate(sel['selected']):
        c = dict(cands[s['cid']])
        c.update(s.get('edits', {}))
        tasks.append({'id': f'T{i + 1:02d}', 'cid': s['cid'], 'target': c['target'], 'question': c['question'],
                      'excerpt': c['excerpt'], 'fact': c['fact'], 'rubric': c['rubric'],
                      'baseline_correct_of_3': summ[s['cid']]['baseline_correct'], 'review_note': s.get('note', '')})
    (HERE / 'tasks.json').write_text(json.dumps(tasks, ensure_ascii=False, indent=1))
    print(len(tasks), 'tasks;', len({t['target'] for t in tasks}), 'distinct targets')


if __name__ == '__main__':
    {'draft': draft, 'baseline': baseline, 'finalize': finalize}[sys.argv[1]]()