"""2.8 (English edition): build the English skill pool from the Chinese edition's pool.

Start from the 139 skills of experiments/2_7_skills/pool.json and apply, per skill:
  1. context-infrastructure skills -> the English mirror grapeot/context-infrastructure-en (rules/skills/<name>.md, committed
     HEAD of the local checkout) whenever it exists. The en mirror is the English edition of that repo, so we take it for
     every skill, not only the Chinese-heavy ones.
  2. writing-skill skills -> its English folder skills_en/<name>.md (committed HEAD) when it exists.
  3. any other skill whose body is mostly Chinese (outside code fences, CJK characters > 0.5 x Latin words) or whose index
     description contains Chinese -> dropped, because no English version exists.
  4. everything else -> the exact text the Chinese run used (pool_text.json), so the two editions share those skills byte for byte.
The dropped slots are refilled with English skills from other public grapeot repos (REPLACEMENTS), so the pool size stays 139.
Same description rule (frontmatter description, else first heading + first sentence), same 240-char index cap, same
cl100k token count as the Chinese run. Writes pool.json, pool_text.json, provenance in pool.json."""
import json, pathlib, re, subprocess, sys
import tiktoken

HERE = pathlib.Path(__file__).resolve().parent
ZH = HERE.parents[1] / 'experiments' / '2_7_skills'
sys.path.insert(0, str(ZH))
from build_pool import describe, scan  # noqa: E402  (same description extraction and secret scan as the zh run)

ADHOC = pathlib.Path('/Users/grapeot/co/knowledge_working/adhoc_jobs')
EXTRA = pathlib.Path('/Users/grapeot/co/knowledge_working/tmp/en_pool_extra')  # shallow clones of public repos not checked out locally
CI_EN = ADHOC / 'context_infrastructure_en'
WS = ADHOC / 'writing_skill'
enc = tiktoken.get_encoding('cl100k_base')

# (name, repo dir, path in repo, origin). All public grapeot repos; English bodies. Chosen as: the extra skills in the
# context-infrastructure-en mirror, ecosystem repos that the zh pool did not pick up, then other public *-skill repos.
REPLACEMENTS = [
    ('send_email', CI_EN, 'rules/skills/send_email.md', 'context-infrastructure-en'),
    ('typefully_post', CI_EN, 'rules/skills/typefully_post.md', 'context-infrastructure-en'),
    ('typefully_metrics', CI_EN, 'rules/skills/typefully_metrics.md', 'context-infrastructure-en'),
    ('workflow_presentation_slides', CI_EN, 'rules/skills/workflow_presentation_slides.md', 'context-infrastructure-en'),
    ('apple_calendar', ADHOC / 'apple_calendar_skill', 'skills/apple_calendar.md', 'apple-calendar-skill'),
    ('colab', ADHOC / 'colab_skill', 'skills/colab/SKILL.md', 'colab-skill'),
    ('ai_session_export', ADHOC / 'ai_session_export', 'skill.md', 'ai_session_export'),
    ('docx', ADHOC / 'docx_skill', 'docs/skill.md', 'docx-skill'),
    ('skill_google_docs', ADHOC / 'gdocs_skill', 'docs/skill_google_docs.md', 'gdocs-skill'),
    ('tiff_icc_profile', EXTRA / 'tiff-icc-profile', 'skills/tiff_icc_profile.md', 'tiff-icc-profile'),
    ('skill_smtp', EXTRA / 'smtp-skill', 'skills/skill_smtp.md', 'smtp-skill'),
    ('superlinear-brand', EXTRA / 'superlinear-brand-skill', 'skills/superlinear-brand/SKILL.md', 'superlinear-brand-skill'),
    ('track17', EXTRA / 'track17-skill', 'skills/track17.md', 'track17-skill'),
    ('chinese_visa_photo', EXTRA / 'chinese-visa-photo-skill', 'skills/chinese_visa_photo.md', 'chinese-visa-photo-skill'),
    ('gpt-live', ADHOC / 'gpt_live_skill', 'skills/gpt-live/SKILL.md', 'gpt-live-skill'),
]


def git_show(repo, rel):
    r = subprocess.run(['git', '-C', str(repo), 'show', f'HEAD:{rel}'], capture_output=True, text=True)
    return r.stdout if r.returncode == 0 else None


def head_sha(repo):
    return subprocess.run(['git', '-C', str(repo), 'rev-parse', '--short', 'HEAD'], capture_output=True, text=True).stdout.strip()


def zh_stats(text):
    prose = re.sub(r'```.*?```', '', text, flags=re.S)
    return len(re.findall(r'[一-鿿]', prose)), len(re.findall(r'[A-Za-z]+', prose))


def chinese_heavy(text, desc):
    c, w = zh_stats(text)
    return c > 0.5 * w or bool(re.search(r'[一-鿿]', desc))


def main():
    zp = json.loads((ZH / 'pool.json').read_text())
    zt = json.loads((ZH / 'pool_text.json').read_text())
    pool, texts, dropped = [], {}, []

    def add(name, text, source, path, action, zh_source=None):
        desc, how = describe(text)
        c, w = zh_stats(text)
        texts[name] = text
        pool.append({'name': name, 'source': source, 'path': path, 'action': action, 'zh_source': zh_source,
                     'bytes': len(text.encode()), 'tokens': len(enc.encode(text)), 'description': desc,
                     'description_from': how, 'cjk_chars': c, 'latin_words': w,
                     'desc_has_cjk': bool(re.search(r'[一-鿿]', desc)), 'scan_hits': scan(text)})

    for s in zp['skills']:
        n, origin = s['name'], s['source'].split(':')[0]
        zh_heavy = chinese_heavy(zt[n], s['description'])
        if origin == 'context-infrastructure' and (CI_EN / f'rules/skills/{n}.md').exists():
            rel = f'rules/skills/{n}.md'
            text = git_show(CI_EN, rel)
            same = text.strip() == zt[n].strip()
            add(n, text, f'context-infrastructure-en@{head_sha(CI_EN)}:{rel}', str(CI_EN / rel),
                'unchanged (en mirror identical)' if same else ('swapped to en mirror' if zh_heavy else 'en mirror (zh copy was already mostly English)'),
                s['source'])
        elif origin == 'writing-skill' and (WS / f'skills_en/{n}.md').exists():
            rel = f'skills_en/{n}.md'
            add(n, git_show(WS, rel), f'writing-skill@{head_sha(WS)}:{rel}', str(WS / rel),
                'swapped to skills_en' if zh_heavy else 'skills_en (zh copy was already mostly English)', s['source'])
        elif zh_heavy:
            c, w = zh_stats(zt[n])
            dropped.append({'name': n, 'zh_source': s['source'], 'reason': f'mostly Chinese, no English version '
                            f'(CJK chars {c}, Latin words {w}, description has CJK: {bool(re.search(r"[一-龥]", s["description"]))})'})
        else:
            add(n, zt[n], s['source'], s['path'], 'unchanged (same text as zh pool)', s['source'])

    for name, repo, rel, origin in REPLACEMENTS[:len(dropped)]:
        text = git_show(repo, rel)
        assert text, (repo, rel)
        assert name not in texts, name
        add(name, text, f'{origin}@{head_sha(repo)}:{rel}', str(repo / rel), 'replacement (English public skill)')

    total = sum(p['tokens'] for p in pool)
    still = [p['name'] for p in pool if p['cjk_chars'] > 0.5 * p['latin_words'] or p['desc_has_cjk']]
    from collections import Counter
    out = {'n_skills': len(pool), 'total_tokens_cl100k': total, 'total_chars': sum(len(t) for t in texts.values()),
           'actions': dict(Counter(p['action'] for p in pool)), 'still_chinese_heavy': still,
           'skills': pool, 'dropped_from_zh_pool': dropped}
    (HERE / 'pool.json').write_text(json.dumps(out, ensure_ascii=False, indent=1), encoding='utf-8')
    (HERE / 'pool_text.json').write_text(json.dumps(texts, ensure_ascii=False), encoding='utf-8')
    print(len(pool), 'skills', total, 'tokens cl100k;', len(dropped), 'dropped;', out['actions'])
    print('still chinese-heavy:', still)
    for p in pool:
        if p['scan_hits']:
            print('SCAN', p['source'], p['scan_hits'])


if __name__ == '__main__':
    main()