"""2.7: build the public skill pool.

Sources (git-tracked files only):
  1. context_infrastructure/rules/skills/*.md (public repo grapeot/context-infrastructure), minus INDEX.md
  2. every repo under adhoc_jobs/*/ whose origin remote name appears in context_infrastructure/docs/SKILL_ECOSYSTEM.md:
     skills/*.md and skills/<dir>/SKILL.md. Sub-files (references/, templates/, non-SKILL.md files inside a skill dir,
     INDEX.md) are skipped. One skill = one primary markdown file.
Duplicates (same skill name from two sources, or two checkouts of the same remote) keep the first one seen.
A regex secret scan flags candidates; files listed in EXCLUDE were checked by hand and dropped.
Writes pool.json (metadata) and pool_text.json (name -> full text)."""
import json, pathlib, re, subprocess
import tiktoken
import yaml

HERE = pathlib.Path(__file__).resolve().parent
ADHOC = pathlib.Path('/Users/grapeot/co/knowledge_working/adhoc_jobs')
CI = ADHOC / 'context_infrastructure'
enc = tiktoken.get_encoding('cl100k_base')

# filled in after reviewing the secret-scan hits by hand (see README)
EXCLUDE = json.loads((HERE / 'exclude.json').read_text()) if (HERE / 'exclude.json').exists() else {}

SECRET_PATTERNS = {
    'openai_key': r'sk-[A-Za-z0-9_-]{20,}',
    'github_token': r'gh[pousr]_[A-Za-z0-9]{20,}',
    'aws_key': r'AKIA[0-9A-Z]{16}',
    'slack_token': r'xox[abpr]-[A-Za-z0-9-]{10,}',
    'bearer_literal': r'Bearer\s+[A-Za-z0-9._-]{24,}',
    'assigned_secret': r'(?i)(api[_-]?key|secret|token|password)\s*[:=]\s*["\']?[A-Za-z0-9/+_.-]{16,}',
    'email': r'[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}',
    'phone': r'(?<![\d.])(?:\+?1[ -.])?\(?\d{3}\)?[ -.]\d{3}[ -.]\d{4}(?!\d)',
}


def git_files(repo, pattern):
    out = subprocess.run(['git', '-C', str(repo), 'ls-files', pattern], capture_output=True, text=True).stdout
    return [l for l in out.splitlines() if l]


def describe(text):
    m = re.match(r'^---\n(.*?)\n---\n', text, re.S)
    if m:
        try:
            fm = yaml.safe_load(m.group(1)) or {}
        except yaml.YAMLError:
            fm = {}
        if isinstance(fm, dict) and fm.get('description'):
            return re.sub(r'\s+', ' ', str(fm['description'])).strip(), 'frontmatter'
        body = text[m.end():]
    else:
        body = text
    h = re.search(r'^#\s+(.+)$', body, re.M)
    head = h.group(1).strip() if h else ''
    rest = body[h.end():] if h else body
    para = ''
    for block in re.split(r'\n\s*\n', rest):
        b = block.strip()
        if b and not b.startswith(('#', '```', '|', '-', '*', '>', '<')) and not re.match(r'\d+\.', b):
            para = re.sub(r'\s+', ' ', b)
            break
    sent = re.split(r'(?<=[。！？])|(?<=[^0-9][.!?])\s', para)[0] if para else ''
    return (f'{head}: {sent}' if sent else head), 'heading+first_sentence'


def scan(text):
    hits = {}
    for k, p in SECRET_PATTERNS.items():
        f = sorted(set(m.group(0) for m in re.finditer(p, text)))
        if f:
            hits[k] = f
    return hits


def main():
    eco = set(re.findall(r'grapeot/([A-Za-z0-9_.-]+)', (CI / 'docs/SKILL_ECOSYSTEM.md').read_text()))
    cands = []  # (name, repo_dir, relpath, origin)
    for rel in git_files(CI, 'rules/skills/*.md'):
        if rel.count('/') == 2 and not rel.endswith('INDEX.md'):
            cands.append((pathlib.Path(rel).stem, CI, rel, 'context-infrastructure'))
    seen_remote = set()
    for d in sorted(ADHOC.iterdir()):
        if not (d / '.git').exists() or d == CI:
            continue
        url = subprocess.run(['git', '-C', str(d), 'remote', 'get-url', 'origin'], capture_output=True, text=True).stdout.strip()
        rname = url.rstrip('/').split('/')[-1].removesuffix('.git')
        if rname not in eco or rname in seen_remote:
            continue
        seen_remote.add(rname)
        for rel in git_files(d, 'skills'):
            p = pathlib.Path(rel)
            parts = p.parts
            if p.suffix != '.md' or p.name == 'INDEX.md' or 'references' in parts or 'templates' in parts:
                continue
            if len(parts) == 2:
                name = p.stem
            elif len(parts) == 3 and p.name == 'SKILL.md':
                name = parts[1]
            else:
                continue
            if name.lower() in ('skill',):
                name = d.name
            cands.append((name, d, rel, rname))
    pool, texts, dropped = [], {}, []
    for name, repo, rel, origin in cands:
        text = (repo / rel).read_text(encoding='utf-8')
        if name in texts:
            old = next(p for p in pool if p['name'] == name)
            if len(text) <= len(texts[name]):
                dropped.append({'name': name, 'source': f'{origin}:{rel}', 'reason': f'duplicate name, kept longer copy {old["source"]}'})
                continue
            dropped.append({'name': name, 'source': old['source'], 'reason': f'duplicate name (migration stub), kept longer copy {origin}:{rel}'})
            pool.remove(old)
            del texts[name]
        hits = scan(text)
        src = f'{origin}:{rel}'
        if src in EXCLUDE:
            dropped.append({'name': name, 'source': src, 'reason': EXCLUDE[src]})
            continue
        desc, how = describe(text)
        texts[name] = text
        pool.append({'name': name, 'source': src, 'path': str(repo / rel), 'bytes': len(text.encode()),
                     'tokens': len(enc.encode(text)), 'description': desc, 'description_from': how,
                     'scan_hits': hits})
    total = sum(p['tokens'] for p in pool)
    out = {'n_skills': len(pool), 'total_tokens_cl100k': total, 'skills': pool, 'dropped': dropped}
    (HERE / 'pool.json').write_text(json.dumps(out, ensure_ascii=False, indent=1), encoding='utf-8')
    (HERE / 'pool_text.json').write_text(json.dumps(texts, ensure_ascii=False), encoding='utf-8')
    print(len(pool), 'skills', total, 'tokens;', len(dropped), 'dropped')
    for p in pool:
        if p['scan_hits']:
            print('SCAN', p['source'], p['scan_hits'])


if __name__ == '__main__':
    main()