#!/usr/bin/env node
// extract.mjs — self-discovering extractor for a model's authored corpus + narration.
// Portable: no manifest, no pre-existing dump. Walks THIS machine's Claude Code transcripts.
//
//   node extract.mjs [--model claude-fable-5] [--out ~/.claude/fable]
//
// Emits under OUT:
//   corpus/<project>/<doc>.md   — every .md the MODEL authored via a Write tool_use (externalized reasoning)
//   narration/excerpts.md       — curated, secret-scrubbed terse MODEL text blocks (how it narrates)
//   manifest.json               — {file, proj, chars, shared} per authored doc
//   index.md                    — BLUF catalog for agentic browsing
//
// HONESTY ANCHOR: only two layers are mineable — authored artifacts (Write content) and narration
// (text blocks). Live reasoning is NOT: `thinking` blocks are stored 0-char (encrypted). Never claim
// cognition was reverse-engineered. Raw transcripts are NEVER copied (not model-authored; secret risk).
import fs from 'node:fs';
import path from 'node:path';
import os from 'node:os';

const arg = (k, d) => { const i = process.argv.indexOf(k); return i > -1 ? process.argv[i + 1] : d; };
const MODEL = arg('--model', 'claude-fable-5');
const OUT = path.resolve(arg('--out', path.join(os.homedir(), '.claude/fable')).replace(/^~/, os.homedir()));
const PROJ_ROOT = path.join(os.homedir(), '.claude/projects');

// ---- secret/PII guard for curated excerpts (defense in depth) ----
const SECRET = [
  /[A-Fa-f0-9]{32,}/,                          // long hex (hashes, keys)
  /[A-Za-z0-9_\-]{40,}/,                        // long opaque token
  /\b(sk|pk|ghp|gho|xox[bap])[-_][A-Za-z0-9]{8,}/,
  /\b(api[_-]?key|secret|password|passwd|bearer|authorization|token)\b\s*[:=]/i,
  /-----BEGIN [A-Z ]+-----/,
  /\b[\w.+-]+@[\w-]+\.[\w.-]+\b/,               // email
];
const looksSensitive = (s) => SECRET.some((re) => re.test(s));

const walk = (d) => {
  let acc = [];
  let ents; try { ents = fs.readdirSync(d, { withFileTypes: true }); } catch { return acc; }
  for (const e of ents) {
    const p = path.join(d, e.name);
    if (e.isDirectory()) acc = acc.concat(walk(p));
    else if (e.name.endsWith('.jsonl')) acc.push(p);
  }
  return acc;
};
// project label: prefer a real source-tree segment, else the transcript's project dir
const projOf = (filePath, transcriptDir) => {
  // real source-tree path (a doc's file_path) — but NOT the .claude/projects transcript store
  const m = filePath.match(/(?<!\.claude)\/(?:Projects|repos|src|code)\/([^/]+)/i);
  if (m) return m[1];
  // else decode the encoded transcript dir name: -home-user-Projects-<name>[-sub...] -> <name>
  const seg = (transcriptDir.split('/').pop() || 'unknown').replace(/^-home-[^-]+-/i, '');
  const parts = seg.replace(/-/g, '/').split('/').filter(Boolean);
  const i = parts.findIndex((p) => /^(Projects|repos|src|code)$/i.test(p));
  return (i > -1 ? parts[i + 1] : parts[0]) || 'unknown';
};

if (!fs.existsSync(PROJ_ROOT)) { console.error(`no transcripts at ${PROJ_ROOT}`); process.exit(2); }
const files = walk(PROJ_ROOT);

// ---- pass 1: authored docs (Write content) + narration text, per model ----
const docs = new Map();                 // fpath -> {proj, chars, content}
const projModels = new Map();           // proj -> Set(models that authored .md there)
const cats = { opener: [], preamble: [], decision: [], done: [] };
const seen = new Set();
const wc = (s) => s.trim().split(/\s+/).length;
const DECISION = /\b(decided|decision|going with|chose|will use|rejected|instead of|because|trade-?off)\b/i;
const DONE = /\b(done|fixed|added|verified|passing|works|complete|shipped|green)\b/i;

for (const f of files) {
  let lines; try { lines = fs.readFileSync(f, 'utf8').split('\n'); } catch { continue; }
  const tdir = path.dirname(f);
  for (const ln of lines) {
    if (!ln.includes(MODEL) && !ln.includes('claude-')) continue;   // cheap prefilter
    let o; try { o = JSON.parse(ln); } catch { continue; }
    const msg = o.message || {};
    if (msg.role !== 'assistant') continue;
    const c = msg.content;
    if (!Array.isArray(c)) continue;
    const model = msg.model || '';

    for (let i = 0; i < c.length; i++) {
      const b = c[i];
      if (!b || typeof b !== 'object') continue;

      // authored .md docs — any model, to track `shared`; corpus keeps only MODEL's
      if (b.type === 'tool_use' && b.name === 'Write') {
        const fp = (b.input || {}).file_path || '';
        if (!fp.endsWith('.md')) continue;
        const proj = projOf(fp, tdir);
        (projModels.get(proj) || projModels.set(proj, new Set()).get(proj)).add(model);
        if (model !== MODEL) continue;
        const content = (b.input || {}).content || '';
        const prev = docs.get(fp);
        if (!prev || content.length > prev.chars) docs.set(fp, { proj, chars: content.length, content });
      }

      // narration — MODEL's own terse text blocks only
      if (b.type === 'text' && model === MODEL) {
        const t = (b.text || '').trim();
        if (!t) continue;
        const words = wc(t);
        if (words < 3 || words > 45 || looksSensitive(t)) continue;
        const key = t.slice(0, 80).toLowerCase();
        if (seen.has(key)) continue;
        const proj = projOf(f, tdir);
        const first = t.split('\n')[0].trim();
        const followedByTool = c.slice(i + 1).some((x) => x && x.type === 'tool_use');
        let cat = null;
        if (DECISION.test(t)) cat = 'decision';
        else if (DONE.test(first) && words <= 28) cat = 'done';
        else if (i === 0) cat = 'opener';
        else if (followedByTool) cat = 'preamble';
        if (cat && cats[cat].length < 40) { seen.add(key); cats[cat].push({ proj, t }); }
      }
    }
  }
}

if (docs.size === 0) {
  console.error(`no ${MODEL}-authored .md docs found. Did this machine run ${MODEL}? ` +
                `Check: grep -rl ${MODEL} ${PROJ_ROOT}`);
  process.exit(3);
}

// ---- write corpus + manifest ----
fs.rmSync(OUT, { recursive: true, force: true });
fs.mkdirSync(path.join(OUT, 'corpus'), { recursive: true });
const manifest = { model: MODEL, docs: [] };
for (const [fp, d] of docs) {
  const shared = (projModels.get(d.proj)?.size || 1) > 1;
  const base = path.basename(fp);
  const destDir = path.join(OUT, 'corpus', d.proj);
  fs.mkdirSync(destDir, { recursive: true });
  fs.writeFileSync(path.join(destDir, base), d.content);
  manifest.docs.push({ file: `${d.proj}__${base}`, proj: d.proj, chars: d.chars, shared });
}
manifest.docs.sort((a, b) => b.chars - a.chars);
fs.writeFileSync(path.join(OUT, 'manifest.json'), JSON.stringify(manifest, null, 2));

// ---- write narration excerpts ----
const CAP = { opener: 24, preamble: 18, decision: 22, done: 16 };
fs.mkdirSync(path.join(OUT, 'narration'), { recursive: true });
const sect = (title, key, note) => {
  const items = cats[key].slice(0, CAP[key]);
  const body = items.map((x) => `- ${x.t.replace(/\n+/g, ' ').trim()}  \`[${x.proj}]\``).join('\n');
  return `## ${title}\n\n*${note}*\n\n${body}\n`;
};
fs.writeFileSync(path.join(OUT, 'narration/excerpts.md'), `# ${MODEL} narration — curated examples (authored by the model)

audience: AI coding agents first. Real, verbatim, terse ${MODEL} conversational lines, auto-filtered to
<=45 words and secret-scrubbed. Imitate the SHAPE, not the content.

${sect('Openers — first line of a turn (result-first, no self-warmup)', 'opener', "open with the object/outcome, not \"I'll\" or \"Let me\"")}
${sect('Tool preambles — text right before a tool call', 'preamble', 'name what + why in a fragment, then run the tool')}
${sect('Decision statements — committed, justified in a clause', 'decision', 'state the call as settled; one-clause why; no enumerated alternatives')}
${sect('Done signals — closing a unit of work', 'done', 'report outcome first; terse')}
`);

// ---- index.md ----
const byProj = {};
for (const d of manifest.docs) (byProj[d.proj] ||= []).push(d);
const total = manifest.docs.reduce((a, b) => a + b.chars, 0);
const med = manifest.docs.map((d) => d.chars).sort((a, b) => a - b)[Math.floor(manifest.docs.length / 2)];
let idx = `# ${MODEL} corpus — index

audience: AI coding agents first. Everything here was **authored by ${MODEL}**: ${manifest.docs.length}
docs (${(total / 1024 / 1024).toFixed(1)} MB, median ${med.toLocaleString()} chars) + curated narration.
Raw transcripts are deliberately excluded (not model-authored; secret risk). Live reasoning is NOT here —
\`thinking\` blocks are 0-char (encrypted). This corpus is behavior + artifacts, never cognition.

## How to use
1. **"What would ${MODEL} do here?" — query precedent FIRST** (planning/audit/architecture):
   \`python3 <skill>/pipeline/query_kb.py "<situation in keywords>"\`
2. Authoring shape? Open 1–2 docs in the relevant \`corpus/<project>/\`; imitate structure, not content.
3. Narration cadence? Read \`narration/excerpts.md\` (~50 lines); imitate line shape.
4. Do NOT load the whole corpus into context. Query/sample on demand.

## Docs by project

`;
for (const p of Object.keys(byProj).sort()) {
  const ds = byProj[p].sort((a, b) => b.chars - a.chars);
  idx += `### ${p} (${ds.length})\n\n`;
  for (const d of ds) idx += `- \`${path.basename(d.file)}\` — ${d.chars.toLocaleString()} chars${d.shared ? ' · shared-project' : ''}\n`;
  idx += '\n';
}
fs.writeFileSync(path.join(OUT, 'index.md'), idx);

console.log(`model: ${MODEL}`);
console.log(`authored docs: ${manifest.docs.length}  (median ${med.toLocaleString()} chars, ${(total / 1024 / 1024).toFixed(1)} MB)`);
console.log('narration:', Object.fromEntries(Object.entries(cats).map(([k, v]) => [k, Math.min(v.length, CAP[k])])));
console.log(`out: ${OUT}`);
