// src/kb/jaccardDedup.ts

function tokenSet(text: string): Set<string> {
  return new Set(text.toLowerCase().split(/\W+/).filter(t => t.length > 2));
}

function jaccard(a: string, b: string): number {
  const sa = tokenSet(a);
  const sb = tokenSet(b);
  let inter = 0;
  for (const t of sa) if (sb.has(t)) inter++;
  const union = sa.size + sb.size - inter;
  return union === 0 ? 0 : inter / union;
}

/**
 * Remove results too similar (Jaccard >= threshold) to an already-kept result.
 * Items must be sorted by score DESC — first of near-duplicates wins.
 */
export function jaccardDedup<T extends { content: string }>(items: T[], threshold: number): T[] {
  const kept: T[] = [];
  for (const item of items) {
    const tooSimilar = kept.some(k => jaccard(item.content, k.content) >= threshold);
    if (!tooSimilar) kept.push(item);
  }
  return kept;
}