feat(quality): measure and gate near-duplicate knowledge pages

src/rag-eval.ts

@@ -20,6 +20,12 @@ export type {
  RagRequiredContext,
 } from './rag-eval/contracts'
 export { scoreKnowledgeBaseIndex } from './rag-eval/knowledge-base'
+export type {
+ NearDuplicateDetectionOptions,
+ NearDuplicatePair,
+ NearDuplicateReport,
+} from './rag-eval/near-duplicates'
+export { detectNearDuplicatePages, normalizePageText } from './rag-eval/near-duplicates'
 export {
  normalizeExternalRagScores,
  toDeepEvalTestCases,

src/rag-eval/contracts.ts

@@ -1,6 +1,7 @@
 import type { ComparisonCost, JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign'
 import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface'
 import type { RagGapFinding } from '../rag-improvement-loop'
+import type { NearDuplicateDetectionOptions, NearDuplicateReport } from './near-duplicates'
 
 export type RagEvalProvider =
  | 'agent-knowledge'
@@ -166,6 +167,10 @@ export interface KnowledgeBaseQualityOptions {
  strict?: boolean
  minCitationRate?: number
  maxStaleSourceRate?: number
+ /** Deterministic exact/near-duplicate detector configuration. */
+ nearDuplicates?: NearDuplicateDetectionOptions
+ /** Maximum allowed fraction of eligible pages implicated in a duplicate pair. Defaults to 1. */
+ maxNearDuplicatePageRate?: number
 }
 
 export interface KnowledgeBaseQualityReport {
@@ -179,7 +184,13 @@ export interface KnowledgeBaseQualityReport {
  duplicate_source_hash_rate: number
  lint_error_count: number
  lint_warning_count: number
+ /** Added in the deterministic near-duplicate quality pass. */
+ near_duplicate_page_rate?: number
+ near_duplicate_page_count?: number
+ near_duplicate_pair_count?: number
  }
+ /** Exact report from the deterministic detector; present when produced by this package. */
+ nearDuplicates?: NearDuplicateReport
  findings: readonly RagGapFinding[]
 }
 

src/rag-eval/knowledge-base.ts

@@ -3,6 +3,7 @@ import type { RagGapFinding } from '../rag-improvement-loop'
 import type { KnowledgeIndex } from '../types'
 import { validateKnowledgeIndex } from '../validate'
 import type { KnowledgeBaseQualityOptions, KnowledgeBaseQualityReport } from './contracts'
+import { detectNearDuplicatePages } from './near-duplicates'
 
 export function scoreKnowledgeBaseIndex(
  index: KnowledgeIndex,
@@ -25,6 +26,11 @@ export function scoreKnowledgeBaseIndex(
  (page) => sourceRefs(page.text).length > 0,
  ).length
  const pagesWithSources = index.pages.filter((page) => page.sourceIds.length > 0).length
+ const nearDuplicates = detectNearDuplicatePages(index.pages, options.nearDuplicates)
+ const maxNearDuplicatePageRate = unitInterval(
+ options.maxNearDuplicatePageRate ?? 1,
+ 'maxNearDuplicatePageRate',
+ )
  const metrics = {
  page_count: index.pages.length,
  source_count: index.sources.length,
@@ -35,6 +41,9 @@ export function scoreKnowledgeBaseIndex(
  index.sources.length === 0 ? 0 : duplicateSourceHashCount / index.sources.length,
  lint_error_count: validation.findings.filter((finding) => finding.severity === 'error').length,
  lint_warning_count: lintFindings.filter((finding) => finding.severity === 'warning').length,
+ near_duplicate_page_rate: nearDuplicates.duplicatePageRate,
+ near_duplicate_page_count: nearDuplicates.duplicatePageCount,
+ near_duplicate_pair_count: nearDuplicates.duplicatePairCount,
  }
  const findings: RagGapFinding[] = []
  if (metrics.lint_error_count > 0) {
@@ -64,7 +73,34 @@ export function scoreKnowledgeBaseIndex(
  evidence: { stale_source_rate: metrics.stale_source_rate },
  })
  }
- return { ok: findings.length === 0, metrics, findings }
+ if (nearDuplicates.truncated && maxNearDuplicatePageRate < 1) {
+ findings.push({
+ id: 'kb:near-duplicate-analysis-truncated',
+ kind: 'unknown',
+ severity: 'error',
+ message:
+ 'Near-duplicate analysis reached a configured bound, so the duplicate-page gate cannot be evaluated completely.',
+ evidence: {
+ candidate_pair_count: nearDuplicates.candidatePairCount,
+ compared_pair_count: nearDuplicates.comparedPairCount,
+ duplicate_pair_count: nearDuplicates.duplicatePairCount,
+ },
+ })
+ } else if (metrics.near_duplicate_page_rate > maxNearDuplicatePageRate) {
+ findings.push({
+ id: 'kb:near-duplicate-page-rate',
+ kind: 'unknown',
+ severity: 'error',
+ message: 'Near-duplicate page rate exceeds the configured maximum.',
+ evidence: {
+ near_duplicate_page_rate: metrics.near_duplicate_page_rate,
+ near_duplicate_page_count: metrics.near_duplicate_page_count,
+ near_duplicate_pair_count: metrics.near_duplicate_pair_count,
+ threshold: nearDuplicates.threshold,
+ },
+ })
+ }
+ return { ok: findings.length === 0, metrics, nearDuplicates, findings }
 }
 
 function sourceRefs(text: string): string[] {
@@ -77,3 +113,10 @@ function sourceRefs(text: string): string[] {
  }
  return refs
 }
+
+function unitInterval(value: number, name: string): number {
+ if (!Number.isFinite(value) || value < 0 || value > 1) {
+ throw new Error(`${name} must be a finite number in [0, 1]`)
+ }
+ return value
+}

src/rag-eval/near-duplicates.test.ts

@@ -0,0 +1,204 @@
+import { describe, expect, it } from 'vitest'
+import type { KnowledgeIndex, KnowledgePage } from '../types'
+import { scoreKnowledgeBaseIndex } from './knowledge-base'
+import { detectNearDuplicatePages, normalizePageText } from './near-duplicates'
+
+const repeated =
+ 'A verified research page explains the mechanism, records the experiment, names the evidence, and preserves the result for later agents. '
+
+function page(id: string, title: string, text: string): KnowledgePage {
+ return {
+ id,
+ path: `knowledge/${id}.md`,
+ title,
+ text,
+ frontmatter: { id, title },
+ sourceIds: [],
+ tags: [],
+ outLinks: [],
+ }
+}
+
+function index(pages: KnowledgePage[]): KnowledgeIndex {
+ return {
+ root: '/kb',
+ generatedAt: '2026-08-17T00:00:00.000Z',
+ sources: [],
+ pages,
+ graph: { nodes: [], edges: [] },
+ }
+}
+
+describe('detectNearDuplicatePages', () => {
+ it('reports exact normalized copies with stable page identity', () => {
+ const pages = [
+ page('copy-a', 'Same title', repeated.repeat(2)),
+ page('copy-b', 'Same title', repeated.repeat(2)),
+ ]
+
+ const report = detectNearDuplicatePages(pages)
+
+ expect(report).toMatchObject({
+ pageCount: 2,
+ eligiblePageCount: 2,
+ candidatePairCount: 1,
+ comparedPairCount: 1,
+ duplicatePairCount: 1,
+ duplicatePageCount: 2,
+ duplicatePageRate: 1,
+ truncated: false,
+ })
+ expect(report.pairs).toEqual([
+ expect.objectContaining({
+ leftPageId: 'copy-a',
+ rightPageId: 'copy-b',
+ similarity: 1,
+ exact: true,
+ }),
+ ])
+ })
+
+ it('finds near copies but not unrelated prose', () => {
+ const base = `${repeated.repeat(2)} The decisive measured value is forty two.`
+ const revised = `${repeated.repeat(2)} The decisive measured value is forty three.`
+ const unrelated =
+ 'A kitchen inventory lists pans, knives, towels, plates, and groceries for a weekend dinner. '.repeat(
+ 3,
+ )
+
+ const report = detectNearDuplicatePages(
+ [
+ page('base', 'Experiment', base),
+ page('revision', 'Experiment', revised),
+ page('unrelated', 'Kitchen', unrelated),
+ ],
+ { threshold: 0.7 },
+ )
+
+ expect(report.duplicatePairCount).toBe(1)
+ expect(report.pairs[0]).toMatchObject({
+ leftPageId: 'base',
+ rightPageId: 'revision',
+ exact: false,
+ })
+ expect(report.pairs[0]!.similarity).toBeGreaterThanOrEqual(0.7)
+ })
+
+ it('is deterministic under input reordering', () => {
+ const pages = [
+ page('z', 'Same title', repeated.repeat(2)),
+ page('a', 'Same title', repeated.repeat(2)),
+ page('m', 'Different title', `${repeated.repeat(2)} extra material`),
+ ]
+
+ const forward = detectNearDuplicatePages(pages, { threshold: 0.7 })
+ const reverse = detectNearDuplicatePages([...pages].reverse(), { threshold: 0.7 })
+
+ expect(reverse).toEqual(forward)
+ })
+
+ it('excludes short pages and reports bounded candidate truncation', () => {
+ const short = detectNearDuplicatePages(
+ [page('short-a', 'Short', 'same'), page('short-b', 'Short', 'same')],
+ { minCharacters: 80 },
+ )
+ expect(short).toMatchObject({ eligiblePageCount: 0, duplicatePairCount: 0 })
+
+ const bounded = detectNearDuplicatePages(
+ Array.from({ length: 4 }, (_, index) =>
+ page(`copy-${index}`, 'Same title', repeated.repeat(2)),
+ ),
+ { maxCandidatePairs: 2 },
+ )
+ expect(bounded).toMatchObject({
+ candidatePairCount: 2,
+ comparedPairCount: 2,
+ truncated: true,
+ })
+ })
+
+ it('normalizes Unicode width, case, whitespace, and URL identity', () => {
+ expect(normalizePageText('ＡLPHA https://example.com/a\nBeta')).toBe('alpha <url> beta')
+ })
+
+ it('refuses malformed detector limits', () => {
+ expect(() => detectNearDuplicatePages([], { threshold: 1.1 })).toThrow(/threshold/)
+ expect(() => detectNearDuplicatePages([], { maxCandidatePairs: 0 })).toThrow(
+ /maxCandidatePairs/,
+ )
+ expect(() => detectNearDuplicatePages([], { minCharacters: -1 })).toThrow(/minCharacters/)
+ })
+})
+
+describe('scoreKnowledgeBaseIndex near-duplicate quality', () => {
+ const duplicatePages = [
+ page('copy-a', 'Same title', repeated.repeat(2)),
+ page('copy-b', 'Same title', repeated.repeat(2)),
+ page(
+ 'independent',
+ 'Independent result',
+ 'A separate proof studies a different theorem, with distinct assumptions and a different verification artifact. '.repeat(
+ 3,
+ ),
+ ),
+ ]
+
+ it('always reports deterministic duplicate metrics without failing by default', () => {
+ const report = scoreKnowledgeBaseIndex(index(duplicatePages))
+
+ expect(report.ok).toBe(true)
+ expect(report.metrics).toMatchObject({
+ near_duplicate_page_rate: 0.666667,
+ near_duplicate_page_count: 2,
+ near_duplicate_pair_count: 1,
+ })
+ expect(report.nearDuplicates?.pairs).toHaveLength(1)
+ })
+
+ it('fails a configured duplicate-page gate with measured evidence', () => {
+ const report = scoreKnowledgeBaseIndex(index(duplicatePages), {
+ maxNearDuplicatePageRate: 0.5,
+ })
+
+ expect(report.ok).toBe(false)
+ expect(report.findings).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ id: 'kb:near-duplicate-page-rate',
+ severity: 'error',
+ evidence: expect.objectContaining({
+ near_duplicate_page_rate: 0.666667,
+ near_duplicate_pair_count: 1,
+ }),
+ }),
+ ]),
+ )
+ })
+
+ it('fails closed when a configured gate rests on truncated analysis', () => {
+ const report = scoreKnowledgeBaseIndex(
+ index(
+ Array.from({ length: 4 }, (_, position) =>
+ page(`copy-${position}`, 'Same title', repeated.repeat(2)),
+ ),
+ ),
+ {
+ nearDuplicates: { maxCandidatePairs: 1 },
+ maxNearDuplicatePageRate: 0.9,
+ },
+ )
+
+ expect(report.ok).toBe(false)
+ expect(report.findings).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ id: 'kb:near-duplicate-analysis-truncated' }),
+ ]),
+ )
+ })
+
+ it('refuses a malformed quality threshold', () => {
+ expect(() =>
+ scoreKnowledgeBaseIndex(index(duplicatePages), { maxNearDuplicatePageRate: -0.1 }),
+ ).toThrow(/maxNearDuplicatePageRate/)
+ })
+})

src/rag-eval.ts

@@ -20,6 +20,12 @@ export type {
  RagRequiredContext,
 } from './rag-eval/contracts'
 export { scoreKnowledgeBaseIndex } from './rag-eval/knowledge-base'
+export type {
+ NearDuplicateDetectionOptions,
+ NearDuplicatePair,
+ NearDuplicateReport,
+} from './rag-eval/near-duplicates'
+export { detectNearDuplicatePages, normalizePageText } from './rag-eval/near-duplicates'
 export {
  normalizeExternalRagScores,
  toDeepEvalTestCases,

src/rag-eval/contracts.ts

@@ -1,6 +1,7 @@
 import type { ComparisonCost, JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign'
 import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface'
 import type { RagGapFinding } from '../rag-improvement-loop'
+import type { NearDuplicateDetectionOptions, NearDuplicateReport } from './near-duplicates'
 
 export type RagEvalProvider =
  | 'agent-knowledge'
@@ -166,6 +167,10 @@ export interface KnowledgeBaseQualityOptions {
  strict?: boolean
  minCitationRate?: number
  maxStaleSourceRate?: number
+ /** Deterministic exact/near-duplicate detector configuration. */
+ nearDuplicates?: NearDuplicateDetectionOptions
+ /** Maximum allowed fraction of eligible pages implicated in a duplicate pair. Defaults to 1. */
+ maxNearDuplicatePageRate?: number
 }
 
 export interface KnowledgeBaseQualityReport {
@@ -179,7 +184,13 @@ export interface KnowledgeBaseQualityReport {
  duplicate_source_hash_rate: number
  lint_error_count: number
  lint_warning_count: number
+ /** Added in the deterministic near-duplicate quality pass. */
+ near_duplicate_page_rate?: number
+ near_duplicate_page_count?: number
+ near_duplicate_pair_count?: number
  }
+ /** Exact report from the deterministic detector; present when produced by this package. */
+ nearDuplicates?: NearDuplicateReport
  findings: readonly RagGapFinding[]
 }
 

src/rag-eval/knowledge-base.ts

@@ -3,6 +3,7 @@ import type { RagGapFinding } from '../rag-improvement-loop'
 import type { KnowledgeIndex } from '../types'
 import { validateKnowledgeIndex } from '../validate'
 import type { KnowledgeBaseQualityOptions, KnowledgeBaseQualityReport } from './contracts'
+import { detectNearDuplicatePages } from './near-duplicates'
 
 export function scoreKnowledgeBaseIndex(
  index: KnowledgeIndex,
@@ -25,6 +26,11 @@ export function scoreKnowledgeBaseIndex(
  (page) => sourceRefs(page.text).length > 0,
  ).length
  const pagesWithSources = index.pages.filter((page) => page.sourceIds.length > 0).length
+ const nearDuplicates = detectNearDuplicatePages(index.pages, options.nearDuplicates)
+ const maxNearDuplicatePageRate = unitInterval(
+ options.maxNearDuplicatePageRate ?? 1,
+ 'maxNearDuplicatePageRate',
+ )
  const metrics = {
  page_count: index.pages.length,
  source_count: index.sources.length,
@@ -35,6 +41,9 @@ export function scoreKnowledgeBaseIndex(
  index.sources.length === 0 ? 0 : duplicateSourceHashCount / index.sources.length,
  lint_error_count: validation.findings.filter((finding) => finding.severity === 'error').length,
  lint_warning_count: lintFindings.filter((finding) => finding.severity === 'warning').length,
+ near_duplicate_page_rate: nearDuplicates.duplicatePageRate,
+ near_duplicate_page_count: nearDuplicates.duplicatePageCount,
+ near_duplicate_pair_count: nearDuplicates.duplicatePairCount,
  }
  const findings: RagGapFinding[] = []
  if (metrics.lint_error_count > 0) {
@@ -64,7 +73,34 @@ export function scoreKnowledgeBaseIndex(
  evidence: { stale_source_rate: metrics.stale_source_rate },
  })
  }
- return { ok: findings.length === 0, metrics, findings }
+ if (nearDuplicates.truncated && maxNearDuplicatePageRate < 1) {
+ findings.push({
+ id: 'kb:near-duplicate-analysis-truncated',
+ kind: 'unknown',
+ severity: 'error',
+ message:
+ 'Near-duplicate analysis reached a configured bound, so the duplicate-page gate cannot be evaluated completely.',
+ evidence: {
+ candidate_pair_count: nearDuplicates.candidatePairCount,
+ compared_pair_count: nearDuplicates.comparedPairCount,
+ duplicate_pair_count: nearDuplicates.duplicatePairCount,
+ },
+ })
+ } else if (metrics.near_duplicate_page_rate > maxNearDuplicatePageRate) {
+ findings.push({
+ id: 'kb:near-duplicate-page-rate',
+ kind: 'unknown',
+ severity: 'error',
+ message: 'Near-duplicate page rate exceeds the configured maximum.',
+ evidence: {
+ near_duplicate_page_rate: metrics.near_duplicate_page_rate,
+ near_duplicate_page_count: metrics.near_duplicate_page_count,
+ near_duplicate_pair_count: metrics.near_duplicate_pair_count,
+ threshold: nearDuplicates.threshold,
+ },
+ })
+ }
+ return { ok: findings.length === 0, metrics, nearDuplicates, findings }
 }
 
 function sourceRefs(text: string): string[] {
@@ -77,3 +113,10 @@ function sourceRefs(text: string): string[] {
  }
  return refs
 }
+
+function unitInterval(value: number, name: string): number {
+ if (!Number.isFinite(value) || value < 0 || value > 1) {
+ throw new Error(`${name} must be a finite number in [0, 1]`)
+ }
+ return value
+}

src/rag-eval/near-duplicates.test.ts

@@ -0,0 +1,204 @@
+import { describe, expect, it } from 'vitest'
+import type { KnowledgeIndex, KnowledgePage } from '../types'
+import { scoreKnowledgeBaseIndex } from './knowledge-base'
+import { detectNearDuplicatePages, normalizePageText } from './near-duplicates'
+
+const repeated =
+ 'A verified research page explains the mechanism, records the experiment, names the evidence, and preserves the result for later agents. '
+
+function page(id: string, title: string, text: string): KnowledgePage {
+ return {
+ id,
+ path: `knowledge/${id}.md`,
+ title,
+ text,
+ frontmatter: { id, title },
+ sourceIds: [],
+ tags: [],
+ outLinks: [],
+ }
+}
+
+function index(pages: KnowledgePage[]): KnowledgeIndex {
+ return {
+ root: '/kb',
+ generatedAt: '2026-08-17T00:00:00.000Z',
+ sources: [],
+ pages,
+ graph: { nodes: [], edges: [] },
+ }
+}
+
+describe('detectNearDuplicatePages', () => {
+ it('reports exact normalized copies with stable page identity', () => {
+ const pages = [
+ page('copy-a', 'Same title', repeated.repeat(2)),
+ page('copy-b', 'Same title', repeated.repeat(2)),
+ ]
+
+ const report = detectNearDuplicatePages(pages)
+
+ expect(report).toMatchObject({
+ pageCount: 2,
+ eligiblePageCount: 2,
+ candidatePairCount: 1,
+ comparedPairCount: 1,
+ duplicatePairCount: 1,
+ duplicatePageCount: 2,
+ duplicatePageRate: 1,
+ truncated: false,
+ })
+ expect(report.pairs).toEqual([
+ expect.objectContaining({
+ leftPageId: 'copy-a',
+ rightPageId: 'copy-b',
+ similarity: 1,
+ exact: true,
+ }),
+ ])
+ })
+
+ it('finds near copies but not unrelated prose', () => {
+ const base = `${repeated.repeat(2)} The decisive measured value is forty two.`
+ const revised = `${repeated.repeat(2)} The decisive measured value is forty three.`
+ const unrelated =
+ 'A kitchen inventory lists pans, knives, towels, plates, and groceries for a weekend dinner. '.repeat(
+ 3,
+ )
+
+ const report = detectNearDuplicatePages(
+ [
+ page('base', 'Experiment', base),
+ page('revision', 'Experiment', revised),
+ page('unrelated', 'Kitchen', unrelated),
+ ],
+ { threshold: 0.7 },
+ )
+
+ expect(report.duplicatePairCount).toBe(1)
+ expect(report.pairs[0]).toMatchObject({
+ leftPageId: 'base',
+ rightPageId: 'revision',
+ exact: false,
+ })
+ expect(report.pairs[0]!.similarity).toBeGreaterThanOrEqual(0.7)
+ })
+
+ it('is deterministic under input reordering', () => {
+ const pages = [
+ page('z', 'Same title', repeated.repeat(2)),
+ page('a', 'Same title', repeated.repeat(2)),
+ page('m', 'Different title', `${repeated.repeat(2)} extra material`),
+ ]
+
+ const forward = detectNearDuplicatePages(pages, { threshold: 0.7 })
+ const reverse = detectNearDuplicatePages([...pages].reverse(), { threshold: 0.7 })
+
+ expect(reverse).toEqual(forward)
+ })
+
+ it('excludes short pages and reports bounded candidate truncation', () => {
+ const short = detectNearDuplicatePages(
+ [page('short-a', 'Short', 'same'), page('short-b', 'Short', 'same')],
+ { minCharacters: 80 },
+ )
+ expect(short).toMatchObject({ eligiblePageCount: 0, duplicatePairCount: 0 })
+
+ const bounded = detectNearDuplicatePages(
+ Array.from({ length: 4 }, (_, index) =>
+ page(`copy-${index}`, 'Same title', repeated.repeat(2)),
+ ),
+ { maxCandidatePairs: 2 },
+ )
+ expect(bounded).toMatchObject({
+ candidatePairCount: 2,
+ comparedPairCount: 2,
+ truncated: true,
+ })
+ })
+
+ it('normalizes Unicode width, case, whitespace, and URL identity', () => {
+ expect(normalizePageText('ＡLPHA https://example.com/a\nBeta')).toBe('alpha <url> beta')
+ })
+
+ it('refuses malformed detector limits', () => {
+ expect(() => detectNearDuplicatePages([], { threshold: 1.1 })).toThrow(/threshold/)
+ expect(() => detectNearDuplicatePages([], { maxCandidatePairs: 0 })).toThrow(
+ /maxCandidatePairs/,
+ )
+ expect(() => detectNearDuplicatePages([], { minCharacters: -1 })).toThrow(/minCharacters/)
+ })
+})
+
+describe('scoreKnowledgeBaseIndex near-duplicate quality', () => {
+ const duplicatePages = [
+ page('copy-a', 'Same title', repeated.repeat(2)),
+ page('copy-b', 'Same title', repeated.repeat(2)),
+ page(
+ 'independent',
+ 'Independent result',
+ 'A separate proof studies a different theorem, with distinct assumptions and a different verification artifact. '.repeat(
+ 3,
+ ),
+ ),
+ ]
+
+ it('always reports deterministic duplicate metrics without failing by default', () => {
+ const report = scoreKnowledgeBaseIndex(index(duplicatePages))
+
+ expect(report.ok).toBe(true)
+ expect(report.metrics).toMatchObject({
+ near_duplicate_page_rate: 0.666667,
+ near_duplicate_page_count: 2,
+ near_duplicate_pair_count: 1,
+ })
+ expect(report.nearDuplicates?.pairs).toHaveLength(1)
+ })
+
+ it('fails a configured duplicate-page gate with measured evidence', () => {
+ const report = scoreKnowledgeBaseIndex(index(duplicatePages), {
+ maxNearDuplicatePageRate: 0.5,
+ })
+
+ expect(report.ok).toBe(false)
+ expect(report.findings).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ id: 'kb:near-duplicate-page-rate',
+ severity: 'error',
+ evidence: expect.objectContaining({
+ near_duplicate_page_rate: 0.666667,
+ near_duplicate_pair_count: 1,
+ }),
+ }),
+ ]),
+ )
+ })
+
+ it('fails closed when a configured gate rests on truncated analysis', () => {
+ const report = scoreKnowledgeBaseIndex(
+ index(
+ Array.from({ length: 4 }, (_, position) =>
+ page(`copy-${position}`, 'Same title', repeated.repeat(2)),
+ ),
+ ),
+ {
+ nearDuplicates: { maxCandidatePairs: 1 },
+ maxNearDuplicatePageRate: 0.9,
+ },
+ )
+
+ expect(report.ok).toBe(false)
+ expect(report.findings).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ id: 'kb:near-duplicate-analysis-truncated' }),
+ ]),
+ )
+ })
+
+ it('refuses a malformed quality threshold', () => {
+ expect(() =>
+ scoreKnowledgeBaseIndex(index(duplicatePages), { maxNearDuplicatePageRate: -0.1 }),
+ ).toThrow(/maxNearDuplicatePageRate/)
+ })
+})

src/rag-eval.ts

@@ -20,6 +20,12 @@ export type {
  RagRequiredContext,
 } from './rag-eval/contracts'
 export { scoreKnowledgeBaseIndex } from './rag-eval/knowledge-base'
+export type {
+ NearDuplicateDetectionOptions,
+ NearDuplicatePair,
+ NearDuplicateReport,
+} from './rag-eval/near-duplicates'
+export { detectNearDuplicatePages, normalizePageText } from './rag-eval/near-duplicates'
 export {
  normalizeExternalRagScores,
  toDeepEvalTestCases,

src/rag-eval/contracts.ts

@@ -1,6 +1,7 @@
 import type { ComparisonCost, JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign'
 import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface'
 import type { RagGapFinding } from '../rag-improvement-loop'
+import type { NearDuplicateDetectionOptions, NearDuplicateReport } from './near-duplicates'
 
 export type RagEvalProvider =
  | 'agent-knowledge'
@@ -166,6 +167,10 @@ export interface KnowledgeBaseQualityOptions {
  strict?: boolean
  minCitationRate?: number
  maxStaleSourceRate?: number
+ /** Deterministic exact/near-duplicate detector configuration. */
+ nearDuplicates?: NearDuplicateDetectionOptions
+ /** Maximum allowed fraction of eligible pages implicated in a duplicate pair. Defaults to 1. */
+ maxNearDuplicatePageRate?: number
 }
 
 export interface KnowledgeBaseQualityReport {
@@ -179,7 +184,13 @@ export interface KnowledgeBaseQualityReport {
  duplicate_source_hash_rate: number
  lint_error_count: number
  lint_warning_count: number
+ /** Added in the deterministic near-duplicate quality pass. */
+ near_duplicate_page_rate?: number
+ near_duplicate_page_count?: number
+ near_duplicate_pair_count?: number
  }
+ /** Exact report from the deterministic detector; present when produced by this package. */
+ nearDuplicates?: NearDuplicateReport
  findings: readonly RagGapFinding[]
 }
 

src/rag-eval/knowledge-base.ts

@@ -3,6 +3,7 @@ import type { RagGapFinding } from '../rag-improvement-loop'
 import type { KnowledgeIndex } from '../types'
 import { validateKnowledgeIndex } from '../validate'
 import type { KnowledgeBaseQualityOptions, KnowledgeBaseQualityReport } from './contracts'
+import { detectNearDuplicatePages } from './near-duplicates'
 
 export function scoreKnowledgeBaseIndex(
  index: KnowledgeIndex,
@@ -25,6 +26,11 @@ export function scoreKnowledgeBaseIndex(
  (page) => sourceRefs(page.text).length > 0,
  ).length
  const pagesWithSources = index.pages.filter((page) => page.sourceIds.length > 0).length
+ const nearDuplicates = detectNearDuplicatePages(index.pages, options.nearDuplicates)
+ const maxNearDuplicatePageRate = unitInterval(
+ options.maxNearDuplicatePageRate ?? 1,
+ 'maxNearDuplicatePageRate',
+ )
  const metrics = {
  page_count: index.pages.length,
  source_count: index.sources.length,
@@ -35,6 +41,9 @@ export function scoreKnowledgeBaseIndex(
  index.sources.length === 0 ? 0 : duplicateSourceHashCount / index.sources.length,
  lint_error_count: validation.findings.filter((finding) => finding.severity === 'error').length,
  lint_warning_count: lintFindings.filter((finding) => finding.severity === 'warning').length,
+ near_duplicate_page_rate: nearDuplicates.duplicatePageRate,
+ near_duplicate_page_count: nearDuplicates.duplicatePageCount,
+ near_duplicate_pair_count: nearDuplicates.duplicatePairCount,
  }
  const findings: RagGapFinding[] = []
  if (metrics.lint_error_count > 0) {
@@ -64,7 +73,34 @@ export function scoreKnowledgeBaseIndex(
  evidence: { stale_source_rate: metrics.stale_source_rate },
  })
  }
- return { ok: findings.length === 0, metrics, findings }
+ if (nearDuplicates.truncated && maxNearDuplicatePageRate < 1) {
+ findings.push({
+ id: 'kb:near-duplicate-analysis-truncated',
+ kind: 'unknown',
+ severity: 'error',
+ message:
+ 'Near-duplicate analysis reached a configured bound, so the duplicate-page gate cannot be evaluated completely.',
+ evidence: {
+ candidate_pair_count: nearDuplicates.candidatePairCount,
+ compared_pair_count: nearDuplicates.comparedPairCount,
+ duplicate_pair_count: nearDuplicates.duplicatePairCount,
+ },
+ })
+ } else if (metrics.near_duplicate_page_rate > maxNearDuplicatePageRate) {
+ findings.push({
+ id: 'kb:near-duplicate-page-rate',
+ kind: 'unknown',
+ severity: 'error',
+ message: 'Near-duplicate page rate exceeds the configured maximum.',
+ evidence: {
+ near_duplicate_page_rate: metrics.near_duplicate_page_rate,
+ near_duplicate_page_count: metrics.near_duplicate_page_count,
+ near_duplicate_pair_count: metrics.near_duplicate_pair_count,
+ threshold: nearDuplicates.threshold,
+ },
+ })
+ }
+ return { ok: findings.length === 0, metrics, nearDuplicates, findings }
 }
 
 function sourceRefs(text: string): string[] {
@@ -77,3 +113,10 @@ function sourceRefs(text: string): string[] {
  }
  return refs
 }
+
+function unitInterval(value: number, name: string): number {
+ if (!Number.isFinite(value) || value < 0 || value > 1) {
+ throw new Error(`${name} must be a finite number in [0, 1]`)
+ }
+ return value
+}

src/rag-eval/near-duplicates.test.ts

@@ -0,0 +1,204 @@
+import { describe, expect, it } from 'vitest'
+import type { KnowledgeIndex, KnowledgePage } from '../types'
+import { scoreKnowledgeBaseIndex } from './knowledge-base'
+import { detectNearDuplicatePages, normalizePageText } from './near-duplicates'
+
+const repeated =
+ 'A verified research page explains the mechanism, records the experiment, names the evidence, and preserves the result for later agents. '
+
+function page(id: string, title: string, text: string): KnowledgePage {
+ return {
+ id,
+ path: `knowledge/${id}.md`,
+ title,
+ text,
+ frontmatter: { id, title },
+ sourceIds: [],
+ tags: [],
+ outLinks: [],
+ }
+}
+
+function index(pages: KnowledgePage[]): KnowledgeIndex {
+ return {
+ root: '/kb',
+ generatedAt: '2026-08-17T00:00:00.000Z',
+ sources: [],
+ pages,
+ graph: { nodes: [], edges: [] },
+ }
+}
+
+describe('detectNearDuplicatePages', () => {
+ it('reports exact normalized copies with stable page identity', () => {
+ const pages = [
+ page('copy-a', 'Same title', repeated.repeat(2)),
+ page('copy-b', 'Same title', repeated.repeat(2)),
+ ]
+
+ const report = detectNearDuplicatePages(pages)
+
+ expect(report).toMatchObject({
+ pageCount: 2,
+ eligiblePageCount: 2,
+ candidatePairCount: 1,
+ comparedPairCount: 1,
+ duplicatePairCount: 1,
+ duplicatePageCount: 2,
+ duplicatePageRate: 1,
+ truncated: false,
+ })
+ expect(report.pairs).toEqual([
+ expect.objectContaining({
+ leftPageId: 'copy-a',
+ rightPageId: 'copy-b',
+ similarity: 1,
+ exact: true,
+ }),
+ ])
+ })
+
+ it('finds near copies but not unrelated prose', () => {
+ const base = `${repeated.repeat(2)} The decisive measured value is forty two.`
+ const revised = `${repeated.repeat(2)} The decisive measured value is forty three.`
+ const unrelated =
+ 'A kitchen inventory lists pans, knives, towels, plates, and groceries for a weekend dinner. '.repeat(
+ 3,
+ )
+
+ const report = detectNearDuplicatePages(
+ [
+ page('base', 'Experiment', base),
+ page('revision', 'Experiment', revised),
+ page('unrelated', 'Kitchen', unrelated),
+ ],
+ { threshold: 0.7 },
+ )
+
+ expect(report.duplicatePairCount).toBe(1)
+ expect(report.pairs[0]).toMatchObject({
+ leftPageId: 'base',
+ rightPageId: 'revision',
+ exact: false,
+ })
+ expect(report.pairs[0]!.similarity).toBeGreaterThanOrEqual(0.7)
+ })
+
+ it('is deterministic under input reordering', () => {
+ const pages = [
+ page('z', 'Same title', repeated.repeat(2)),
+ page('a', 'Same title', repeated.repeat(2)),
+ page('m', 'Different title', `${repeated.repeat(2)} extra material`),
+ ]
+
+ const forward = detectNearDuplicatePages(pages, { threshold: 0.7 })
+ const reverse = detectNearDuplicatePages([...pages].reverse(), { threshold: 0.7 })
+
+ expect(reverse).toEqual(forward)
+ })
+
+ it('excludes short pages and reports bounded candidate truncation', () => {
+ const short = detectNearDuplicatePages(
+ [page('short-a', 'Short', 'same'), page('short-b', 'Short', 'same')],
+ { minCharacters: 80 },
+ )
+ expect(short).toMatchObject({ eligiblePageCount: 0, duplicatePairCount: 0 })
+
+ const bounded = detectNearDuplicatePages(
+ Array.from({ length: 4 }, (_, index) =>
+ page(`copy-${index}`, 'Same title', repeated.repeat(2)),
+ ),
+ { maxCandidatePairs: 2 },
+ )
+ expect(bounded).toMatchObject({
+ candidatePairCount: 2,
+ comparedPairCount: 2,
+ truncated: true,
+ })
+ })
+
+ it('normalizes Unicode width, case, whitespace, and URL identity', () => {
+ expect(normalizePageText('ＡLPHA https://example.com/a\nBeta')).toBe('alpha <url> beta')
+ })
+
+ it('refuses malformed detector limits', () => {
+ expect(() => detectNearDuplicatePages([], { threshold: 1.1 })).toThrow(/threshold/)
+ expect(() => detectNearDuplicatePages([], { maxCandidatePairs: 0 })).toThrow(
+ /maxCandidatePairs/,
+ )
+ expect(() => detectNearDuplicatePages([], { minCharacters: -1 })).toThrow(/minCharacters/)
+ })
+})
+
+describe('scoreKnowledgeBaseIndex near-duplicate quality', () => {
+ const duplicatePages = [
+ page('copy-a', 'Same title', repeated.repeat(2)),
+ page('copy-b', 'Same title', repeated.repeat(2)),
+ page(
+ 'independent',
+ 'Independent result',
+ 'A separate proof studies a different theorem, with distinct assumptions and a different verification artifact. '.repeat(
+ 3,
+ ),
+ ),
+ ]
+
+ it('always reports deterministic duplicate metrics without failing by default', () => {
+ const report = scoreKnowledgeBaseIndex(index(duplicatePages))
+
+ expect(report.ok).toBe(true)
+ expect(report.metrics).toMatchObject({
+ near_duplicate_page_rate: 0.666667,
+ near_duplicate_page_count: 2,
+ near_duplicate_pair_count: 1,
+ })
+ expect(report.nearDuplicates?.pairs).toHaveLength(1)
+ })
+
+ it('fails a configured duplicate-page gate with measured evidence', () => {
+ const report = scoreKnowledgeBaseIndex(index(duplicatePages), {
+ maxNearDuplicatePageRate: 0.5,
+ })
+
+ expect(report.ok).toBe(false)
+ expect(report.findings).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ id: 'kb:near-duplicate-page-rate',
+ severity: 'error',
+ evidence: expect.objectContaining({
+ near_duplicate_page_rate: 0.666667,
+ near_duplicate_pair_count: 1,
+ }),
+ }),
+ ]),
+ )
+ })
+
+ it('fails closed when a configured gate rests on truncated analysis', () => {
+ const report = scoreKnowledgeBaseIndex(
+ index(
+ Array.from({ length: 4 }, (_, position) =>
+ page(`copy-${position}`, 'Same title', repeated.repeat(2)),
+ ),
+ ),
+ {
+ nearDuplicates: { maxCandidatePairs: 1 },
+ maxNearDuplicatePageRate: 0.9,
+ },
+ )
+
+ expect(report.ok).toBe(false)
+ expect(report.findings).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ id: 'kb:near-duplicate-analysis-truncated' }),
+ ]),
+ )
+ })
+
+ it('refuses a malformed quality threshold', () => {
+ expect(() =>
+ scoreKnowledgeBaseIndex(index(duplicatePages), { maxNearDuplicatePageRate: -0.1 }),
+ ).toThrow(/maxNearDuplicatePageRate/)
+ })
+})

src/rag-eval.ts

@@ -20,6 +20,12 @@ export type {
  RagRequiredContext,
 } from './rag-eval/contracts'
 export { scoreKnowledgeBaseIndex } from './rag-eval/knowledge-base'
+export type {
+ NearDuplicateDetectionOptions,
+ NearDuplicatePair,
+ NearDuplicateReport,
+} from './rag-eval/near-duplicates'
+export { detectNearDuplicatePages, normalizePageText } from './rag-eval/near-duplicates'
 export {
  normalizeExternalRagScores,
  toDeepEvalTestCases,

src/rag-eval/contracts.ts

@@ -1,6 +1,7 @@
 import type { ComparisonCost, JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign'
 import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface'
 import type { RagGapFinding } from '../rag-improvement-loop'
+import type { NearDuplicateDetectionOptions, NearDuplicateReport } from './near-duplicates'
 
 export type RagEvalProvider =
  | 'agent-knowledge'
@@ -166,6 +167,10 @@ export interface KnowledgeBaseQualityOptions {
  strict?: boolean
  minCitationRate?: number
  maxStaleSourceRate?: number
+ /** Deterministic exact/near-duplicate detector configuration. */
+ nearDuplicates?: NearDuplicateDetectionOptions
+ /** Maximum allowed fraction of eligible pages implicated in a duplicate pair. Defaults to 1. */
+ maxNearDuplicatePageRate?: number
 }
 
 export interface KnowledgeBaseQualityReport {
@@ -179,7 +184,13 @@ export interface KnowledgeBaseQualityReport {
  duplicate_source_hash_rate: number
  lint_error_count: number
  lint_warning_count: number
+ /** Added in the deterministic near-duplicate quality pass. */
+ near_duplicate_page_rate?: number
+ near_duplicate_page_count?: number
+ near_duplicate_pair_count?: number
  }
+ /** Exact report from the deterministic detector; present when produced by this package. */
+ nearDuplicates?: NearDuplicateReport
  findings: readonly RagGapFinding[]
 }
 

src/rag-eval/knowledge-base.ts

@@ -3,6 +3,7 @@ import type { RagGapFinding } from '../rag-improvement-loop'
 import type { KnowledgeIndex } from '../types'
 import { validateKnowledgeIndex } from '../validate'
 import type { KnowledgeBaseQualityOptions, KnowledgeBaseQualityReport } from './contracts'
+import { detectNearDuplicatePages } from './near-duplicates'
 
 export function scoreKnowledgeBaseIndex(
  index: KnowledgeIndex,
@@ -25,6 +26,11 @@ export function scoreKnowledgeBaseIndex(
  (page) => sourceRefs(page.text).length > 0,
  ).length
  const pagesWithSources = index.pages.filter((page) => page.sourceIds.length > 0).length
+ const nearDuplicates = detectNearDuplicatePages(index.pages, options.nearDuplicates)
+ const maxNearDuplicatePageRate = unitInterval(
+ options.maxNearDuplicatePageRate ?? 1,
+ 'maxNearDuplicatePageRate',
+ )
  const metrics = {
  page_count: index.pages.length,
  source_count: index.sources.length,
@@ -35,6 +41,9 @@ export function scoreKnowledgeBaseIndex(
  index.sources.length === 0 ? 0 : duplicateSourceHashCount / index.sources.length,
  lint_error_count: validation.findings.filter((finding) => finding.severity === 'error').length,
  lint_warning_count: lintFindings.filter((finding) => finding.severity === 'warning').length,
+ near_duplicate_page_rate: nearDuplicates.duplicatePageRate,
+ near_duplicate_page_count: nearDuplicates.duplicatePageCount,
+ near_duplicate_pair_count: nearDuplicates.duplicatePairCount,
  }
  const findings: RagGapFinding[] = []
  if (metrics.lint_error_count > 0) {
@@ -64,7 +73,34 @@ export function scoreKnowledgeBaseIndex(
  evidence: { stale_source_rate: metrics.stale_source_rate },
  })
  }
- return { ok: findings.length === 0, metrics, findings }
+ if (nearDuplicates.truncated && maxNearDuplicatePageRate < 1) {
+ findings.push({
+ id: 'kb:near-duplicate-analysis-truncated',
+ kind: 'unknown',
+ severity: 'error',
+ message:
+ 'Near-duplicate analysis reached a configured bound, so the duplicate-page gate cannot be evaluated completely.',
+ evidence: {
+ candidate_pair_count: nearDuplicates.candidatePairCount,
+ compared_pair_count: nearDuplicates.comparedPairCount,
+ duplicate_pair_count: nearDuplicates.duplicatePairCount,
+ },
+ })
+ } else if (metrics.near_duplicate_page_rate > maxNearDuplicatePageRate) {
+ findings.push({
+ id: 'kb:near-duplicate-page-rate',
+ kind: 'unknown',
+ severity: 'error',
+ message: 'Near-duplicate page rate exceeds the configured maximum.',
+ evidence: {
+ near_duplicate_page_rate: metrics.near_duplicate_page_rate,
+ near_duplicate_page_count: metrics.near_duplicate_page_count,
+ near_duplicate_pair_count: metrics.near_duplicate_pair_count,
+ threshold: nearDuplicates.threshold,
+ },
+ })
+ }
+ return { ok: findings.length === 0, metrics, nearDuplicates, findings }
 }
 
 function sourceRefs(text: string): string[] {
@@ -77,3 +113,10 @@ function sourceRefs(text: string): string[] {
  }
  return refs
 }
+
+function unitInterval(value: number, name: string): number {
+ if (!Number.isFinite(value) || value < 0 || value > 1) {
+ throw new Error(`${name} must be a finite number in [0, 1]`)
+ }
+ return value
+}

src/rag-eval/near-duplicates.test.ts

@@ -0,0 +1,204 @@
+import { describe, expect, it } from 'vitest'
+import type { KnowledgeIndex, KnowledgePage } from '../types'
+import { scoreKnowledgeBaseIndex } from './knowledge-base'
+import { detectNearDuplicatePages, normalizePageText } from './near-duplicates'
+
+const repeated =
+ 'A verified research page explains the mechanism, records the experiment, names the evidence, and preserves the result for later agents. '
+
+function page(id: string, title: string, text: string): KnowledgePage {
+ return {
+ id,
+ path: `knowledge/${id}.md`,
+ title,
+ text,
+ frontmatter: { id, title },
+ sourceIds: [],
+ tags: [],
+ outLinks: [],
+ }
+}
+
+function index(pages: KnowledgePage[]): KnowledgeIndex {
+ return {
+ root: '/kb',
+ generatedAt: '2026-08-17T00:00:00.000Z',
+ sources: [],
+ pages,
+ graph: { nodes: [], edges: [] },
+ }
+}
+
+describe('detectNearDuplicatePages', () => {
+ it('reports exact normalized copies with stable page identity', () => {
+ const pages = [
+ page('copy-a', 'Same title', repeated.repeat(2)),
+ page('copy-b', 'Same title', repeated.repeat(2)),
+ ]
+
+ const report = detectNearDuplicatePages(pages)
+
+ expect(report).toMatchObject({
+ pageCount: 2,
+ eligiblePageCount: 2,
+ candidatePairCount: 1,
+ comparedPairCount: 1,
+ duplicatePairCount: 1,
+ duplicatePageCount: 2,
+ duplicatePageRate: 1,
+ truncated: false,
+ })
+ expect(report.pairs).toEqual([
+ expect.objectContaining({
+ leftPageId: 'copy-a',
+ rightPageId: 'copy-b',
+ similarity: 1,
+ exact: true,
+ }),
+ ])
+ })
+
+ it('finds near copies but not unrelated prose', () => {
+ const base = `${repeated.repeat(2)} The decisive measured value is forty two.`
+ const revised = `${repeated.repeat(2)} The decisive measured value is forty three.`
+ const unrelated =
+ 'A kitchen inventory lists pans, knives, towels, plates, and groceries for a weekend dinner. '.repeat(
+ 3,
+ )
+
+ const report = detectNearDuplicatePages(
+ [
+ page('base', 'Experiment', base),
+ page('revision', 'Experiment', revised),
+ page('unrelated', 'Kitchen', unrelated),
+ ],
+ { threshold: 0.7 },
+ )
+
+ expect(report.duplicatePairCount).toBe(1)
+ expect(report.pairs[0]).toMatchObject({
+ leftPageId: 'base',
+ rightPageId: 'revision',
+ exact: false,
+ })
+ expect(report.pairs[0]!.similarity).toBeGreaterThanOrEqual(0.7)
+ })
+
+ it('is deterministic under input reordering', () => {
+ const pages = [
+ page('z', 'Same title', repeated.repeat(2)),
+ page('a', 'Same title', repeated.repeat(2)),
+ page('m', 'Different title', `${repeated.repeat(2)} extra material`),
+ ]
+
+ const forward = detectNearDuplicatePages(pages, { threshold: 0.7 })
+ const reverse = detectNearDuplicatePages([...pages].reverse(), { threshold: 0.7 })
+
+ expect(reverse).toEqual(forward)
+ })
+
+ it('excludes short pages and reports bounded candidate truncation', () => {
+ const short = detectNearDuplicatePages(
+ [page('short-a', 'Short', 'same'), page('short-b', 'Short', 'same')],
+ { minCharacters: 80 },
+ )
+ expect(short).toMatchObject({ eligiblePageCount: 0, duplicatePairCount: 0 })
+
+ const bounded = detectNearDuplicatePages(
+ Array.from({ length: 4 }, (_, index) =>
+ page(`copy-${index}`, 'Same title', repeated.repeat(2)),
+ ),
+ { maxCandidatePairs: 2 },
+ )
+ expect(bounded).toMatchObject({
+ candidatePairCount: 2,
+ comparedPairCount: 2,
+ truncated: true,
+ })
+ })
+
+ it('normalizes Unicode width, case, whitespace, and URL identity', () => {
+ expect(normalizePageText('ＡLPHA https://example.com/a\nBeta')).toBe('alpha <url> beta')
+ })
+
+ it('refuses malformed detector limits', () => {
+ expect(() => detectNearDuplicatePages([], { threshold: 1.1 })).toThrow(/threshold/)
+ expect(() => detectNearDuplicatePages([], { maxCandidatePairs: 0 })).toThrow(
+ /maxCandidatePairs/,
+ )
+ expect(() => detectNearDuplicatePages([], { minCharacters: -1 })).toThrow(/minCharacters/)
+ })
+})
+
+describe('scoreKnowledgeBaseIndex near-duplicate quality', () => {
+ const duplicatePages = [
+ page('copy-a', 'Same title', repeated.repeat(2)),
+ page('copy-b', 'Same title', repeated.repeat(2)),
+ page(
+ 'independent',
+ 'Independent result',
+ 'A separate proof studies a different theorem, with distinct assumptions and a different verification artifact. '.repeat(
+ 3,
+ ),
+ ),
+ ]
+
+ it('always reports deterministic duplicate metrics without failing by default', () => {
+ const report = scoreKnowledgeBaseIndex(index(duplicatePages))
+
+ expect(report.ok).toBe(true)
+ expect(report.metrics).toMatchObject({
+ near_duplicate_page_rate: 0.666667,
+ near_duplicate_page_count: 2,
+ near_duplicate_pair_count: 1,
+ })
+ expect(report.nearDuplicates?.pairs).toHaveLength(1)
+ })
+
+ it('fails a configured duplicate-page gate with measured evidence', () => {
+ const report = scoreKnowledgeBaseIndex(index(duplicatePages), {
+ maxNearDuplicatePageRate: 0.5,
+ })
+
+ expect(report.ok).toBe(false)
+ expect(report.findings).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ id: 'kb:near-duplicate-page-rate',
+ severity: 'error',
+ evidence: expect.objectContaining({
+ near_duplicate_page_rate: 0.666667,
+ near_duplicate_pair_count: 1,
+ }),
+ }),
+ ]),
+ )
+ })
+
+ it('fails closed when a configured gate rests on truncated analysis', () => {
+ const report = scoreKnowledgeBaseIndex(
+ index(
+ Array.from({ length: 4 }, (_, position) =>
+ page(`copy-${position}`, 'Same title', repeated.repeat(2)),
+ ),
+ ),
+ {
+ nearDuplicates: { maxCandidatePairs: 1 },
+ maxNearDuplicatePageRate: 0.9,
+ },
+ )
+
+ expect(report.ok).toBe(false)
+ expect(report.findings).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ id: 'kb:near-duplicate-analysis-truncated' }),
+ ]),
+ )
+ })
+
+ it('refuses a malformed quality threshold', () => {
+ expect(() =>
+ scoreKnowledgeBaseIndex(index(duplicatePages), { maxNearDuplicatePageRate: -0.1 }),
+ ).toThrow(/maxNearDuplicatePageRate/)
+ })
+})

src/rag-eval.ts

@@ -20,6 +20,12 @@ export type {
  RagRequiredContext,
 } from './rag-eval/contracts'
 export { scoreKnowledgeBaseIndex } from './rag-eval/knowledge-base'
+export type {
+ NearDuplicateDetectionOptions,
+ NearDuplicatePair,
+ NearDuplicateReport,
+} from './rag-eval/near-duplicates'
+export { detectNearDuplicatePages, normalizePageText } from './rag-eval/near-duplicates'
 export {
  normalizeExternalRagScores,
  toDeepEvalTestCases,

src/rag-eval/contracts.ts

@@ -1,6 +1,7 @@
 import type { ComparisonCost, JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign'
 import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface'
 import type { RagGapFinding } from '../rag-improvement-loop'
+import type { NearDuplicateDetectionOptions, NearDuplicateReport } from './near-duplicates'
 
 export type RagEvalProvider =
  | 'agent-knowledge'
@@ -166,6 +167,10 @@ export interface KnowledgeBaseQualityOptions {
  strict?: boolean
  minCitationRate?: number
  maxStaleSourceRate?: number
+ /** Deterministic exact/near-duplicate detector configuration. */
+ nearDuplicates?: NearDuplicateDetectionOptions
+ /** Maximum allowed fraction of eligible pages implicated in a duplicate pair. Defaults to 1. */
+ maxNearDuplicatePageRate?: number
 }
 
 export interface KnowledgeBaseQualityReport {
@@ -179,7 +184,13 @@ export interface KnowledgeBaseQualityReport {
  duplicate_source_hash_rate: number
  lint_error_count: number
  lint_warning_count: number
+ /** Added in the deterministic near-duplicate quality pass. */
+ near_duplicate_page_rate?: number
+ near_duplicate_page_count?: number
+ near_duplicate_pair_count?: number
  }
+ /** Exact report from the deterministic detector; present when produced by this package. */
+ nearDuplicates?: NearDuplicateReport
  findings: readonly RagGapFinding[]
 }
 

src/rag-eval/knowledge-base.ts

@@ -3,6 +3,7 @@ import type { RagGapFinding } from '../rag-improvement-loop'
 import type { KnowledgeIndex } from '../types'
 import { validateKnowledgeIndex } from '../validate'
 import type { KnowledgeBaseQualityOptions, KnowledgeBaseQualityReport } from './contracts'
+import { detectNearDuplicatePages } from './near-duplicates'
 
 export function scoreKnowledgeBaseIndex(
  index: KnowledgeIndex,
@@ -25,6 +26,11 @@ export function scoreKnowledgeBaseIndex(
  (page) => sourceRefs(page.text).length > 0,
  ).length
  const pagesWithSources = index.pages.filter((page) => page.sourceIds.length > 0).length
+ const nearDuplicates = detectNearDuplicatePages(index.pages, options.nearDuplicates)
+ const maxNearDuplicatePageRate = unitInterval(
+ options.maxNearDuplicatePageRate ?? 1,
+ 'maxNearDuplicatePageRate',
+ )
  const metrics = {
  page_count: index.pages.length,
  source_count: index.sources.length,
@@ -35,6 +41,9 @@ export function scoreKnowledgeBaseIndex(
  index.sources.length === 0 ? 0 : duplicateSourceHashCount / index.sources.length,
  lint_error_count: validation.findings.filter((finding) => finding.severity === 'error').length,
  lint_warning_count: lintFindings.filter((finding) => finding.severity === 'warning').length,
+ near_duplicate_page_rate: nearDuplicates.duplicatePageRate,
+ near_duplicate_page_count: nearDuplicates.duplicatePageCount,
+ near_duplicate_pair_count: nearDuplicates.duplicatePairCount,
  }
  const findings: RagGapFinding[] = []
  if (metrics.lint_error_count > 0) {
@@ -64,7 +73,34 @@ export function scoreKnowledgeBaseIndex(
  evidence: { stale_source_rate: metrics.stale_source_rate },
  })
  }
- return { ok: findings.length === 0, metrics, findings }
+ if (nearDuplicates.truncated && maxNearDuplicatePageRate < 1) {
+ findings.push({
+ id: 'kb:near-duplicate-analysis-truncated',
+ kind: 'unknown',
+ severity: 'error',
+ message:
+ 'Near-duplicate analysis reached a configured bound, so the duplicate-page gate cannot be evaluated completely.',
+ evidence: {
+ candidate_pair_count: nearDuplicates.candidatePairCount,
+ compared_pair_count: nearDuplicates.comparedPairCount,
+ duplicate_pair_count: nearDuplicates.duplicatePairCount,
+ },
+ })
+ } else if (metrics.near_duplicate_page_rate > maxNearDuplicatePageRate) {
+ findings.push({
+ id: 'kb:near-duplicate-page-rate',
+ kind: 'unknown',
+ severity: 'error',
+ message: 'Near-duplicate page rate exceeds the configured maximum.',
+ evidence: {
+ near_duplicate_page_rate: metrics.near_duplicate_page_rate,
+ near_duplicate_page_count: metrics.near_duplicate_page_count,
+ near_duplicate_pair_count: metrics.near_duplicate_pair_count,
+ threshold: nearDuplicates.threshold,
+ },
+ })
+ }
+ return { ok: findings.length === 0, metrics, nearDuplicates, findings }
 }
 
 function sourceRefs(text: string): string[] {
@@ -77,3 +113,10 @@ function sourceRefs(text: string): string[] {
  }
  return refs
 }
+
+function unitInterval(value: number, name: string): number {
+ if (!Number.isFinite(value) || value < 0 || value > 1) {
+ throw new Error(`${name} must be a finite number in [0, 1]`)
+ }
+ return value
+}

src/rag-eval/near-duplicates.test.ts

@@ -0,0 +1,204 @@
+import { describe, expect, it } from 'vitest'
+import type { KnowledgeIndex, KnowledgePage } from '../types'
+import { scoreKnowledgeBaseIndex } from './knowledge-base'
+import { detectNearDuplicatePages, normalizePageText } from './near-duplicates'
+
+const repeated =
+ 'A verified research page explains the mechanism, records the experiment, names the evidence, and preserves the result for later agents. '
+
+function page(id: string, title: string, text: string): KnowledgePage {
+ return {
+ id,
+ path: `knowledge/${id}.md`,
+ title,
+ text,
+ frontmatter: { id, title },
+ sourceIds: [],
+ tags: [],
+ outLinks: [],
+ }
+}
+
+function index(pages: KnowledgePage[]): KnowledgeIndex {
+ return {
+ root: '/kb',
+ generatedAt: '2026-08-17T00:00:00.000Z',
+ sources: [],
+ pages,
+ graph: { nodes: [], edges: [] },
+ }
+}
+
+describe('detectNearDuplicatePages', () => {
+ it('reports exact normalized copies with stable page identity', () => {
+ const pages = [
+ page('copy-a', 'Same title', repeated.repeat(2)),
+ page('copy-b', 'Same title', repeated.repeat(2)),
+ ]
+
+ const report = detectNearDuplicatePages(pages)
+
+ expect(report).toMatchObject({
+ pageCount: 2,
+ eligiblePageCount: 2,
+ candidatePairCount: 1,
+ comparedPairCount: 1,
+ duplicatePairCount: 1,
+ duplicatePageCount: 2,
+ duplicatePageRate: 1,
+ truncated: false,
+ })
+ expect(report.pairs).toEqual([
+ expect.objectContaining({
+ leftPageId: 'copy-a',
+ rightPageId: 'copy-b',
+ similarity: 1,
+ exact: true,
+ }),
+ ])
+ })
+
+ it('finds near copies but not unrelated prose', () => {
+ const base = `${repeated.repeat(2)} The decisive measured value is forty two.`
+ const revised = `${repeated.repeat(2)} The decisive measured value is forty three.`
+ const unrelated =
+ 'A kitchen inventory lists pans, knives, towels, plates, and groceries for a weekend dinner. '.repeat(
+ 3,
+ )
+
+ const report = detectNearDuplicatePages(
+ [
+ page('base', 'Experiment', base),
+ page('revision', 'Experiment', revised),
+ page('unrelated', 'Kitchen', unrelated),
+ ],
+ { threshold: 0.7 },
+ )
+
+ expect(report.duplicatePairCount).toBe(1)
+ expect(report.pairs[0]).toMatchObject({
+ leftPageId: 'base',
+ rightPageId: 'revision',
+ exact: false,
+ })
+ expect(report.pairs[0]!.similarity).toBeGreaterThanOrEqual(0.7)
+ })
+
+ it('is deterministic under input reordering', () => {
+ const pages = [
+ page('z', 'Same title', repeated.repeat(2)),
+ page('a', 'Same title', repeated.repeat(2)),
+ page('m', 'Different title', `${repeated.repeat(2)} extra material`),
+ ]
+
+ const forward = detectNearDuplicatePages(pages, { threshold: 0.7 })
+ const reverse = detectNearDuplicatePages([...pages].reverse(), { threshold: 0.7 })
+
+ expect(reverse).toEqual(forward)
+ })
+
+ it('excludes short pages and reports bounded candidate truncation', () => {
+ const short = detectNearDuplicatePages(
+ [page('short-a', 'Short', 'same'), page('short-b', 'Short', 'same')],
+ { minCharacters: 80 },
+ )
+ expect(short).toMatchObject({ eligiblePageCount: 0, duplicatePairCount: 0 })
+
+ const bounded = detectNearDuplicatePages(
+ Array.from({ length: 4 }, (_, index) =>
+ page(`copy-${index}`, 'Same title', repeated.repeat(2)),
+ ),
+ { maxCandidatePairs: 2 },
+ )
+ expect(bounded).toMatchObject({
+ candidatePairCount: 2,
+ comparedPairCount: 2,
+ truncated: true,
+ })
+ })
+
+ it('normalizes Unicode width, case, whitespace, and URL identity', () => {
+ expect(normalizePageText('ＡLPHA https://example.com/a\nBeta')).toBe('alpha <url> beta')
+ })
+
+ it('refuses malformed detector limits', () => {
+ expect(() => detectNearDuplicatePages([], { threshold: 1.1 })).toThrow(/threshold/)
+ expect(() => detectNearDuplicatePages([], { maxCandidatePairs: 0 })).toThrow(
+ /maxCandidatePairs/,
+ )
+ expect(() => detectNearDuplicatePages([], { minCharacters: -1 })).toThrow(/minCharacters/)
+ })
+})
+
+describe('scoreKnowledgeBaseIndex near-duplicate quality', () => {
+ const duplicatePages = [
+ page('copy-a', 'Same title', repeated.repeat(2)),
+ page('copy-b', 'Same title', repeated.repeat(2)),
+ page(
+ 'independent',
+ 'Independent result',
+ 'A separate proof studies a different theorem, with distinct assumptions and a different verification artifact. '.repeat(
+ 3,
+ ),
+ ),
+ ]
+
+ it('always reports deterministic duplicate metrics without failing by default', () => {
+ const report = scoreKnowledgeBaseIndex(index(duplicatePages))
+
+ expect(report.ok).toBe(true)
+ expect(report.metrics).toMatchObject({
+ near_duplicate_page_rate: 0.666667,
+ near_duplicate_page_count: 2,
+ near_duplicate_pair_count: 1,
+ })
+ expect(report.nearDuplicates?.pairs).toHaveLength(1)
+ })
+
+ it('fails a configured duplicate-page gate with measured evidence', () => {
+ const report = scoreKnowledgeBaseIndex(index(duplicatePages), {
+ maxNearDuplicatePageRate: 0.5,
+ })
+
+ expect(report.ok).toBe(false)
+ expect(report.findings).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ id: 'kb:near-duplicate-page-rate',
+ severity: 'error',
+ evidence: expect.objectContaining({
+ near_duplicate_page_rate: 0.666667,
+ near_duplicate_pair_count: 1,
+ }),
+ }),
+ ]),
+ )
+ })
+
+ it('fails closed when a configured gate rests on truncated analysis', () => {
+ const report = scoreKnowledgeBaseIndex(
+ index(
+ Array.from({ length: 4 }, (_, position) =>
+ page(`copy-${position}`, 'Same title', repeated.repeat(2)),
+ ),
+ ),
+ {
+ nearDuplicates: { maxCandidatePairs: 1 },
+ maxNearDuplicatePageRate: 0.9,
+ },
+ )
+
+ expect(report.ok).toBe(false)
+ expect(report.findings).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ id: 'kb:near-duplicate-analysis-truncated' }),
+ ]),
+ )
+ })
+
+ it('refuses a malformed quality threshold', () => {
+ expect(() =>
+ scoreKnowledgeBaseIndex(index(duplicatePages), { maxNearDuplicatePageRate: -0.1 }),
+ ).toThrow(/maxNearDuplicatePageRate/)
+ })
+})

src/rag-eval.ts

@@ -20,6 +20,12 @@ export type {
  RagRequiredContext,
 } from './rag-eval/contracts'
 export { scoreKnowledgeBaseIndex } from './rag-eval/knowledge-base'
+export type {
+ NearDuplicateDetectionOptions,
+ NearDuplicatePair,
+ NearDuplicateReport,
+} from './rag-eval/near-duplicates'
+export { detectNearDuplicatePages, normalizePageText } from './rag-eval/near-duplicates'
 export {
  normalizeExternalRagScores,
  toDeepEvalTestCases,

src/rag-eval/contracts.ts

@@ -1,6 +1,7 @@
 import type { ComparisonCost, JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign'
 import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface'
 import type { RagGapFinding } from '../rag-improvement-loop'
+import type { NearDuplicateDetectionOptions, NearDuplicateReport } from './near-duplicates'
 
 export type RagEvalProvider =
  | 'agent-knowledge'
@@ -166,6 +167,10 @@ export interface KnowledgeBaseQualityOptions {
  strict?: boolean
  minCitationRate?: number
  maxStaleSourceRate?: number
+ /** Deterministic exact/near-duplicate detector configuration. */
+ nearDuplicates?: NearDuplicateDetectionOptions
+ /** Maximum allowed fraction of eligible pages implicated in a duplicate pair. Defaults to 1. */
+ maxNearDuplicatePageRate?: number
 }
 
 export interface KnowledgeBaseQualityReport {
@@ -179,7 +184,13 @@ export interface KnowledgeBaseQualityReport {
  duplicate_source_hash_rate: number
  lint_error_count: number
  lint_warning_count: number
+ /** Added in the deterministic near-duplicate quality pass. */
+ near_duplicate_page_rate?: number
+ near_duplicate_page_count?: number
+ near_duplicate_pair_count?: number
  }
+ /** Exact report from the deterministic detector; present when produced by this package. */
+ nearDuplicates?: NearDuplicateReport
  findings: readonly RagGapFinding[]
 }
 

src/rag-eval/knowledge-base.ts

@@ -3,6 +3,7 @@ import type { RagGapFinding } from '../rag-improvement-loop'
 import type { KnowledgeIndex } from '../types'
 import { validateKnowledgeIndex } from '../validate'
 import type { KnowledgeBaseQualityOptions, KnowledgeBaseQualityReport } from './contracts'
+import { detectNearDuplicatePages } from './near-duplicates'
 
 export function scoreKnowledgeBaseIndex(
  index: KnowledgeIndex,
@@ -25,6 +26,11 @@ export function scoreKnowledgeBaseIndex(
  (page) => sourceRefs(page.text).length > 0,
  ).length
  const pagesWithSources = index.pages.filter((page) => page.sourceIds.length > 0).length
+ const nearDuplicates = detectNearDuplicatePages(index.pages, options.nearDuplicates)
+ const maxNearDuplicatePageRate = unitInterval(
+ options.maxNearDuplicatePageRate ?? 1,
+ 'maxNearDuplicatePageRate',
+ )
  const metrics = {
  page_count: index.pages.length,
  source_count: index.sources.length,
@@ -35,6 +41,9 @@ export function scoreKnowledgeBaseIndex(
  index.sources.length === 0 ? 0 : duplicateSourceHashCount / index.sources.length,
  lint_error_count: validation.findings.filter((finding) => finding.severity === 'error').length,
  lint_warning_count: lintFindings.filter((finding) => finding.severity === 'warning').length,
+ near_duplicate_page_rate: nearDuplicates.duplicatePageRate,
+ near_duplicate_page_count: nearDuplicates.duplicatePageCount,
+ near_duplicate_pair_count: nearDuplicates.duplicatePairCount,
  }
  const findings: RagGapFinding[] = []
  if (metrics.lint_error_count > 0) {
@@ -64,7 +73,34 @@ export function scoreKnowledgeBaseIndex(
  evidence: { stale_source_rate: metrics.stale_source_rate },
  })
  }
- return { ok: findings.length === 0, metrics, findings }
+ if (nearDuplicates.truncated && maxNearDuplicatePageRate < 1) {
+ findings.push({
+ id: 'kb:near-duplicate-analysis-truncated',
+ kind: 'unknown',
+ severity: 'error',
+ message:
+ 'Near-duplicate analysis reached a configured bound, so the duplicate-page gate cannot be evaluated completely.',
+ evidence: {
+ candidate_pair_count: nearDuplicates.candidatePairCount,
+ compared_pair_count: nearDuplicates.comparedPairCount,
+ duplicate_pair_count: nearDuplicates.duplicatePairCount,
+ },
+ })
+ } else if (metrics.near_duplicate_page_rate > maxNearDuplicatePageRate) {
+ findings.push({
+ id: 'kb:near-duplicate-page-rate',
+ kind: 'unknown',
+ severity: 'error',
+ message: 'Near-duplicate page rate exceeds the configured maximum.',
+ evidence: {
+ near_duplicate_page_rate: metrics.near_duplicate_page_rate,
+ near_duplicate_page_count: metrics.near_duplicate_page_count,
+ near_duplicate_pair_count: metrics.near_duplicate_pair_count,
+ threshold: nearDuplicates.threshold,
+ },
+ })
+ }
+ return { ok: findings.length === 0, metrics, nearDuplicates, findings }
 }
 
 function sourceRefs(text: string): string[] {
@@ -77,3 +113,10 @@ function sourceRefs(text: string): string[] {
  }
  return refs
 }
+
+function unitInterval(value: number, name: string): number {
+ if (!Number.isFinite(value) || value < 0 || value > 1) {
+ throw new Error(`${name} must be a finite number in [0, 1]`)
+ }
+ return value
+}

src/rag-eval/near-duplicates.test.ts

@@ -0,0 +1,204 @@
+import { describe, expect, it } from 'vitest'
+import type { KnowledgeIndex, KnowledgePage } from '../types'
+import { scoreKnowledgeBaseIndex } from './knowledge-base'
+import { detectNearDuplicatePages, normalizePageText } from './near-duplicates'
+
+const repeated =
+ 'A verified research page explains the mechanism, records the experiment, names the evidence, and preserves the result for later agents. '
+
+function page(id: string, title: string, text: string): KnowledgePage {
+ return {
+ id,
+ path: `knowledge/${id}.md`,
+ title,
+ text,
+ frontmatter: { id, title },
+ sourceIds: [],
+ tags: [],
+ outLinks: [],
+ }
+}
+
+function index(pages: KnowledgePage[]): KnowledgeIndex {
+ return {
+ root: '/kb',
+ generatedAt: '2026-08-17T00:00:00.000Z',
+ sources: [],
+ pages,
+ graph: { nodes: [], edges: [] },
+ }
+}
+
+describe('detectNearDuplicatePages', () => {
+ it('reports exact normalized copies with stable page identity', () => {
+ const pages = [
+ page('copy-a', 'Same title', repeated.repeat(2)),
+ page('copy-b', 'Same title', repeated.repeat(2)),
+ ]
+
+ const report = detectNearDuplicatePages(pages)
+
+ expect(report).toMatchObject({
+ pageCount: 2,
+ eligiblePageCount: 2,
+ candidatePairCount: 1,
+ comparedPairCount: 1,
+ duplicatePairCount: 1,
+ duplicatePageCount: 2,
+ duplicatePageRate: 1,
+ truncated: false,
+ })
+ expect(report.pairs).toEqual([
+ expect.objectContaining({
+ leftPageId: 'copy-a',
+ rightPageId: 'copy-b',
+ similarity: 1,
+ exact: true,
+ }),
+ ])
+ })
+
+ it('finds near copies but not unrelated prose', () => {
+ const base = `${repeated.repeat(2)} The decisive measured value is forty two.`
+ const revised = `${repeated.repeat(2)} The decisive measured value is forty three.`
+ const unrelated =
+ 'A kitchen inventory lists pans, knives, towels, plates, and groceries for a weekend dinner. '.repeat(
+ 3,
+ )
+
+ const report = detectNearDuplicatePages(
+ [
+ page('base', 'Experiment', base),
+ page('revision', 'Experiment', revised),
+ page('unrelated', 'Kitchen', unrelated),
+ ],
+ { threshold: 0.7 },
+ )
+
+ expect(report.duplicatePairCount).toBe(1)
+ expect(report.pairs[0]).toMatchObject({
+ leftPageId: 'base',
+ rightPageId: 'revision',
+ exact: false,
+ })
+ expect(report.pairs[0]!.similarity).toBeGreaterThanOrEqual(0.7)
+ })
+
+ it('is deterministic under input reordering', () => {
+ const pages = [
+ page('z', 'Same title', repeated.repeat(2)),
+ page('a', 'Same title', repeated.repeat(2)),
+ page('m', 'Different title', `${repeated.repeat(2)} extra material`),
+ ]
+
+ const forward = detectNearDuplicatePages(pages, { threshold: 0.7 })
+ const reverse = detectNearDuplicatePages([...pages].reverse(), { threshold: 0.7 })
+
+ expect(reverse).toEqual(forward)
+ })
+
+ it('excludes short pages and reports bounded candidate truncation', () => {
+ const short = detectNearDuplicatePages(
+ [page('short-a', 'Short', 'same'), page('short-b', 'Short', 'same')],
+ { minCharacters: 80 },
+ )
+ expect(short).toMatchObject({ eligiblePageCount: 0, duplicatePairCount: 0 })
+
+ const bounded = detectNearDuplicatePages(
+ Array.from({ length: 4 }, (_, index) =>
+ page(`copy-${index}`, 'Same title', repeated.repeat(2)),
+ ),
+ { maxCandidatePairs: 2 },
+ )
+ expect(bounded).toMatchObject({
+ candidatePairCount: 2,
+ comparedPairCount: 2,
+ truncated: true,
+ })
+ })
+
+ it('normalizes Unicode width, case, whitespace, and URL identity', () => {
+ expect(normalizePageText('ＡLPHA https://example.com/a\nBeta')).toBe('alpha <url> beta')
+ })
+
+ it('refuses malformed detector limits', () => {
+ expect(() => detectNearDuplicatePages([], { threshold: 1.1 })).toThrow(/threshold/)
+ expect(() => detectNearDuplicatePages([], { maxCandidatePairs: 0 })).toThrow(
+ /maxCandidatePairs/,
+ )
+ expect(() => detectNearDuplicatePages([], { minCharacters: -1 })).toThrow(/minCharacters/)
+ })
+})
+
+describe('scoreKnowledgeBaseIndex near-duplicate quality', () => {
+ const duplicatePages = [
+ page('copy-a', 'Same title', repeated.repeat(2)),
+ page('copy-b', 'Same title', repeated.repeat(2)),
+ page(
+ 'independent',
+ 'Independent result',
+ 'A separate proof studies a different theorem, with distinct assumptions and a different verification artifact. '.repeat(
+ 3,
+ ),
+ ),
+ ]
+
+ it('always reports deterministic duplicate metrics without failing by default', () => {
+ const report = scoreKnowledgeBaseIndex(index(duplicatePages))
+
+ expect(report.ok).toBe(true)
+ expect(report.metrics).toMatchObject({
+ near_duplicate_page_rate: 0.666667,
+ near_duplicate_page_count: 2,
+ near_duplicate_pair_count: 1,
+ })
+ expect(report.nearDuplicates?.pairs).toHaveLength(1)
+ })
+
+ it('fails a configured duplicate-page gate with measured evidence', () => {
+ const report = scoreKnowledgeBaseIndex(index(duplicatePages), {
+ maxNearDuplicatePageRate: 0.5,
+ })
+
+ expect(report.ok).toBe(false)
+ expect(report.findings).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ id: 'kb:near-duplicate-page-rate',
+ severity: 'error',
+ evidence: expect.objectContaining({
+ near_duplicate_page_rate: 0.666667,
+ near_duplicate_pair_count: 1,
+ }),
+ }),
+ ]),
+ )
+ })
+
+ it('fails closed when a configured gate rests on truncated analysis', () => {
+ const report = scoreKnowledgeBaseIndex(
+ index(
+ Array.from({ length: 4 }, (_, position) =>
+ page(`copy-${position}`, 'Same title', repeated.repeat(2)),
+ ),
+ ),
+ {
+ nearDuplicates: { maxCandidatePairs: 1 },
+ maxNearDuplicatePageRate: 0.9,
+ },
+ )
+
+ expect(report.ok).toBe(false)
+ expect(report.findings).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ id: 'kb:near-duplicate-analysis-truncated' }),
+ ]),
+ )
+ })
+
+ it('refuses a malformed quality threshold', () => {
+ expect(() =>
+ scoreKnowledgeBaseIndex(index(duplicatePages), { maxNearDuplicatePageRate: -0.1 }),
+ ).toThrow(/maxNearDuplicatePageRate/)
+ })
+})

src/rag-eval.ts

@@ -20,6 +20,12 @@ export type {
  RagRequiredContext,
 } from './rag-eval/contracts'
 export { scoreKnowledgeBaseIndex } from './rag-eval/knowledge-base'
+export type {
+ NearDuplicateDetectionOptions,
+ NearDuplicatePair,
+ NearDuplicateReport,
+} from './rag-eval/near-duplicates'
+export { detectNearDuplicatePages, normalizePageText } from './rag-eval/near-duplicates'
 export {
  normalizeExternalRagScores,
  toDeepEvalTestCases,

src/rag-eval/contracts.ts

@@ -1,6 +1,7 @@
 import type { ComparisonCost, JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign'
 import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface'
 import type { RagGapFinding } from '../rag-improvement-loop'
+import type { NearDuplicateDetectionOptions, NearDuplicateReport } from './near-duplicates'
 
 export type RagEvalProvider =
  | 'agent-knowledge'
@@ -166,6 +167,10 @@ export interface KnowledgeBaseQualityOptions {
  strict?: boolean
  minCitationRate?: number
  maxStaleSourceRate?: number
+ /** Deterministic exact/near-duplicate detector configuration. */
+ nearDuplicates?: NearDuplicateDetectionOptions
+ /** Maximum allowed fraction of eligible pages implicated in a duplicate pair. Defaults to 1. */
+ maxNearDuplicatePageRate?: number
 }
 
 export interface KnowledgeBaseQualityReport {
@@ -179,7 +184,13 @@ export interface KnowledgeBaseQualityReport {
  duplicate_source_hash_rate: number
  lint_error_count: number
  lint_warning_count: number
+ /** Added in the deterministic near-duplicate quality pass. */
+ near_duplicate_page_rate?: number
+ near_duplicate_page_count?: number
+ near_duplicate_pair_count?: number
  }
+ /** Exact report from the deterministic detector; present when produced by this package. */
+ nearDuplicates?: NearDuplicateReport
  findings: readonly RagGapFinding[]
 }
 

src/rag-eval/knowledge-base.ts

@@ -3,6 +3,7 @@ import type { RagGapFinding } from '../rag-improvement-loop'
 import type { KnowledgeIndex } from '../types'
 import { validateKnowledgeIndex } from '../validate'
 import type { KnowledgeBaseQualityOptions, KnowledgeBaseQualityReport } from './contracts'
+import { detectNearDuplicatePages } from './near-duplicates'
 
 export function scoreKnowledgeBaseIndex(
  index: KnowledgeIndex,
@@ -25,6 +26,11 @@ export function scoreKnowledgeBaseIndex(
  (page) => sourceRefs(page.text).length > 0,
  ).length
  const pagesWithSources = index.pages.filter((page) => page.sourceIds.length > 0).length
+ const nearDuplicates = detectNearDuplicatePages(index.pages, options.nearDuplicates)
+ const maxNearDuplicatePageRate = unitInterval(
+ options.maxNearDuplicatePageRate ?? 1,
+ 'maxNearDuplicatePageRate',
+ )
  const metrics = {
  page_count: index.pages.length,
  source_count: index.sources.length,
@@ -35,6 +41,9 @@ export function scoreKnowledgeBaseIndex(
  index.sources.length === 0 ? 0 : duplicateSourceHashCount / index.sources.length,
  lint_error_count: validation.findings.filter((finding) => finding.severity === 'error').length,
  lint_warning_count: lintFindings.filter((finding) => finding.severity === 'warning').length,
+ near_duplicate_page_rate: nearDuplicates.duplicatePageRate,
+ near_duplicate_page_count: nearDuplicates.duplicatePageCount,
+ near_duplicate_pair_count: nearDuplicates.duplicatePairCount,
  }
  const findings: RagGapFinding[] = []
  if (metrics.lint_error_count > 0) {
@@ -64,7 +73,34 @@ export function scoreKnowledgeBaseIndex(
  evidence: { stale_source_rate: metrics.stale_source_rate },
  })
  }
- return { ok: findings.length === 0, metrics, findings }
+ if (nearDuplicates.truncated && maxNearDuplicatePageRate < 1) {
+ findings.push({
+ id: 'kb:near-duplicate-analysis-truncated',
+ kind: 'unknown',
+ severity: 'error',
+ message:
+ 'Near-duplicate analysis reached a configured bound, so the duplicate-page gate cannot be evaluated completely.',
+ evidence: {
+ candidate_pair_count: nearDuplicates.candidatePairCount,
+ compared_pair_count: nearDuplicates.comparedPairCount,
+ duplicate_pair_count: nearDuplicates.duplicatePairCount,
+ },
+ })
+ } else if (metrics.near_duplicate_page_rate > maxNearDuplicatePageRate) {
+ findings.push({
+ id: 'kb:near-duplicate-page-rate',
+ kind: 'unknown',
+ severity: 'error',
+ message: 'Near-duplicate page rate exceeds the configured maximum.',
+ evidence: {
+ near_duplicate_page_rate: metrics.near_duplicate_page_rate,
+ near_duplicate_page_count: metrics.near_duplicate_page_count,
+ near_duplicate_pair_count: metrics.near_duplicate_pair_count,
+ threshold: nearDuplicates.threshold,
+ },
+ })
+ }
+ return { ok: findings.length === 0, metrics, nearDuplicates, findings }
 }
 
 function sourceRefs(text: string): string[] {
@@ -77,3 +113,10 @@ function sourceRefs(text: string): string[] {
  }
  return refs
 }
+
+function unitInterval(value: number, name: string): number {
+ if (!Number.isFinite(value) || value < 0 || value > 1) {
+ throw new Error(`${name} must be a finite number in [0, 1]`)
+ }
+ return value
+}

src/rag-eval/near-duplicates.test.ts

@@ -0,0 +1,204 @@
+import { describe, expect, it } from 'vitest'
+import type { KnowledgeIndex, KnowledgePage } from '../types'
+import { scoreKnowledgeBaseIndex } from './knowledge-base'
+import { detectNearDuplicatePages, normalizePageText } from './near-duplicates'
+
+const repeated =
+ 'A verified research page explains the mechanism, records the experiment, names the evidence, and preserves the result for later agents. '
+
+function page(id: string, title: string, text: string): KnowledgePage {
+ return {
+ id,
+ path: `knowledge/${id}.md`,
+ title,
+ text,
+ frontmatter: { id, title },
+ sourceIds: [],
+ tags: [],
+ outLinks: [],
+ }
+}
+
+function index(pages: KnowledgePage[]): KnowledgeIndex {
+ return {
+ root: '/kb',
+ generatedAt: '2026-08-17T00:00:00.000Z',
+ sources: [],
+ pages,
+ graph: { nodes: [], edges: [] },
+ }
+}
+
+describe('detectNearDuplicatePages', () => {
+ it('reports exact normalized copies with stable page identity', () => {
+ const pages = [
+ page('copy-a', 'Same title', repeated.repeat(2)),
+ page('copy-b', 'Same title', repeated.repeat(2)),
+ ]
+
+ const report = detectNearDuplicatePages(pages)
+
+ expect(report).toMatchObject({
+ pageCount: 2,
+ eligiblePageCount: 2,
+ candidatePairCount: 1,
+ comparedPairCount: 1,
+ duplicatePairCount: 1,
+ duplicatePageCount: 2,
+ duplicatePageRate: 1,
+ truncated: false,
+ })
+ expect(report.pairs).toEqual([
+ expect.objectContaining({
+ leftPageId: 'copy-a',
+ rightPageId: 'copy-b',
+ similarity: 1,
+ exact: true,
+ }),
+ ])
+ })
+
+ it('finds near copies but not unrelated prose', () => {
+ const base = `${repeated.repeat(2)} The decisive measured value is forty two.`
+ const revised = `${repeated.repeat(2)} The decisive measured value is forty three.`
+ const unrelated =
+ 'A kitchen inventory lists pans, knives, towels, plates, and groceries for a weekend dinner. '.repeat(
+ 3,
+ )
+
+ const report = detectNearDuplicatePages(
+ [
+ page('base', 'Experiment', base),
+ page('revision', 'Experiment', revised),
+ page('unrelated', 'Kitchen', unrelated),
+ ],
+ { threshold: 0.7 },
+ )
+
+ expect(report.duplicatePairCount).toBe(1)
+ expect(report.pairs[0]).toMatchObject({
+ leftPageId: 'base',
+ rightPageId: 'revision',
+ exact: false,
+ })
+ expect(report.pairs[0]!.similarity).toBeGreaterThanOrEqual(0.7)
+ })
+
+ it('is deterministic under input reordering', () => {
+ const pages = [
+ page('z', 'Same title', repeated.repeat(2)),
+ page('a', 'Same title', repeated.repeat(2)),
+ page('m', 'Different title', `${repeated.repeat(2)} extra material`),
+ ]
+
+ const forward = detectNearDuplicatePages(pages, { threshold: 0.7 })
+ const reverse = detectNearDuplicatePages([...pages].reverse(), { threshold: 0.7 })
+
+ expect(reverse).toEqual(forward)
+ })
+
+ it('excludes short pages and reports bounded candidate truncation', () => {
+ const short = detectNearDuplicatePages(
+ [page('short-a', 'Short', 'same'), page('short-b', 'Short', 'same')],
+ { minCharacters: 80 },
+ )
+ expect(short).toMatchObject({ eligiblePageCount: 0, duplicatePairCount: 0 })
+
+ const bounded = detectNearDuplicatePages(
+ Array.from({ length: 4 }, (_, index) =>
+ page(`copy-${index}`, 'Same title', repeated.repeat(2)),
+ ),
+ { maxCandidatePairs: 2 },
+ )
+ expect(bounded).toMatchObject({
+ candidatePairCount: 2,
+ comparedPairCount: 2,
+ truncated: true,
+ })
+ })
+
+ it('normalizes Unicode width, case, whitespace, and URL identity', () => {
+ expect(normalizePageText('ＡLPHA https://example.com/a\nBeta')).toBe('alpha <url> beta')
+ })
+
+ it('refuses malformed detector limits', () => {
+ expect(() => detectNearDuplicatePages([], { threshold: 1.1 })).toThrow(/threshold/)
+ expect(() => detectNearDuplicatePages([], { maxCandidatePairs: 0 })).toThrow(
+ /maxCandidatePairs/,
+ )
+ expect(() => detectNearDuplicatePages([], { minCharacters: -1 })).toThrow(/minCharacters/)
+ })
+})
+
+describe('scoreKnowledgeBaseIndex near-duplicate quality', () => {
+ const duplicatePages = [
+ page('copy-a', 'Same title', repeated.repeat(2)),
+ page('copy-b', 'Same title', repeated.repeat(2)),
+ page(
+ 'independent',
+ 'Independent result',
+ 'A separate proof studies a different theorem, with distinct assumptions and a different verification artifact. '.repeat(
+ 3,
+ ),
+ ),
+ ]
+
+ it('always reports deterministic duplicate metrics without failing by default', () => {
+ const report = scoreKnowledgeBaseIndex(index(duplicatePages))
+
+ expect(report.ok).toBe(true)
+ expect(report.metrics).toMatchObject({
+ near_duplicate_page_rate: 0.666667,
+ near_duplicate_page_count: 2,
+ near_duplicate_pair_count: 1,
+ })
+ expect(report.nearDuplicates?.pairs).toHaveLength(1)
+ })
+
+ it('fails a configured duplicate-page gate with measured evidence', () => {
+ const report = scoreKnowledgeBaseIndex(index(duplicatePages), {
+ maxNearDuplicatePageRate: 0.5,
+ })
+
+ expect(report.ok).toBe(false)
+ expect(report.findings).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ id: 'kb:near-duplicate-page-rate',
+ severity: 'error',
+ evidence: expect.objectContaining({
+ near_duplicate_page_rate: 0.666667,
+ near_duplicate_pair_count: 1,
+ }),
+ }),
+ ]),
+ )
+ })
+
+ it('fails closed when a configured gate rests on truncated analysis', () => {
+ const report = scoreKnowledgeBaseIndex(
+ index(
+ Array.from({ length: 4 }, (_, position) =>
+ page(`copy-${position}`, 'Same title', repeated.repeat(2)),
+ ),
+ ),
+ {
+ nearDuplicates: { maxCandidatePairs: 1 },
+ maxNearDuplicatePageRate: 0.9,
+ },
+ )
+
+ expect(report.ok).toBe(false)
+ expect(report.findings).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ id: 'kb:near-duplicate-analysis-truncated' }),
+ ]),
+ )
+ })
+
+ it('refuses a malformed quality threshold', () => {
+ expect(() =>
+ scoreKnowledgeBaseIndex(index(duplicatePages), { maxNearDuplicatePageRate: -0.1 }),
+ ).toThrow(/maxNearDuplicatePageRate/)
+ })
+})

src/rag-eval.ts

@@ -20,6 +20,12 @@ export type {
  RagRequiredContext,
 } from './rag-eval/contracts'
 export { scoreKnowledgeBaseIndex } from './rag-eval/knowledge-base'
+export type {
+ NearDuplicateDetectionOptions,
+ NearDuplicatePair,
+ NearDuplicateReport,
+} from './rag-eval/near-duplicates'
+export { detectNearDuplicatePages, normalizePageText } from './rag-eval/near-duplicates'
 export {
  normalizeExternalRagScores,
  toDeepEvalTestCases,

src/rag-eval/contracts.ts

@@ -1,6 +1,7 @@
 import type { ComparisonCost, JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign'
 import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface'
 import type { RagGapFinding } from '../rag-improvement-loop'
+import type { NearDuplicateDetectionOptions, NearDuplicateReport } from './near-duplicates'
 
 export type RagEvalProvider =
  | 'agent-knowledge'
@@ -166,6 +167,10 @@ export interface KnowledgeBaseQualityOptions {
  strict?: boolean
  minCitationRate?: number
  maxStaleSourceRate?: number
+ /** Deterministic exact/near-duplicate detector configuration. */
+ nearDuplicates?: NearDuplicateDetectionOptions
+ /** Maximum allowed fraction of eligible pages implicated in a duplicate pair. Defaults to 1. */
+ maxNearDuplicatePageRate?: number
 }
 
 export interface KnowledgeBaseQualityReport {
@@ -179,7 +184,13 @@ export interface KnowledgeBaseQualityReport {
  duplicate_source_hash_rate: number
  lint_error_count: number
  lint_warning_count: number
+ /** Added in the deterministic near-duplicate quality pass. */
+ near_duplicate_page_rate?: number
+ near_duplicate_page_count?: number
+ near_duplicate_pair_count?: number
  }
+ /** Exact report from the deterministic detector; present when produced by this package. */
+ nearDuplicates?: NearDuplicateReport
  findings: readonly RagGapFinding[]
 }
 

src/rag-eval/knowledge-base.ts

@@ -3,6 +3,7 @@ import type { RagGapFinding } from '../rag-improvement-loop'
 import type { KnowledgeIndex } from '../types'
 import { validateKnowledgeIndex } from '../validate'
 import type { KnowledgeBaseQualityOptions, KnowledgeBaseQualityReport } from './contracts'
+import { detectNearDuplicatePages } from './near-duplicates'
 
 export function scoreKnowledgeBaseIndex(
  index: KnowledgeIndex,
@@ -25,6 +26,11 @@ export function scoreKnowledgeBaseIndex(
  (page) => sourceRefs(page.text).length > 0,
  ).length
  const pagesWithSources = index.pages.filter((page) => page.sourceIds.length > 0).length
+ const nearDuplicates = detectNearDuplicatePages(index.pages, options.nearDuplicates)
+ const maxNearDuplicatePageRate = unitInterval(
+ options.maxNearDuplicatePageRate ?? 1,
+ 'maxNearDuplicatePageRate',
+ )
  const metrics = {
  page_count: index.pages.length,
  source_count: index.sources.length,
@@ -35,6 +41,9 @@ export function scoreKnowledgeBaseIndex(
  index.sources.length === 0 ? 0 : duplicateSourceHashCount / index.sources.length,
  lint_error_count: validation.findings.filter((finding) => finding.severity === 'error').length,
  lint_warning_count: lintFindings.filter((finding) => finding.severity === 'warning').length,
+ near_duplicate_page_rate: nearDuplicates.duplicatePageRate,
+ near_duplicate_page_count: nearDuplicates.duplicatePageCount,
+ near_duplicate_pair_count: nearDuplicates.duplicatePairCount,
  }
  const findings: RagGapFinding[] = []
  if (metrics.lint_error_count > 0) {
@@ -64,7 +73,34 @@ export function scoreKnowledgeBaseIndex(
  evidence: { stale_source_rate: metrics.stale_source_rate },
  })
  }
- return { ok: findings.length === 0, metrics, findings }
+ if (nearDuplicates.truncated && maxNearDuplicatePageRate < 1) {
+ findings.push({
+ id: 'kb:near-duplicate-analysis-truncated',
+ kind: 'unknown',
+ severity: 'error',
+ message:
+ 'Near-duplicate analysis reached a configured bound, so the duplicate-page gate cannot be evaluated completely.',
+ evidence: {
+ candidate_pair_count: nearDuplicates.candidatePairCount,
+ compared_pair_count: nearDuplicates.comparedPairCount,
+ duplicate_pair_count: nearDuplicates.duplicatePairCount,
+ },
+ })
+ } else if (metrics.near_duplicate_page_rate > maxNearDuplicatePageRate) {
+ findings.push({
+ id: 'kb:near-duplicate-page-rate',
+ kind: 'unknown',
+ severity: 'error',
+ message: 'Near-duplicate page rate exceeds the configured maximum.',
+ evidence: {
+ near_duplicate_page_rate: metrics.near_duplicate_page_rate,
+ near_duplicate_page_count: metrics.near_duplicate_page_count,
+ near_duplicate_pair_count: metrics.near_duplicate_pair_count,
+ threshold: nearDuplicates.threshold,
+ },
+ })
+ }
+ return { ok: findings.length === 0, metrics, nearDuplicates, findings }
 }
 
 function sourceRefs(text: string): string[] {
@@ -77,3 +113,10 @@ function sourceRefs(text: string): string[] {
  }
  return refs
 }
+
+function unitInterval(value: number, name: string): number {
+ if (!Number.isFinite(value) || value < 0 || value > 1) {
+ throw new Error(`${name} must be a finite number in [0, 1]`)
+ }
+ return value
+}

src/rag-eval/near-duplicates.test.ts

@@ -0,0 +1,204 @@
+import { describe, expect, it } from 'vitest'
+import type { KnowledgeIndex, KnowledgePage } from '../types'
+import { scoreKnowledgeBaseIndex } from './knowledge-base'
+import { detectNearDuplicatePages, normalizePageText } from './near-duplicates'
+
+const repeated =
+ 'A verified research page explains the mechanism, records the experiment, names the evidence, and preserves the result for later agents. '
+
+function page(id: string, title: string, text: string): KnowledgePage {
+ return {
+ id,
+ path: `knowledge/${id}.md`,
+ title,
+ text,
+ frontmatter: { id, title },
+ sourceIds: [],
+ tags: [],
+ outLinks: [],
+ }
+}
+
+function index(pages: KnowledgePage[]): KnowledgeIndex {
+ return {
+ root: '/kb',
+ generatedAt: '2026-08-17T00:00:00.000Z',
+ sources: [],
+ pages,
+ graph: { nodes: [], edges: [] },
+ }
+}
+
+describe('detectNearDuplicatePages', () => {
+ it('reports exact normalized copies with stable page identity', () => {
+ const pages = [
+ page('copy-a', 'Same title', repeated.repeat(2)),
+ page('copy-b', 'Same title', repeated.repeat(2)),
+ ]
+
+ const report = detectNearDuplicatePages(pages)
+
+ expect(report).toMatchObject({
+ pageCount: 2,
+ eligiblePageCount: 2,
+ candidatePairCount: 1,
+ comparedPairCount: 1,
+ duplicatePairCount: 1,
+ duplicatePageCount: 2,
+ duplicatePageRate: 1,
+ truncated: false,
+ })
+ expect(report.pairs).toEqual([
+ expect.objectContaining({
+ leftPageId: 'copy-a',
+ rightPageId: 'copy-b',
+ similarity: 1,
+ exact: true,
+ }),
+ ])
+ })
+
+ it('finds near copies but not unrelated prose', () => {
+ const base = `${repeated.repeat(2)} The decisive measured value is forty two.`
+ const revised = `${repeated.repeat(2)} The decisive measured value is forty three.`
+ const unrelated =
+ 'A kitchen inventory lists pans, knives, towels, plates, and groceries for a weekend dinner. '.repeat(
+ 3,
+ )
+
+ const report = detectNearDuplicatePages(
+ [
+ page('base', 'Experiment', base),
+ page('revision', 'Experiment', revised),
+ page('unrelated', 'Kitchen', unrelated),
+ ],
+ { threshold: 0.7 },
+ )
+
+ expect(report.duplicatePairCount).toBe(1)
+ expect(report.pairs[0]).toMatchObject({
+ leftPageId: 'base',
+ rightPageId: 'revision',
+ exact: false,
+ })
+ expect(report.pairs[0]!.similarity).toBeGreaterThanOrEqual(0.7)
+ })
+
+ it('is deterministic under input reordering', () => {
+ const pages = [
+ page('z', 'Same title', repeated.repeat(2)),
+ page('a', 'Same title', repeated.repeat(2)),
+ page('m', 'Different title', `${repeated.repeat(2)} extra material`),
+ ]
+
+ const forward = detectNearDuplicatePages(pages, { threshold: 0.7 })
+ const reverse = detectNearDuplicatePages([...pages].reverse(), { threshold: 0.7 })
+
+ expect(reverse).toEqual(forward)
+ })
+
+ it('excludes short pages and reports bounded candidate truncation', () => {
+ const short = detectNearDuplicatePages(
+ [page('short-a', 'Short', 'same'), page('short-b', 'Short', 'same')],
+ { minCharacters: 80 },
+ )
+ expect(short).toMatchObject({ eligiblePageCount: 0, duplicatePairCount: 0 })
+
+ const bounded = detectNearDuplicatePages(
+ Array.from({ length: 4 }, (_, index) =>
+ page(`copy-${index}`, 'Same title', repeated.repeat(2)),
+ ),
+ { maxCandidatePairs: 2 },
+ )
+ expect(bounded).toMatchObject({
+ candidatePairCount: 2,
+ comparedPairCount: 2,
+ truncated: true,
+ })
+ })
+
+ it('normalizes Unicode width, case, whitespace, and URL identity', () => {
+ expect(normalizePageText('ＡLPHA https://example.com/a\nBeta')).toBe('alpha <url> beta')
+ })
+
+ it('refuses malformed detector limits', () => {
+ expect(() => detectNearDuplicatePages([], { threshold: 1.1 })).toThrow(/threshold/)
+ expect(() => detectNearDuplicatePages([], { maxCandidatePairs: 0 })).toThrow(
+ /maxCandidatePairs/,
+ )
+ expect(() => detectNearDuplicatePages([], { minCharacters: -1 })).toThrow(/minCharacters/)
+ })
+})
+
+describe('scoreKnowledgeBaseIndex near-duplicate quality', () => {
+ const duplicatePages = [
+ page('copy-a', 'Same title', repeated.repeat(2)),
+ page('copy-b', 'Same title', repeated.repeat(2)),
+ page(
+ 'independent',
+ 'Independent result',
+ 'A separate proof studies a different theorem, with distinct assumptions and a different verification artifact. '.repeat(
+ 3,
+ ),
+ ),
+ ]
+
+ it('always reports deterministic duplicate metrics without failing by default', () => {
+ const report = scoreKnowledgeBaseIndex(index(duplicatePages))
+
+ expect(report.ok).toBe(true)
+ expect(report.metrics).toMatchObject({
+ near_duplicate_page_rate: 0.666667,
+ near_duplicate_page_count: 2,
+ near_duplicate_pair_count: 1,
+ })
+ expect(report.nearDuplicates?.pairs).toHaveLength(1)
+ })
+
+ it('fails a configured duplicate-page gate with measured evidence', () => {
+ const report = scoreKnowledgeBaseIndex(index(duplicatePages), {
+ maxNearDuplicatePageRate: 0.5,
+ })
+
+ expect(report.ok).toBe(false)
+ expect(report.findings).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ id: 'kb:near-duplicate-page-rate',
+ severity: 'error',
+ evidence: expect.objectContaining({
+ near_duplicate_page_rate: 0.666667,
+ near_duplicate_pair_count: 1,
+ }),
+ }),
+ ]),
+ )
+ })
+
+ it('fails closed when a configured gate rests on truncated analysis', () => {
+ const report = scoreKnowledgeBaseIndex(
+ index(
+ Array.from({ length: 4 }, (_, position) =>
+ page(`copy-${position}`, 'Same title', repeated.repeat(2)),
+ ),
+ ),
+ {
+ nearDuplicates: { maxCandidatePairs: 1 },
+ maxNearDuplicatePageRate: 0.9,
+ },
+ )
+
+ expect(report.ok).toBe(false)
+ expect(report.findings).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ id: 'kb:near-duplicate-analysis-truncated' }),
+ ]),
+ )
+ })
+
+ it('refuses a malformed quality threshold', () => {
+ expect(() =>
+ scoreKnowledgeBaseIndex(index(duplicatePages), { maxNearDuplicatePageRate: -0.1 }),
+ ).toThrow(/maxNearDuplicatePageRate/)
+ })
+})