From dd79bfdcd2189db66f36c191bf9552e46868f30f Mon Sep 17 00:00:00 2001 From: ntorbinskiy Date: Thu, 3 Sep 2026 15:23:44 +0300 Subject: [PATCH 1/2] feat(reins): port knowledge base UX and index-state work from the Mazda fork (CLEAN-60) --- .../agent/components/agent/knowledge/Tab.vue | 4 +- .../knowledge/SourceStatusBadge.vue | 34 + .../components/knowledge/create/Provider.vue | 2 +- .../components/knowledge/edit/Provider.vue | 49 -- .../knowledge/graph/LabelPicker.vue | 144 ++++ .../components/knowledge/graph/Provider.vue | 78 +- .../knowledge/indexStatus/Badge.vue | 25 +- .../components/knowledge/list/Provider.vue | 2 +- .../knowledge/overview/Provider.vue | 337 ++++++++ .../components/knowledge/setup/Wizard.vue | 4 +- .../components/knowledge/sources/AddForm.vue | 178 ----- .../knowledge/sources/AddFromArchiveForm.vue | 97 --- .../knowledge/sources/AddFromSitemapForm.vue | 108 --- .../components/knowledge/sources/AddSheet.vue | 358 +++++++++ .../knowledge/sources/ImportProgress.vue | 98 +++ .../knowledge/sources/PreviewSheet.vue | 166 ++++ .../components/knowledge/sources/Provider.vue | 473 ++++++++++-- admin/slices/reins/data/knowledge.gateway.ts | 107 ++- admin/slices/reins/data/knowledge.mapper.ts | 111 ++- admin/slices/reins/domain/format.ts | 49 ++ admin/slices/reins/domain/index.ts | 1 + .../slices/reins/domain/knowledge.gateway.ts | 18 +- .../slices/reins/domain/knowledge.service.ts | 29 +- admin/slices/reins/domain/knowledge.types.ts | 78 ++ admin/slices/reins/pages/knowledges/[id].vue | 230 ++++-- .../reins/pages/knowledges/[id]/edit.vue | 3 - .../reins/pages/knowledges/[id]/index.vue | 13 +- admin/slices/reins/stores/knowledge.ts | 103 ++- .../api/data/repositories/api/schemas.gen.ts | 198 ++++- .../api/data/repositories/api/sdk.gen.ts | 62 +- .../api/data/repositories/api/types.gen.ts | 154 +++- .../components/ui/sidebar/SidebarInset.vue | 9 +- api/package.json | 2 + .../migration.sql | 5 + api/src/slices/aws/s3/s3.repository.ts | 35 + api/src/slices/aws/s3/s3.types.ts | 12 + api/src/slices/reins/README.md | 143 ++++ .../reins/knowledge/data/knowledge.gateway.ts | 37 +- .../reins/knowledge/data/knowledge.mapper.ts | 4 +- .../domain/indexReconcile.service.ts | 149 ++++ .../knowledge/domain/indexReconcile.spec.ts | 215 ++++++ .../knowledge/domain/knowledge.gateway.ts | 18 +- .../knowledge/domain/knowledge.service.ts | 201 +++-- .../reins/knowledge/domain/knowledge.types.ts | 20 +- .../reins/knowledge/dtos/knowledge.dto.ts | 13 + .../reins/knowledge/knowledge.controller.ts | 15 +- .../knowledge/knowledge.isolation.spec.ts | 19 + .../reins/knowledge/knowledge.module.ts | 2 + .../reins/knowledge/knowledge.tool.spec.ts | 78 +- .../slices/reins/knowledge/knowledge.tool.ts | 23 +- .../lightrag/data/lightragHttp.client.ts | 195 +++-- .../slices/reins/lightrag/data/workspace.ts | 17 + .../reins/lightrag/domain/lightrag.client.ts | 11 + .../reins/lightrag/domain/lightrag.types.ts | 76 +- .../domain/migration.service.spec.ts | 57 +- .../migration/domain/migration.service.ts | 4 +- .../reins/source/data/source.gateway.spec.ts | 482 ++++++++++++ .../reins/source/data/source.gateway.ts | 724 +++++++++++++++++- .../reins/source/data/source.mapper.spec.ts | 32 + .../slices/reins/source/data/source.mapper.ts | 21 + .../source/data/sourceArchive.writer.spec.ts | 153 ++++ .../reins/source/data/sourceArchive.writer.ts | 131 ++++ .../reins/source/domain/archiveLimit.ts | 15 + .../source/domain/importJob.registry.spec.ts | 93 +++ .../reins/source/domain/importJob.registry.ts | 92 +++ .../reins/source/domain/indexBudget.spec.ts | 97 +++ .../slices/reins/source/domain/indexBudget.ts | 101 +++ .../reins/source/domain/source.gateway.ts | 38 + .../reins/source/domain/source.service.ts | 140 +++- .../reins/source/domain/source.types.ts | 109 +++ .../reins/source/dtos/addFromArchive.dto.ts | 11 +- .../reins/source/dtos/exportSources.dto.ts | 39 + .../reins/source/dtos/filterSources.dto.ts | 41 + .../slices/reins/source/dtos/importJob.dto.ts | 25 + api/src/slices/reins/source/dtos/index.ts | 5 + .../slices/reins/source/dtos/source.dto.ts | 23 +- .../source/dtos/sourceContent.query.dto.ts | 21 + .../reins/source/dtos/sourcePage.dto.ts | 11 + .../slices/reins/source/source.controller.ts | 200 ++++- api/src/slices/reins/source/source.module.ts | 2 + api/src/slices/reins/source/source.prisma | 1 + bun.lock | 30 +- 82 files changed, 6456 insertions(+), 854 deletions(-) create mode 100644 admin/slices/reins/components/knowledge/SourceStatusBadge.vue delete mode 100644 admin/slices/reins/components/knowledge/edit/Provider.vue create mode 100644 admin/slices/reins/components/knowledge/graph/LabelPicker.vue create mode 100644 admin/slices/reins/components/knowledge/overview/Provider.vue delete mode 100644 admin/slices/reins/components/knowledge/sources/AddForm.vue delete mode 100644 admin/slices/reins/components/knowledge/sources/AddFromArchiveForm.vue delete mode 100644 admin/slices/reins/components/knowledge/sources/AddFromSitemapForm.vue create mode 100644 admin/slices/reins/components/knowledge/sources/AddSheet.vue create mode 100644 admin/slices/reins/components/knowledge/sources/ImportProgress.vue create mode 100644 admin/slices/reins/components/knowledge/sources/PreviewSheet.vue create mode 100644 admin/slices/reins/domain/format.ts delete mode 100644 admin/slices/reins/pages/knowledges/[id]/edit.vue create mode 100644 api/prisma/migrations/20260903130000_source_list_index/migration.sql create mode 100644 api/src/slices/reins/README.md create mode 100644 api/src/slices/reins/knowledge/domain/indexReconcile.service.ts create mode 100644 api/src/slices/reins/knowledge/domain/indexReconcile.spec.ts create mode 100644 api/src/slices/reins/source/data/source.gateway.spec.ts create mode 100644 api/src/slices/reins/source/data/source.mapper.spec.ts create mode 100644 api/src/slices/reins/source/data/sourceArchive.writer.spec.ts create mode 100644 api/src/slices/reins/source/data/sourceArchive.writer.ts create mode 100644 api/src/slices/reins/source/domain/archiveLimit.ts create mode 100644 api/src/slices/reins/source/domain/importJob.registry.spec.ts create mode 100644 api/src/slices/reins/source/domain/importJob.registry.ts create mode 100644 api/src/slices/reins/source/domain/indexBudget.spec.ts create mode 100644 api/src/slices/reins/source/domain/indexBudget.ts create mode 100644 api/src/slices/reins/source/dtos/exportSources.dto.ts create mode 100644 api/src/slices/reins/source/dtos/filterSources.dto.ts create mode 100644 api/src/slices/reins/source/dtos/importJob.dto.ts create mode 100644 api/src/slices/reins/source/dtos/sourceContent.query.dto.ts create mode 100644 api/src/slices/reins/source/dtos/sourcePage.dto.ts diff --git a/admin/slices/agent/agent/components/agent/knowledge/Tab.vue b/admin/slices/agent/agent/components/agent/knowledge/Tab.vue index ed37bb4e..51170aa8 100644 --- a/admin/slices/agent/agent/components/agent/knowledge/Tab.vue +++ b/admin/slices/agent/agent/components/agent/knowledge/Tab.vue @@ -152,7 +152,7 @@ function unbindMissing(id: string): void { v-for="k in resolved" :key="k.id" class="cursor-pointer" - @click="navigateTo(`/knowledges/${k.id}/edit`)" + @click="navigateTo(`/knowledges/${k.id}`)" > {{ k.name }} @@ -167,7 +167,7 @@ function unbindMissing(id: string): void {
diff --git a/admin/slices/reins/components/knowledge/SourceStatusBadge.vue b/admin/slices/reins/components/knowledge/SourceStatusBadge.vue new file mode 100644 index 00000000..5cb695ca --- /dev/null +++ b/admin/slices/reins/components/knowledge/SourceStatusBadge.vue @@ -0,0 +1,34 @@ + + + diff --git a/admin/slices/reins/components/knowledge/create/Provider.vue b/admin/slices/reins/components/knowledge/create/Provider.vue index aac841a4..910264a2 100644 --- a/admin/slices/reins/components/knowledge/create/Provider.vue +++ b/admin/slices/reins/components/knowledge/create/Provider.vue @@ -14,7 +14,7 @@ async function onSubmit(values: IKnowledgeFormValues) { description: values.description, }); if (created) { - await navigateTo(`/knowledges/${created.id}/edit`); + await navigateTo(`/knowledges/${created.id}`); } else { await navigateTo('/knowledges'); } diff --git a/admin/slices/reins/components/knowledge/edit/Provider.vue b/admin/slices/reins/components/knowledge/edit/Provider.vue deleted file mode 100644 index 8a992bf7..00000000 --- a/admin/slices/reins/components/knowledge/edit/Provider.vue +++ /dev/null @@ -1,49 +0,0 @@ - - - diff --git a/admin/slices/reins/components/knowledge/graph/LabelPicker.vue b/admin/slices/reins/components/knowledge/graph/LabelPicker.vue new file mode 100644 index 00000000..6362d4a4 --- /dev/null +++ b/admin/slices/reins/components/knowledge/graph/LabelPicker.vue @@ -0,0 +1,144 @@ + + + diff --git a/admin/slices/reins/components/knowledge/graph/Provider.vue b/admin/slices/reins/components/knowledge/graph/Provider.vue index 9dbf5f28..6300fce7 100644 --- a/admin/slices/reins/components/knowledge/graph/Provider.vue +++ b/admin/slices/reins/components/knowledge/graph/Provider.vue @@ -3,6 +3,7 @@ import Sigma from 'sigma'; import type { SigmaNodeEventPayload, SigmaEdgeEventPayload } from 'sigma/types'; import Graph from 'graphology'; import forceAtlas2 from 'graphology-layout-forceatlas2'; +import FA2Layout from 'graphology-layout-forceatlas2/worker'; import { ComboboxAnchor, ComboboxContent, @@ -36,8 +37,17 @@ const selectedNode = ref(null); const selectedEdge = ref(null); const nodes = ref([]); +// How long the layout is allowed to keep running after a graph is drawn. +// It runs in a worker, so this only bounds how long positions keep moving. +const LAYOUT_RUN_MS = 4000; +// Above this many nodes the O(n^2) repulsion pass gets a Barnes-Hut +// approximation; below it the exact pass is cheaper than the quadtree. +const BARNES_HUT_FROM_NODES = 50; + let renderer: Sigma | null = null; let graph: Graph | null = null; +let layout: FA2Layout | null = null; +let layoutTimer: ReturnType | null = null; const nodeIndex = new Map(); const edgeIndex = new Map(); @@ -105,6 +115,8 @@ function renderGraph(data: GraphDto): void { ensureRenderer(); if (!graph) return; + // The worker mutates this graph; it has to go before the contents do. + stopLayout(); graph.clear(); nodeIndex.clear(); edgeIndex.clear(); @@ -114,15 +126,20 @@ function renderGraph(data: GraphDto): void { const nodeSize = data.nodes.length > 50 ? 6 : data.nodes.length > 20 ? 8 : 10; - for (const node of data.nodes) { - graph.addNode(node.id, { - x: Math.random(), - y: Math.random(), + // Seeded on a circle rather than at random: the layout starts from an + // untangled state, so it converges in far fewer passes and the first frame + // is already readable instead of a knot. + const g = graph; + data.nodes.forEach((node, i) => { + const angle = (2 * Math.PI * i) / data.nodes.length; + g.addNode(node.id, { + x: Math.cos(angle), + y: Math.sin(angle), size: nodeSize, label: node.label, color: colorForType(node.entityType), }); - } + }); for (const edge of data.edges) { if (!graph.hasNode(edge.source) || !graph.hasNode(edge.target)) continue; @@ -134,21 +151,53 @@ function renderGraph(data: GraphDto): void { } if (graph.order > 0) { - const settings = forceAtlas2.inferSettings(graph); - forceAtlas2.assign(graph, { - iterations: 200, - settings: { - ...settings, - gravity: 1, - scalingRatio: 10, - }, - }); normalizeAndFit(); + startLayout(); } renderer?.refresh(); } +/** + * ForceAtlas2 in a web worker. The synchronous `forceAtlas2.assign` this + * replaces ran 200 passes on the main thread, which froze the whole tab for + * ~15s on a dense graph (repulsion is quadratic in nodes, attraction linear in + * edges). The worker keeps the UI interactive while positions settle, and the + * run is cut off after LAYOUT_RUN_MS so it cannot spin forever. + */ +function startLayout(): void { + stopLayout(); + if (!graph || graph.order === 0) return; + + const settings = forceAtlas2.inferSettings(graph); + layout = new FA2Layout(graph, { + settings: { + ...settings, + gravity: 1, + scalingRatio: 10, + barnesHutOptimize: graph.order > BARNES_HUT_FROM_NODES, + }, + }); + layout.start(); + + layoutTimer = setTimeout(() => { + stopLayout(); + normalizeAndFit(); + renderer?.refresh(); + }, LAYOUT_RUN_MS); +} + +function stopLayout(): void { + if (layoutTimer !== null) { + clearTimeout(layoutTimer); + layoutTimer = null; + } + if (layout) { + layout.kill(); + layout = null; + } +} + function normalizeAndFit(): void { if (!graph || !renderer) return; const g = graph; @@ -253,6 +302,7 @@ onMounted(async () => { }); onBeforeUnmount(() => { + stopLayout(); renderer?.kill(); renderer = null; graph = null; diff --git a/admin/slices/reins/components/knowledge/indexStatus/Badge.vue b/admin/slices/reins/components/knowledge/indexStatus/Badge.vue index 8c228a71..3f6ad314 100644 --- a/admin/slices/reins/components/knowledge/indexStatus/Badge.vue +++ b/admin/slices/reins/components/knowledge/indexStatus/Badge.vue @@ -2,9 +2,22 @@ import type { IndexStatus } from '#reins/stores/knowledge'; import { Badge as UiBadge } from '#theme/components/ui/badge'; -const props = defineProps<{ status: IndexStatus }>(); +// `processing` is not a fifth status, it is what the stored one cannot say. +// A run stops waiting long before LightRAG finishes a large document, so the +// row reads `ready` while a third of the content is still being chunked. The +// green "Ready" badge on a base that is only two thirds searchable is what +// sent people off believing an import had landed. Kept out of `IndexStatus` +// on purpose: that value also gates the Index button and the 3s poll, and a +// base waiting on the pipeline must stay indexable - new sources arrive while +// old ones are still cooking. +const props = defineProps<{ status: IndexStatus; processing?: boolean }>(); + +const stillWorking = computed( + () => props.processing === true && props.status === 'ready', +); const label = computed(() => { + if (stillWorking.value) return 'Ready · still indexing'; switch (props.status) { case 'idle': return 'Idle'; @@ -25,6 +38,8 @@ const label = computed(() => { const variant = computed<'default' | 'secondary' | 'destructive' | 'outline'>( () => { + // Deliberately not the green `default`: searchable, but not all of it. + if (stillWorking.value) return 'secondary'; switch (props.status) { case 'ready': return 'default'; @@ -38,8 +53,14 @@ const variant = computed<'default' | 'secondary' | 'destructive' | 'outline'>( } }, ); + +const title = computed(() => + stillWorking.value + ? 'Some documents are still moving through LightRAG. They are confirmed automatically as it finishes.' + : undefined, +); diff --git a/admin/slices/reins/components/knowledge/list/Provider.vue b/admin/slices/reins/components/knowledge/list/Provider.vue index 60ffbcda..01f5be22 100644 --- a/admin/slices/reins/components/knowledge/list/Provider.vue +++ b/admin/slices/reins/components/knowledge/list/Provider.vue @@ -125,7 +125,7 @@ async function onRemove(item: IKnowledge) {