Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 12 additions & 8 deletions scripts/backfill-smart-index.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -533,6 +533,10 @@ async function main() {
console.log("Metadata-only mode: repairing chunk metadata, document-level fields, labels, and quality rows.");

for (const document of documents ?? []) {
const indexingDocument = {
...document,
metadata: metadataRecord(document.metadata),
};
const [chunks, images] = await Promise.all([
loadRows<Record<string, unknown>>(
reader,
Expand DownExpand Up@@ -562,13 +566,13 @@ async function main() {
const summary = await loadDocumentSummary({ supabase, documentId: document.id });
const generationId = await repairChunkIndexingMetadata({
supabase,
document,
document: indexingDocument,
chunks: chunks as Record<string, unknown>[],
ragDeepMemoryVersion,
});
const embeddingFields = await upsertDocumentLevelEmbeddingFields({
supabase,
document,
document: indexingDocument,
chunks: chunks as Record<string, unknown>[],
summary,
embedTexts,
Expand All@@ -578,7 +582,7 @@ async function main() {
if (args.metadataOnly) {
fallbackLabels = await ensureGeneratedFallbackLabel({
supabase,
document,
document: indexingDocument,
ragEnrichmentVersion,
});
const [sectionCount, memoryCardCount] = await Promise.all([
Expand All@@ -587,7 +591,7 @@ async function main() {
]);
const quality = await upsertIndexQuality({
supabase,
document,
document: indexingDocument,
chunks: chunks as Record<string, unknown>[],
images: images as Record<string, unknown>[],
sectionCount,
Expand All@@ -602,25 +606,25 @@ async function main() {

const enrichment = await upsertDocumentEnrichment({
supabase,
document: document as Parameters<typeof upsertDocumentEnrichment>[0]["document"],
document: indexingDocument,
chunks: chunks as never,
images: images as never,
});
fallbackLabels = await ensureGeneratedFallbackLabel({
supabase,
document,
document: indexingDocument,
ragEnrichmentVersion,
});
const memory = await upsertDocumentDeepMemory({
supabase,
document: document as Parameters<typeof upsertDocumentDeepMemory>[0]["document"],
document: indexingDocument,
chunks: chunks as never,
images: images as never,
summary: enrichment.summary.summary,
});
const quality = await upsertIndexQuality({
supabase,
document,
document: indexingDocument,
chunks: chunks as Record<string, unknown>[],
images: images as Record<string, unknown>[],
sectionCount: memory.sections.length,
Expand Down
Loading