From dc2e3275d2be5f62796fa43e8877e685a60498b9 Mon Sep 17 00:00:00 2001 From: dfed25 Date: Sat, 25 Apr 2026 10:35:07 -0700 Subject: [PATCH 1/6] fix: restore manager source flow and add safe hire recovery Allow demo-mode hire access for manager source workflows, add double-confirm archive behavior, and provide archived-hire restore actions to prevent accidental irreversible removals. Made-with: Cursor --- .env.example | 4 ++- src/app/manager/tasks/page.tsx | 66 ++++++++++++++++++++++++++++------ src/lib/apiAuth.ts | 11 +++++- 3 files changed, 68 insertions(+), 13 deletions(-) diff --git a/.env.example b/.env.example index 5ba0f67..6f57435 100644 --- a/.env.example +++ b/.env.example @@ -11,4 +11,6 @@ SLACK_BOT_TOKEN= SLACK_ONBOARDING_CHANNEL_ID= # Optional: used by manager control plane in hybrid mode. -# Managers can always add links in the UI; provider credentials unlock richer ingestion. \ No newline at end of file +# Managers can always add links in the UI; provider credentials unlock richer ingestion. +# Set to "false" to require real auth on hire-scoped manager/sync/chat endpoints. +RUNBOOK_ALLOW_UNAUTH_HIRE_ACCESS=true \ No newline at end of file diff --git a/src/app/manager/tasks/page.tsx b/src/app/manager/tasks/page.tsx index 02221cd..a8b9aea 100644 --- a/src/app/manager/tasks/page.tsx +++ b/src/app/manager/tasks/page.tsx @@ -25,6 +25,9 @@ export default function ManagerTasksPage() { const [form, setForm] = useState({ title: "", description: "", assigneeId: "", estimatedTime: "", sourceTitle: "" }); const [duplicateTargets, setDuplicateTargets] = useState>({}); + const activeHires = hires.filter((hire) => hire.active); + const archivedHires = hires.filter((hire) => !hire.active); + useEffect(() => { const timer = window.setTimeout(async () => { try { @@ -33,9 +36,9 @@ export default function ManagerTasksPage() { const hiresData = await hiresRes.json(); if (tasksRes.ok) setTasks(tasksData); if (hiresRes.ok) { - const activeHires = (hiresData.hires || []).filter((hire: Hire) => hire.active); - setHires(activeHires); - const defaultHire = activeHires[0]?.id || ""; + const allHires = (hiresData.hires || []) as Hire[]; + const defaultHire = allHires.find((hire) => hire.active)?.id || ""; + setHires(allHires); setSelectedHireId(defaultHire); setForm((prev) => ({ ...prev, assigneeId: defaultHire })); } @@ -137,21 +140,45 @@ export default function ManagerTasksPage() { const data = await res.json(); if (!res.ok) return setMessage(data.error || "Failed to create hire."); const created = data.hire as Hire; - setHires((prev) => [...prev, created]); + setHires((prev) => [...prev, { ...created, active: true }]); setSelectedHireId(created.id); setForm((prev) => ({ ...prev, assigneeId: created.id })); setHireForm({ name: "", role: "", email: "" }); } async function deleteHire(hireId: string) { - const res = await fetch(`/api/manager/hires/${hireId}`, { method: "DELETE" }); + const firstConfirm = window.confirm("Remove this hire from active onboarding?"); + if (!firstConfirm) return; + const secondConfirm = window.confirm( + "Please confirm again: this hire will be archived and removed from active workflows." + ); + if (!secondConfirm) return; + const res = await fetch(`/api/manager/hires/${hireId}`, { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ active: false }) + }); const data = await res.json(); if (!res.ok) return setMessage(data.error || "Failed to remove hire."); - setHires((prev) => prev.filter((hire) => hire.id !== hireId)); - setSelectedHireId(""); + setHires((prev) => prev.map((hire) => (hire.id === hireId ? { ...hire, active: false } : hire))); + setSelectedHireId((current) => (current === hireId ? (activeHires.find((hire) => hire.id !== hireId)?.id || "") : current)); setSources([]); setForm((prev) => (prev.assigneeId === hireId ? { ...prev, assigneeId: "" } : prev)); - setMessage("Hire removed."); + setMessage("Hire archived. You can restore them below."); + } + + async function restoreHire(hireId: string) { + const res = await fetch(`/api/manager/hires/${hireId}`, { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ active: true }) + }); + const data = await res.json(); + if (!res.ok) return setMessage(data.error || "Failed to restore hire."); + setHires((prev) => prev.map((hire) => (hire.id === hireId ? { ...hire, active: true } : hire))); + setSelectedHireId(hireId); + setForm((prev) => ({ ...prev, assigneeId: hireId })); + setMessage("Hire restored."); } async function addSource(event: FormEvent) { @@ -208,7 +235,7 @@ export default function ManagerTasksPage() { Add hire
- {hires.map((hire) => ( + {activeHires.map((hire) => ( {selectedHireId ? void deleteHire(selectedHireId)} className="mt-3 px-3 py-1 text-xs">Remove selected hire : null} + {archivedHires.length > 0 ? ( +
+

Archived hires

+
+ {archivedHires.map((hire) => ( + void restoreHire(hire.id)} + > + Restore {hire.name} + + ))} +
+
+ ) : null} setForm((p) => ({ ...p, description: e.target.value }))} required />
setForm((p) => ({ ...p, estimatedTime: e.target.value }))} /> setForm((p) => ({ ...p, sourceTitle: e.target.value }))} /> @@ -279,7 +323,7 @@ export default function ManagerTasksPage() {

Duplicate to hires:

- {hires.filter((hire) => hire.id !== task.assigneeId).map((hire) => { + {activeHires.filter((hire) => hire.id !== task.assigneeId).map((hire) => { const active = (duplicateTargets[task.id] || []).includes(hire.id); return ( { try { const cookieStore = await cookies(); @@ -29,7 +32,13 @@ async function isUserAuthorizedForHire(_userId: string, hireId: string): Promise export async function requireHireAccess(hireId: string): Promise { const userId = await getCurrentUserId(); if (!userId) { - return { ok: false, status: 401 }; + if (!allowUnauthedDemoAccess) { + return { ok: false, status: 401 }; + } + const hires = await getHires(); + const exists = hires.some((hire) => hire.id === hireId); + if (!exists) return { ok: false, status: 403 }; + return { ok: true, status: 200, userId: "demo-user" }; } const authorized = await isUserAuthorizedForHire(userId, hireId); if (!authorized) { From 184ab9ae75f4154ead99e2d4723862a18417905b Mon Sep 17 00:00:00 2001 From: dfed25 Date: Sat, 25 Apr 2026 10:45:03 -0700 Subject: [PATCH 2/6] fix: auto-sync new sources and relax scoped retrieval threshold Sync hire knowledge immediately after source creation and improve hire-scoped retrieval recall so newly added docs are available in chat without manual sync friction. Made-with: Cursor --- src/app/manager/tasks/page.tsx | 14 ++++++++++++++ src/lib/retrieval.ts | 5 +++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/app/manager/tasks/page.tsx b/src/app/manager/tasks/page.tsx index a8b9aea..a42942b 100644 --- a/src/app/manager/tasks/page.tsx +++ b/src/app/manager/tasks/page.tsx @@ -193,6 +193,20 @@ export default function ManagerTasksPage() { if (!res.ok) return setMessage(data.error || "Failed to add source."); setSources((prev) => [...prev, data.source]); setSourceForm((prev) => ({ ...prev, title: "", url: "" })); + setMessage("Source added. Syncing now..."); + setSyncing(true); + const syncRes = await fetch("/api/sync/knowledge", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ hireId: selectedHireId }), + }); + const syncData = await syncRes.json().catch(() => ({})); + setSyncing(false); + if (!syncRes.ok) { + setMessage(syncData.error || "Source added, but sync failed. Try syncing manually."); + return; + } + setMessage(`Source added and synced: ${syncData.result?.synced ?? 0}/${syncData.result?.scanned ?? 0} docs.`); } async function deleteSource(sourceId: string) { diff --git a/src/lib/retrieval.ts b/src/lib/retrieval.ts index d847992..feafa8e 100644 --- a/src/lib/retrieval.ts +++ b/src/lib/retrieval.ts @@ -35,13 +35,14 @@ function matchesHireScope(content: string, hireId?: string): boolean { export async function retrieveDocs(question: string, hireId?: string): Promise { try { const TOP_K = 3; + const OVERFETCH = hireId ? TOP_K * 12 : TOP_K * 3; const embedding = await generateEmbedding(question); if (!embedding) return []; const { data: documents, error } = await supabaseAdmin.rpc("match_documents", { query_embedding: `[${embedding.join(",")}]`, - match_threshold: 0.7, - match_count: hireId ? TOP_K * 10 : TOP_K + match_threshold: hireId ? 0.45 : 0.6, + match_count: OVERFETCH }); if (error) { From 64d7d68f84b644f71d61d412167b256c5f036c46 Mon Sep 17 00:00:00 2001 From: dfed25 Date: Sat, 25 Apr 2026 10:56:04 -0700 Subject: [PATCH 3/6] Fix hire chat retrieval: strict hire scope, backoff search, keyword fallback; stop filtering non-positive similarity in chat API Made-with: Cursor --- src/app/api/chat/route.ts | 6 +- src/lib/retrieval.ts | 223 ++++++++++++++++++++++++++++++++++---- 2 files changed, 208 insertions(+), 21 deletions(-) diff --git a/src/app/api/chat/route.ts b/src/app/api/chat/route.ts index 2ddaf85..cf325e3 100644 --- a/src/app/api/chat/route.ts +++ b/src/app/api/chat/route.ts @@ -58,7 +58,11 @@ export async function POST(req: Request) { } const retrieved = await retrieveDocs(question, hireId); - const validDocs = retrieved.filter(r => r.score > 0).map(r => r.doc); + // NOTE: `match_documents` uses cosine *distance* semantics; similarity scores are not guaranteed to be > 0. + const validDocs = retrieved + .slice() + .sort((a, b) => b.score - a.score) + .map((r) => r.doc); const sources: ChatSource[] = validDocs.map((d) => ({ title: d.title, diff --git a/src/lib/retrieval.ts b/src/lib/retrieval.ts index feafa8e..9f7c68b 100644 --- a/src/lib/retrieval.ts +++ b/src/lib/retrieval.ts @@ -21,41 +21,224 @@ interface MatchedDocument { similarity: number; } -function matchesHireScope(content: string, hireId?: string): boolean { - if (!hireId) return true; - const scopeMarkers = content +const STOPWORDS = new Set([ + "a", + "an", + "and", + "are", + "as", + "at", + "be", + "but", + "by", + "can", + "did", + "do", + "does", + "for", + "from", + "had", + "has", + "have", + "how", + "i", + "if", + "in", + "into", + "is", + "it", + "its", + "me", + "my", + "of", + "on", + "or", + "our", + "please", + "so", + "tell", + "that", + "the", + "their", + "them", + "then", + "there", + "these", + "they", + "this", + "to", + "too", + "was", + "we", + "were", + "what", + "when", + "where", + "which", + "who", + "why", + "with", + "you", + "your", +]); + +function extractScopeMarkers(content: string): string[] { + return content .split("\n") .map((line) => line.trim()) .filter((line) => /^\[hire:[^\]]+\]$/.test(line)); - if (scopeMarkers.length === 0) return true; - if (scopeMarkers.includes(`[hire:${hireId}]`)) return true; - return scopeMarkers.includes("[hire:global]"); } -export async function retrieveDocs(question: string, hireId?: string): Promise { - try { - const TOP_K = 3; - const OVERFETCH = hireId ? TOP_K * 12 : TOP_K * 3; - const embedding = await generateEmbedding(question); - if (!embedding) return []; +function matchesRetrievalScope(content: string, hireId?: string): boolean { + const markers = extractScopeMarkers(content); + + // Hire chat: only allow explicitly hire-scoped chunks for that hire. + // This prevents global Slack onboarding dumps from masquerading as hire-specific docs. + if (hireId) { + return markers.includes(`[hire:${hireId}]`); + } + + // Global chat: allow unscoped legacy rows, or explicitly global rows. + // If a chunk is explicitly scoped to a specific hire, do not surface it in global chat. + if (markers.length === 0) return true; + return markers.includes("[hire:global]"); +} + +function tokenizeQuestion(question: string): string[] { + const raw = question + .toLowerCase() + .replace(/https?:\/\/\S+/g, " ") + .replace(/[^a-z0-9]+/g, " ") + .split(/\s+/g) + .map((t) => t.trim()) + .filter((t) => t.length >= 3 && !STOPWORDS.has(t)); + + // De-dupe while preserving order + const out: string[] = []; + const seen = new Set(); + for (const t of raw) { + if (seen.has(t)) continue; + seen.add(t); + out.push(t); + } + return out.slice(0, 8); +} + +async function fetchHireKeywordFallback(question: string, hireId: string): Promise { + const tokens = tokenizeQuestion(question); + if (tokens.length === 0) return []; + + const hirePrefix = `${hireId}:`; + const orFilter = tokens.map((t) => `title.ilike.%${t}%`).join(","); + + const { data, error } = await supabaseAdmin + .from("runbook_documents") + .select("id,title,content,url,provider") + .ilike("external_id", `${hirePrefix}%`) + .or(orFilter) + .order("created_at", { ascending: false }) + .limit(12); + + if (error) { + console.error("Hire keyword fallback error:", error); + return []; + } + + return (data || []).map((row) => ({ + id: String(row.id), + title: String(row.title || ""), + content: String(row.content || ""), + url: (row.url as string | null) ?? null, + provider: String(row.provider || "manual"), + similarity: 0.2, + })); +} + +async function matchDocumentsWithBackoff( + embedding: number[], + hireId?: string +): Promise { + const TOP_K = 3; + const thresholds = hireId ? [0.45, 0.35, 0.25, 0.15] : [0.6, 0.45, 0.35]; + const matchCounts = hireId ? [TOP_K * 12, TOP_K * 18, TOP_K * 24, TOP_K * 30] : [TOP_K * 3, TOP_K * 6, TOP_K * 10]; + + let best: MatchedDocument[] = []; + for (let i = 0; i < thresholds.length; i++) { + const threshold = thresholds[i]!; + const match_count = matchCounts[Math.min(i, matchCounts.length - 1)]!; const { data: documents, error } = await supabaseAdmin.rpc("match_documents", { query_embedding: `[${embedding.join(",")}]`, - match_threshold: hireId ? 0.45 : 0.6, - match_count: OVERFETCH + match_threshold: threshold, + match_count, }); if (error) { console.error("Vector Search Error:", error); - return []; + return best; } - if (!documents) return []; + const batch = (documents || []) as MatchedDocument[]; + const merged = new Map(); + for (const doc of [...best, ...batch]) merged.set(doc.id, doc); + best = Array.from(merged.values()).sort((a, b) => b.similarity - a.similarity); + + const scopedCount = best.filter((d) => matchesRetrievalScope(d.content, hireId)).length; + if (scopedCount >= TOP_K) break; + } + + return best; +} + +export async function retrieveDocs(question: string, hireId?: string): Promise { + try { + const TOP_K = 3; + const embedding = await generateEmbedding(question); + if (!embedding) { + if (!hireId) return []; + const fallbackOnly = await fetchHireKeywordFallback(question, hireId); + return fallbackOnly + .filter((doc) => matchesRetrievalScope(doc.content, hireId)) + .slice(0, TOP_K) + .map((doc) => ({ + doc: { + id: doc.id, + title: doc.title, + content: doc.content, + url: doc.url, + provider: doc.provider, + }, + score: doc.similarity, + })); + } - const scoped = (documents as MatchedDocument[]) - .filter((doc) => matchesHireScope(doc.content, hireId)) + const documents = await matchDocumentsWithBackoff(embedding, hireId); + const vectorScoped = documents + .filter((doc) => matchesRetrievalScope(doc.content, hireId)) .slice(0, TOP_K); - return scoped.map((doc) => ({ + + if (hireId && vectorScoped.length < TOP_K) { + const fallback = await fetchHireKeywordFallback(question, hireId); + const merged = new Map(); + for (const doc of [...vectorScoped, ...fallback]) merged.set(doc.id, doc); + const filled = Array.from(merged.values()) + .filter((doc) => matchesRetrievalScope(doc.content, hireId)) + .sort((a, b) => b.similarity - a.similarity) + .slice(0, TOP_K); + + return filled.map((doc) => ({ + doc: { + id: doc.id, + title: doc.title, + content: doc.content, + url: doc.url, + provider: doc.provider, + }, + score: doc.similarity, + })); + } + + return vectorScoped.map((doc) => ({ doc: { id: doc.id, title: doc.title, @@ -63,7 +246,7 @@ export async function retrieveDocs(question: string, hireId?: string): Promise Date: Sat, 25 Apr 2026 10:57:27 -0700 Subject: [PATCH 4/6] Hire retrieval: merge vector + title keyword hits and rerank so named sources surface Made-with: Cursor --- src/lib/retrieval.ts | 97 +++++++++++++++++++++++++++++--------------- 1 file changed, 64 insertions(+), 33 deletions(-) diff --git a/src/lib/retrieval.ts b/src/lib/retrieval.ts index 9f7c68b..11e209c 100644 --- a/src/lib/retrieval.ts +++ b/src/lib/retrieval.ts @@ -154,6 +154,33 @@ async function fetchHireKeywordFallback(question: string, hireId: string): Promi })); } +function titleKeywordScore(title: string, tokens: string[]): number { + const hay = title.toLowerCase(); + let score = 0; + for (const t of tokens) { + if (!t) continue; + if (hay.includes(t)) score += 1; + } + return score; +} + +function rankHireDocuments(docs: MatchedDocument[], tokens: string[]): MatchedDocument[] { + const scored = docs.map((d) => { + const kw = titleKeywordScore(d.title, tokens); + // Strongly prefer explicit title matches from the user's question, then embedding similarity. + const rank = kw * 2.5 + d.similarity; + return { d, rank, kw }; + }); + + scored.sort((a, b) => { + if (b.rank !== a.rank) return b.rank - a.rank; + if (b.kw !== a.kw) return b.kw - a.kw; + return b.d.similarity - a.d.similarity; + }); + + return scored.map((s) => s.d); +} + async function matchDocumentsWithBackoff( embedding: number[], hireId?: string @@ -193,40 +220,40 @@ async function matchDocumentsWithBackoff( export async function retrieveDocs(question: string, hireId?: string): Promise { try { const TOP_K = 3; + const hireTokens = hireId ? tokenizeQuestion(question) : []; const embedding = await generateEmbedding(question); if (!embedding) { if (!hireId) return []; const fallbackOnly = await fetchHireKeywordFallback(question, hireId); - return fallbackOnly - .filter((doc) => matchesRetrievalScope(doc.content, hireId)) - .slice(0, TOP_K) - .map((doc) => ({ - doc: { - id: doc.id, - title: doc.title, - content: doc.content, - url: doc.url, - provider: doc.provider, - }, - score: doc.similarity, - })); + const ranked = rankHireDocuments( + fallbackOnly.filter((doc) => matchesRetrievalScope(doc.content, hireId)), + hireTokens, + ); + return ranked.slice(0, TOP_K).map((doc) => ({ + doc: { + id: doc.id, + title: doc.title, + content: doc.content, + url: doc.url, + provider: doc.provider, + }, + score: doc.similarity, + })); } const documents = await matchDocumentsWithBackoff(embedding, hireId); - const vectorScoped = documents - .filter((doc) => matchesRetrievalScope(doc.content, hireId)) - .slice(0, TOP_K); + const vectorScoped = documents.filter((doc) => matchesRetrievalScope(doc.content, hireId)); - if (hireId && vectorScoped.length < TOP_K) { - const fallback = await fetchHireKeywordFallback(question, hireId); + if (hireId) { + const fallback = hireTokens.length > 0 ? await fetchHireKeywordFallback(question, hireId) : []; const merged = new Map(); for (const doc of [...vectorScoped, ...fallback]) merged.set(doc.id, doc); - const filled = Array.from(merged.values()) - .filter((doc) => matchesRetrievalScope(doc.content, hireId)) - .sort((a, b) => b.similarity - a.similarity) - .slice(0, TOP_K); + const ranked = rankHireDocuments( + Array.from(merged.values()).filter((doc) => matchesRetrievalScope(doc.content, hireId)), + hireTokens, + ); - return filled.map((doc) => ({ + return ranked.slice(0, TOP_K).map((doc) => ({ doc: { id: doc.id, title: doc.title, @@ -238,16 +265,20 @@ export async function retrieveDocs(question: string, hireId?: string): Promise ({ - doc: { - id: doc.id, - title: doc.title, - content: doc.content, - url: doc.url, - provider: doc.provider, - }, - score: doc.similarity, - })); + return vectorScoped + .slice() + .sort((a, b) => b.similarity - a.similarity) + .slice(0, TOP_K) + .map((doc) => ({ + doc: { + id: doc.id, + title: doc.title, + content: doc.content, + url: doc.url, + provider: doc.provider, + }, + score: doc.similarity, + })); } catch (err) { console.error("Retrieval Pipeline Error:", err); return []; From 2797384fd2806ef4ffd2053b66a475a12de59fb9 Mon Sep 17 00:00:00 2001 From: dfed25 Date: Sat, 25 Apr 2026 11:18:29 -0700 Subject: [PATCH 5/6] Ingest Google Drive folder/file URLs via Drive API for hire sync (not web UI HTML) Made-with: Cursor --- src/lib/ingestion/gdrive.ts | 250 +++++++++++++++++++++++++++++++++++- src/lib/vectorizer.ts | 21 ++- 2 files changed, 264 insertions(+), 7 deletions(-) diff --git a/src/lib/ingestion/gdrive.ts b/src/lib/ingestion/gdrive.ts index 4fecec2..45b05e0 100644 --- a/src/lib/ingestion/gdrive.ts +++ b/src/lib/ingestion/gdrive.ts @@ -6,7 +6,11 @@ interface IngestionDoc { content: string; } -const getDriveClient = () => { +const MAX_FOLDER_ITEMS = 45; +const MAX_CHARS_PER_FILE = 60_000; +const MAX_TOTAL_FOLDER_CHARS = 350_000; + +function buildDriveClient() { const clientId = process.env.GOOGLE_CLIENT_ID; const clientSecret = process.env.GOOGLE_CLIENT_SECRET; const refreshToken = process.env.GOOGLE_REFRESH_TOKEN; @@ -20,10 +24,246 @@ const getDriveClient = () => { return null; } return google.drive({ version: "v3", auth }); -}; +} + +type DriveClient = NonNullable>; + +function truncateText(text: string, max: number): string { + const t = text.trim(); + if (t.length <= max) return t; + return `${t.slice(0, max)}\n\n...[truncated ${t.length - max} characters]...`; +} + +/** + * Recognize folder / file / open?id= links on drive.google.com. + */ +export function parseDriveGoogleUrl(rawUrl: string): { kind: "folder" | "file" | "open"; id: string } | null { + try { + const url = new URL(rawUrl.trim()); + const host = url.hostname.toLowerCase(); + if (host !== "drive.google.com") return null; + + const folderMatch = url.pathname.match(/\/folders\/([a-zA-Z0-9_-]+)/); + if (folderMatch?.[1]) return { kind: "folder", id: folderMatch[1] }; + + const fileMatch = url.pathname.match(/\/file\/d\/([a-zA-Z0-9_-]+)/); + if (fileMatch?.[1]) return { kind: "file", id: fileMatch[1] }; + + const idParam = url.searchParams.get("id"); + if (idParam && /^[a-zA-Z0-9_-]+$/.test(idParam)) return { kind: "open", id: idParam }; + + return null; + } catch { + return null; + } +} + +async function driveFileMeta(drive: DriveClient, fileId: string) { + const { data } = await drive.files.get({ + fileId, + fields: "id, name, mimeType", + supportsAllDrives: true, + }); + return { + id: data.id || fileId, + name: data.name || "Untitled", + mimeType: data.mimeType || "", + }; +} + +async function exportGoogleWorkspacePlain( + drive: DriveClient, + fileId: string, + mimeType: string +): Promise { + try { + if (mimeType === "application/vnd.google-apps.document") { + const res = await drive.files.export({ + fileId, + mimeType: "text/plain", + }); + return typeof res.data === "string" ? res.data : null; + } + if (mimeType === "application/vnd.google-apps.spreadsheet") { + const res = await drive.files.export({ + fileId, + mimeType: "text/csv", + }); + return typeof res.data === "string" ? res.data : null; + } + if (mimeType === "application/vnd.google-apps.presentation") { + const res = await drive.files.export({ + fileId, + mimeType: "text/plain", + }); + return typeof res.data === "string" ? res.data : null; + } + return null; + } catch { + return null; + } +} + +async function downloadPlainFile( + drive: DriveClient, + fileId: string, + mimeType: string +): Promise { + const textish = + mimeType.startsWith("text/") || + mimeType === "application/json" || + mimeType === "application/javascript" || + mimeType.endsWith("+json") || + mimeType.endsWith("+xml"); + + if (!textish) return null; + + try { + const res = await drive.files.get( + { fileId, alt: "media", supportsAllDrives: true }, + { responseType: "arraybuffer" } + ); + const buf = Buffer.from(res.data as ArrayBuffer); + return buf.toString("utf8"); + } catch { + return null; + } +} + +async function extractDriveFileText( + drive: DriveClient, + fileId: string, + name: string, + mimeType: string +): Promise { + if (mimeType === "application/vnd.google-apps.shortcut") { + return `Shortcut "${name}" — replace this with the target file or folder URL to index its contents.`; + } + if (mimeType === "application/vnd.google-apps.folder") { + return `Subfolder "${name}" — add this folder URL as its own knowledge source to index its files.`; + } + + const exported = await exportGoogleWorkspacePlain(drive, fileId, mimeType); + if (exported) return truncateText(exported, MAX_CHARS_PER_FILE); + + const plain = await downloadPlainFile(drive, fileId, mimeType); + if (plain) return truncateText(plain, MAX_CHARS_PER_FILE); + + return `No extractable text for "${name}" (${mimeType}). Open in Drive: https://drive.google.com/file/d/${fileId}/view`; +} + +async function ingestSingleDriveFile( + drive: DriveClient, + fileId: string, + hintName?: string +): Promise<{ title: string; content: string }> { + const meta = await driveFileMeta(drive, fileId); + const body = await extractDriveFileText(drive, meta.id, meta.name, meta.mimeType); + const content = [`Google Drive file: ${meta.name}`, `MIME type: ${meta.mimeType}`, `URL: https://drive.google.com/file/d/${meta.id}/view`, "", body].join("\n"); + return { title: `Drive: ${hintName ?? meta.name}`, content }; +} + +async function ingestDriveFolder( + drive: DriveClient, + folderId: string, + folderName: string, + sourceUrl: string +): Promise<{ title: string; content: string }> { + const lines: string[] = []; + lines.push(`Google Drive folder: ${folderName}`); + lines.push(`Folder ID: ${folderId}`); + lines.push(`Folder URL: ${sourceUrl}`); + lines.push(""); + + const list = await drive.files.list({ + q: `'${folderId}' in parents and trashed = false`, + pageSize: MAX_FOLDER_ITEMS, + fields: "files(id, name, mimeType)", + supportsAllDrives: true, + includeItemsFromAllDrives: true, + orderBy: "folder,name", + }); + + const files = list.data.files || []; + if (files.length === 0) { + lines.push("(This folder has no files, or the service account cannot list them.)"); + return { title: `Drive folder: ${folderName}`, content: lines.join("\n") }; + } + + lines.push(`Items (${files.length}):`); + lines.push(""); + + for (const f of files) { + const id = f.id; + const name = f.name || "Untitled"; + const mime = f.mimeType || ""; + if (!id) continue; + + lines.push(`━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━`); + lines.push(`## ${name}`); + lines.push(`MIME: ${mime}`); + lines.push(""); + + try { + const chunk = await extractDriveFileText(drive, id, name, mime); + lines.push(chunk); + } catch (err) { + lines.push(`(Failed to read this item: ${err instanceof Error ? err.message : String(err)})`); + } + lines.push(""); + if (lines.join("\n").length > MAX_TOTAL_FOLDER_CHARS) { + lines.push(`...[stopped: folder content capped at ${MAX_TOTAL_FOLDER_CHARS} characters]...`); + break; + } + } + + return { + title: `Drive folder: ${folderName}`, + content: truncateText(lines.join("\n"), MAX_TOTAL_FOLDER_CHARS), + }; +} + +/** + * Fetch real Drive content for a hire-scoped URL (folder listing + per-file text). + * Returns null if URL is not a Drive link, credentials are missing, or the API call fails. + * Callers should fall back to generic URL fetch only for non-Drive pages. + */ +export async function fetchGoogleDriveUrlContent( + rawUrl: string +): Promise<{ title: string; content: string } | null> { + const parsed = parseDriveGoogleUrl(rawUrl); + if (!parsed) return null; + + const drive = buildDriveClient(); + if (!drive) return null; + + try { + if (parsed.kind === "folder") { + const meta = await driveFileMeta(drive, parsed.id); + if (meta.mimeType !== "application/vnd.google-apps.folder") { + return ingestSingleDriveFile(drive, parsed.id, meta.name); + } + return await ingestDriveFolder(drive, parsed.id, meta.name, rawUrl.trim()); + } + + if (parsed.kind === "file") { + return await ingestSingleDriveFile(drive, parsed.id); + } + + // open?id= — resolve via metadata + const meta = await driveFileMeta(drive, parsed.id); + if (meta.mimeType === "application/vnd.google-apps.folder") { + return await ingestDriveFolder(drive, meta.id, meta.name, rawUrl.trim()); + } + return await ingestSingleDriveFile(drive, meta.id, meta.name); + } catch (e) { + console.error("Google Drive URL ingest error:", e); + return null; + } +} export async function fetchDriveDocuments(): Promise { - const drive = getDriveClient(); + const drive = buildDriveClient(); if (!drive) { console.warn("Skipping Google Drive Sync: No credentials configured."); return []; @@ -33,7 +273,7 @@ export async function fetchDriveDocuments(): Promise { const res = await drive.files.list({ pageSize: 10, fields: "nextPageToken, files(id, name, mimeType)", - q: "mimeType='application/vnd.google-apps.document'" + q: "mimeType='application/vnd.google-apps.document'", }); const files = res.data.files || []; @@ -44,7 +284,7 @@ export async function fetchDriveDocuments(): Promise { try { const exported = await drive.files.export({ fileId: f.id!, - mimeType: "text/plain" + mimeType: "text/plain", }); content = typeof exported.data === "string" ? exported.data : ""; } catch { diff --git a/src/lib/vectorizer.ts b/src/lib/vectorizer.ts index ce1f765..dc5e86c 100644 --- a/src/lib/vectorizer.ts +++ b/src/lib/vectorizer.ts @@ -1,5 +1,5 @@ import { fetchNotionPages } from "./ingestion/notion"; -import { fetchDriveDocuments } from "./ingestion/gdrive"; +import { fetchDriveDocuments, fetchGoogleDriveUrlContent, parseDriveGoogleUrl } from "./ingestion/gdrive"; import { fetchSlackChannelHistory } from "./ingestion/slack"; import { fetchUrlDocument } from "./ingestion/url"; import { supabaseAdmin } from "./supabase-admin"; @@ -69,7 +69,24 @@ Source title: ${source.title} Source URL: ${source.url} Source type: ${source.type}`; - const fetched = await fetchUrlDocument(source.url); + const url = source.url.trim(); + const driveParsed = parseDriveGoogleUrl(url); + let fetched: { title: string; content: string } | null = null; + + // drive.google.com folder/file links must use the Drive API — HTTP fetch only returns the web UI shell. + if (driveParsed) { + fetched = await fetchGoogleDriveUrlContent(url); + if (!fetched) { + fetched = { + title: source.title || "Google Drive", + content: `Could not read this Google Drive resource via the Drive API. Confirm GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET, and GOOGLE_REFRESH_TOKEN (with Drive file read scope), then re-sync.\nLink: ${url}`, + }; + } + } else { + const web = await fetchUrlDocument(url); + fetched = web ? { title: web.title, content: web.content } : null; + } + const content = fetched?.content || fallbackContent; const title = fetched?.title || source.title || "Knowledge source"; const scopedContent = content.includes(scopeToken) ? content : `${scopeToken}\n${content}`; From 97851cc7c17e83a2281cdec27f00bf9ac6d3bba8 Mon Sep 17 00:00:00 2001 From: dfed25 Date: Sat, 25 Apr 2026 11:27:48 -0700 Subject: [PATCH 6/6] Chat: structured markdown-style answers in prompt; readable UI (headings, lists, bold); cleaner source excerpts Made-with: Cursor --- src/app/api/chat/route.ts | 17 +++-- src/app/dashboard/page.tsx | 17 +++-- src/components/ui/ChatMessageBody.tsx | 92 +++++++++++++++++++++++++++ src/lib/prompts.ts | 17 +++-- 4 files changed, 127 insertions(+), 16 deletions(-) create mode 100644 src/components/ui/ChatMessageBody.tsx diff --git a/src/app/api/chat/route.ts b/src/app/api/chat/route.ts index cf325e3..cff6176 100644 --- a/src/app/api/chat/route.ts +++ b/src/app/api/chat/route.ts @@ -32,6 +32,14 @@ function escapeRegExp(string: string) { return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); } +/** Strip hire scope lines and collapse whitespace for compact source previews. */ +function excerptForChatSource(content: string, maxLen: number): string { + const lines = (content || "").split("\n").filter((line) => !/^\[hire:[^\]]+\]$/.test(line.trim())); + const body = lines.join(" ").replace(/\s+/g, " ").trim(); + if (!body) return ""; + return body.length > maxLen ? `${body.slice(0, maxLen)}…` : body; +} + export async function POST(req: Request) { try { const body = await req.json(); @@ -65,11 +73,8 @@ export async function POST(req: Request) { .map((r) => r.doc); const sources: ChatSource[] = validDocs.map((d) => ({ - title: d.title, - excerpt: (() => { - const content = d.content || ""; - return content.length > 180 ? `${content.slice(0, 180)}...` : content; - })(), + title: d.title.replace(/^\[hire:[^\]]+\]\s*/i, "").trim() || d.title, + excerpt: excerptForChatSource(d.content || "", 260), url: d.url || undefined, })); const context = validDocs @@ -87,7 +92,7 @@ export async function POST(req: Request) { } try { - const userPrompt = `Company Context:\n${context || "No context found."}\n\nUser Question: ${question}\n\nAnswer with concise guidance and ground it in the provided sources.`; + const userPrompt = `Company Context:\n${context || "No context found."}\n\nUser Question: ${question}\n\nWrite the answer in clear, scannable markdown: use "## " section headings, "- " bullets or numbered steps where appropriate, and **bold** for key terms. Ground every claim in the sources above.`; const answer = await generateFromGemini(CHAT_SYSTEM_PROMPT, userPrompt); return NextResponse.json({ answer, sources }); } catch (e) { diff --git a/src/app/dashboard/page.tsx b/src/app/dashboard/page.tsx index 4477ab8..3502611 100644 --- a/src/app/dashboard/page.tsx +++ b/src/app/dashboard/page.tsx @@ -4,6 +4,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { DEMO_PERSONAS, DEMO_QUESTIONS } from "@/lib/demoScenario"; import { ChatSource, Hire, Lesson, LessonSlide, OnboardingTask } from "@/lib/types"; import { AppButton } from "@/components/ui/AppButton"; +import { ChatMessageBody } from "@/components/ui/ChatMessageBody"; import { SectionCard } from "@/components/ui/SectionCard"; import { StatusBadge } from "@/components/ui/StatusBadge"; @@ -301,13 +302,15 @@ export default function DashboardPage() {
{messages.map((message) => (
-

- {message.text} -

+ +
{message.role === "assistant" && message.sources?.length ? (
{message.sources.map((source, idx) => ( @@ -324,7 +327,9 @@ export default function DashboardPage() { ) : (

{source.title}

)} -

{source.excerpt}

+

+ {source.excerpt} +

))}
diff --git a/src/components/ui/ChatMessageBody.tsx b/src/components/ui/ChatMessageBody.tsx new file mode 100644 index 0000000..c3cb024 --- /dev/null +++ b/src/components/ui/ChatMessageBody.tsx @@ -0,0 +1,92 @@ +import { Fragment, type ReactNode } from "react"; + +function formatInline(text: string): ReactNode[] { + const parts = text.split(/(\*\*[^*]+\*\*)/g); + return parts.map((part, i) => { + const m = part.match(/^\*\*(.+)\*\*$/); + if (m) { + return ( + + {m[1]} + + ); + } + return part ? {part} : null; + }); +} + +function isBulletLine(line: string): boolean { + return /^\s*[-*]\s+/.test(line) || /^\s*\d+\.\s+/.test(line); +} + +function stripBulletPrefix(line: string): string { + return line.replace(/^\s*[-*]\s+/, "").replace(/^\s*\d+\.\s+/, ""); +} + +/** + * Renders assistant chat with readable structure (markdown-lite from the model). + * User messages stay plain. + */ +export function ChatMessageBody({ role, text }: { role: "user" | "assistant"; text: string }) { + if (role === "user") { + return {text}; + } + + const blocks = text.trim().split(/\n\n+/).filter((b) => b.trim()); + if (blocks.length === 0) { + return —; + } + + return ( +
+ {blocks.map((block, bi) => { + const lines = block.split("\n"); + const first = lines[0]?.trim() ?? ""; + + if (first.startsWith("## ")) { + const title = first.replace(/^##\s+/, ""); + const rest = lines + .slice(1) + .join("\n") + .trim(); + return ( +
+

{formatInline(title)}

+ {rest ? ( +

{formatInline(rest)}

+ ) : null} +
+ ); + } + + const nonEmpty = lines.filter((l) => l.trim()); + const allBullets = nonEmpty.length > 0 && nonEmpty.every((l) => isBulletLine(l)); + if (allBullets) { + return ( +
    + {nonEmpty.map((line, li) => ( +
  • + {formatInline(stripBulletPrefix(line))} +
    • + ))} +
+ ); + } + + return ( +

+ {lines.map((line, li) => ( + + {li > 0 ?
: null} + {formatInline(line)} + + ))} +

+ ); + })} +
+ ); +} diff --git a/src/lib/prompts.ts b/src/lib/prompts.ts index ada854f..bda983d 100644 --- a/src/lib/prompts.ts +++ b/src/lib/prompts.ts @@ -1,9 +1,18 @@ export const CHAT_SYSTEM_PROMPT = `You are Runbook, an onboarding copilot for new employees. Use only the provided company documents. -Answer clearly and practically. -If the user asks how to do something, give numbered steps. -Always cite the source document titles inline or at the end if applicable. -If the information is missing, say what is missing and who the user should ask.`; + +Formatting (required): +- Use short sections separated by a blank line. +- Start each section with a markdown-style heading on its own line, e.g. "## Summary" then the paragraph below it. +- Use bullet lists with "- " for multiple items; use numbered lists ("1. ", "2. ") for sequences or steps. +- Bold the most important phrases using **double asterisks** (sparingly). +- Keep paragraphs under ~4 sentences; avoid one giant wall of text. +- When summarizing a Drive folder or several files, use a "## What’s in this folder" (or similar) section and bullets for each file or theme. + +Substance: +- Answer clearly and practically. If the user asks how to do something, use numbered steps. +- Mention source document titles in the answer where helpful (you may also reference them at the end). +- If the information is missing, say what is missing and who the user should ask.`; export const TASK_GENERATION_SYSTEM_PROMPT = `Given these company docs, generate a first-week onboarding checklist for a new engineer. Return strictly a JSON array with objects containing fields: id, title, description, estimatedTime, sourceTitle, and status ("todo", "in_progress", or "complete").