diff --git a/.gitignore b/.gitignore index 7ae3df88..0eb32b7b 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,9 @@ dist/ *.sqlite *.sqlite3 +# ---- RAG data (large files — stored in Supabase Storage) ---- +backend/data/*.json + # ---- Env / secrets ---- .env .env.* diff --git a/backend/data/scrape_summary.json b/backend/data/scrape_summary.json new file mode 100644 index 00000000..b22086d5 --- /dev/null +++ b/backend/data/scrape_summary.json @@ -0,0 +1,210 @@ +{ + "total_courses": 8720, + "total_errors": 50, + "errors": [ + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-gd-650a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-gd-512a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-pe-530a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-en-522a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-or-530a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-rs-529a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-rs-640a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-ph-512a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-os-520a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-rs-641a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-rs-525a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-pe-521a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-pd-640a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-ph-530a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-pd-530a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-pe-640a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-od-642a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-gd-520a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-os-530a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-gd-511a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-od-522a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-rs-522a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-ph-541a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-os-532a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-gd-660a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-ph-521a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-od-531a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-rs-523a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-en-521a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-od-644a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-gd-581a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-pa-530a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-rs-534a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-md-531a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-os-521a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-rs-546a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-rs-542a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-en-640a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-gd-642a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-rs-519a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-pe-520a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-rs-527a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-rs-521a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-rs-642a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-os-640a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-ph-544a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-gd-540a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-rs-524a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-ph-524a/", + "error": "empty response" + }, + { + "url": "https://www.bu.edu/academics/sdm/courses/sdm-rs-532a/", + "error": "empty response" + } + ], + "elapsed_seconds": 1890, + "completed_at": "2026-06-27T01:39:55.726124+00:00", + "semester_tag": "fall_2026", + "output_file": "C:\\Users\\Jack\\Desktop\\VS Code\\sapling\\backend\\data\\bu_catalog_fall_2026.json" +} \ No newline at end of file diff --git a/backend/db/connection.py b/backend/db/connection.py index cf075e51..94844100 100644 --- a/backend/db/connection.py +++ b/backend/db/connection.py @@ -101,3 +101,11 @@ def delete(self, filters: dict) -> list: def table(name: str) -> SupabaseTable: return SupabaseTable(name) + + +def rpc(function_name: str, params: dict) -> list: + """Call a Supabase Postgres function via /rest/v1/rpc/{function_name}.""" + url = f"{REST_URL}/rpc/{function_name}" + r = _client.post(url, json=params) + r.raise_for_status() + return r.json() diff --git a/backend/routes/documents.py b/backend/routes/documents.py index 055fcfca..5af82261 100644 --- a/backend/routes/documents.py +++ b/backend/routes/documents.py @@ -785,6 +785,7 @@ async def event_stream(): ("invalidate_study_guide_cache", _invalidate_study_guide_cache, user_id, course_id), ("update_course_context", update_course_context, course_id), ("check_upload_achievements", _check_upload_achievements, user_id), + ("index_document_chunks", _index_document_chunks, doc_id, course_id, user_id, extracted_text, classification.category, getattr(summary, "abstract", "")), ) yield sapling_event_to_sse(SaplingEvent( @@ -898,6 +899,124 @@ def _check_upload_achievements(user_id: str) -> None: pass +def _chunk_text(text: str, chunk_size: int = 800, overlap: int = 100) -> list[str]: + """Split text into overlapping character-window chunks.""" + chunks = [] + start = 0 + while start < len(text): + end = start + chunk_size + chunks.append(text[start:end].strip()) + start += chunk_size - overlap + return [c for c in chunks if len(c) > 50] # drop near-empty tail chunks + + +def _index_document_chunks( + doc_id: str, + course_id: str, # Sapling UUID — resolved to BU code internally + user_id: str, + extracted_text: str, + category: str, + doc_summary: str = "", +) -> None: + """Chunk, embed, and upsert a document into course_chunks. + + Runs in a background thread via _spawn_post_roll after the document + is persisted, so it never blocks the SSE stream. + """ + import hashlib + import math + from google import genai as _genai + from google.genai import types as genai_types + from db.connection import table + import os, time + + MIN_COURSE_RELEVANCE = 0.35 # below this, document is likely off-topic for the course + + try: + # Resolve BU course code from Sapling UUID + rows = table("courses").select( + "course_code", filters={"id": f"eq.{course_id}"}, limit=1 + ) + bu_course_id = (rows[0].get("course_code") or course_id) if rows else course_id + + chunks = _chunk_text(extracted_text) + if not chunks: + return + + _gclient = _genai.Client(api_key=os.getenv("GEMINI_API_KEY", "")) + + def _embed_texts(texts: list[str]) -> list[list[float]]: + resp = _gclient.models.embed_content( + model="gemini-embedding-001", + contents=texts, + config=genai_types.EmbedContentConfig(output_dimensionality=768), + ) + return [list(e.values) for e in resp.embeddings] + + # ── Relevance gate ──────────────────────────────────────────────────── + # Fetch the catalog chunk embedding for this course and compare against + # the document's first chunk. Irrelevant documents are skipped to keep + # the index clean. + catalog_rows = table("course_chunks").select( + "embedding", + filters={"course_id": f"eq.{bu_course_id}", "category": "eq.catalog"}, + limit=1, + ) + if catalog_rows and catalog_rows[0].get("embedding"): + catalog_vec = catalog_rows[0]["embedding"] + # Use the AI-generated summary as the document representative — + # it's more reliable than raw first-chunk text (avoids cover pages, + # tables of contents, and boilerplate skewing the score). + sample_text = doc_summary or chunks[0] + doc_sample_vec = _embed_texts([sample_text])[0] + time.sleep(1.5) + # cosine similarity (vectors are unit-norm from the model) + dot = sum(a * b for a, b in zip(doc_sample_vec, catalog_vec)) + if dot < MIN_COURSE_RELEVANCE: + logger.warning( + "[RAG] doc %s skipped — relevance to %s is %.3f (< %.2f)", + doc_id, bu_course_id, dot, MIN_COURSE_RELEVANCE, + ) + return + + records = [] + for i, chunk_text in enumerate(chunks): + raw = f"{doc_id}::{i}::{chunk_text}" + cid = hashlib.sha256(raw.encode()).hexdigest() + records.append({ + "id": cid, + "course_id": bu_course_id, + "doc_id": doc_id, + "uploader_id": user_id, + "chunk_index": i, + "chunk_text": chunk_text, + "chunk_hash": cid, + "embedding": None, + "category": category, + "semester": "current", + "section_id": None, + "school": "", + }) + + # Embed in batches of 50 + BATCH = 50 + for i in range(0, len(records), BATCH): + batch = records[i : i + BATCH] + texts = [r["chunk_text"] for r in batch] + try: + vecs = _embed_texts(texts) + for rec, vec in zip(batch, vecs): + rec["embedding"] = vec + except Exception as e: + logger.warning("[RAG] embed failed for doc %s batch %d: %s", doc_id, i, e) + time.sleep(1.5) # stay under 3000 req/min quota + + table("course_chunks").upsert(records, on_conflict="id") + logger.info("[RAG] indexed %d chunks for doc %s", len(records), doc_id) + except Exception: + logger.exception("[RAG] _index_document_chunks failed for doc %s", doc_id) + + def _spawn_post_roll(*tasks: tuple) -> None: """Fire-and-forget post-roll work for SSE / non-FastAPI-BackgroundTasks contexts. Each tuple is (label, callable, *args). Exceptions in the diff --git a/backend/routes/learn.py b/backend/routes/learn.py index 26353dc7..a8487ba7 100644 --- a/backend/routes/learn.py +++ b/backend/routes/learn.py @@ -236,6 +236,23 @@ def _get_course_info(course_id: str) -> dict: return {"course_code": "", "course_name": ""} +def _get_catalog_chunk(course_code: str) -> str: + """Return the catalog chunk_text for a BU course code, or empty string.""" + if not course_code: + return "" + try: + rows = table("course_chunks").select( + "chunk_text", + filters={"course_id": f"eq.{course_code}", "category": "eq.catalog"}, + limit=1, + ) + if rows: + return rows[0].get("chunk_text", "") + except Exception as e: + print(f"[RAG] Failed to load catalog chunk for {course_code!r}: {e}") + return "" + + def build_system_prompt( mode: str, student_name: str, @@ -280,17 +297,22 @@ def build_system_prompt( + "\n\n---\n\n".join(doc_blocks) ) - if use_shared_context and course_id: - ctx = get_course_context(course_id) - if ctx: - course_info = _get_course_info(course_id) - course_label = f"{course_info['course_code']} - {course_info['course_name']}" if course_info['course_code'] else course_info['course_name'] - shared_block = ( - SHARED_CONTEXT_TEMPLATE - .replace("{course_name}", course_label) - .replace("{shared_context_json}", json.dumps(ctx, indent=2)) - ) - parts.append(shared_block) + if course_id: + course_info = _get_course_info(course_id) + catalog_text = _get_catalog_chunk(course_info.get("course_code", "")) + if catalog_text: + parts.append("COURSE CATALOG INFO (BU official course data):\n\n" + catalog_text) + + if use_shared_context: + ctx = get_course_context(course_id) + if ctx: + course_label = f"{course_info['course_code']} - {course_info['course_name']}" if course_info['course_code'] else course_info['course_name'] + shared_block = ( + SHARED_CONTEXT_TEMPLATE + .replace("{course_name}", course_label) + .replace("{shared_context_json}", json.dumps(ctx, indent=2)) + ) + parts.append(shared_block) parts.append(MODE_PROMPTS.get(mode, MODE_PROMPTS["socratic"])) return "\n\n".join(parts) @@ -478,6 +500,13 @@ async def _chat_via_agent( session_id=session_id, ) + from services.rag_service import retrieve_chunks, format_rag_context + bu_code = _get_course_info(course_id).get("course_code") if course_id else None + rag_chunks = retrieve_chunks(user_message, course_id=bu_code or None, k=5) + rag_block = format_rag_context(rag_chunks) + if rag_block: + user_message = rag_block + "\n\n[STUDENT QUESTION]\n" + user_message + if not use_shared_context: user_message = ( user_message @@ -528,11 +557,18 @@ async def _legacy_chat(body: ChatBody, request: Request) -> dict: course_id = _get_session_course_id(body.session_id) documents = _get_course_documents(body.user_id, course_id) + from services.rag_service import retrieve_chunks, format_rag_context + bu_code = _get_course_info(course_id).get("course_code") if course_id else None + rag_chunks = retrieve_chunks(body.message, course_id=bu_code or None, k=5) + rag_block = format_rag_context(rag_chunks) + system_prompt = build_system_prompt( body.mode, student_name, json.dumps(graph_data, indent=2), course_id=course_id, use_shared_context=body.use_shared_context, documents=documents, ) + if rag_block: + system_prompt = system_prompt + "\n\n" + rag_block try: raw = call_gemini_multiturn( diff --git a/backend/scripts/ingest_catalog.py b/backend/scripts/ingest_catalog.py new file mode 100644 index 00000000..bee7b1e9 --- /dev/null +++ b/backend/scripts/ingest_catalog.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +""" +Layer 0 ingestion: bu_catalog_fall_2026.json -> course_chunks table. + +Reads the scraped BU catalog JSON, builds one chunk per course +(title + description + prerequisites), embeds with Gemini +text-embedding-004, and upserts into course_chunks. + +Run from repo root: + python backend/scripts/ingest_catalog.py + +Resume-safe: already-ingested chunks are skipped via upsert on_conflict=id. +""" + +import hashlib +import json +import os +import sys +import time +from pathlib import Path + +from dotenv import load_dotenv +from google import genai +from google.genai import types as genai_types + +# ── Bootstrap paths ──────────────────────────────────────────────────────────── +load_dotenv(Path(__file__).parent.parent / ".env") +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from db.connection import table + +# ── Config ───────────────────────────────────────────────────────────────────── + +CATALOG_FILE = Path(__file__).parent.parent / "data" / "bu_catalog_fall_2026.json" +EMBED_MODEL = "gemini-embedding-001" +SEMESTER_TAG = "fall_2026" +BATCH_SIZE = 50 # courses per Supabase upsert batch +EMBED_BATCH = 100 # texts per Gemini embed_content call +RATE_DELAY = 3.0 # 100 texts / 3s = ~2,000 texts/min (limit is 3,000/min) + +_gemini = genai.Client(api_key=os.getenv("GEMINI_API_KEY", "")) + +# ── Helpers ──────────────────────────────────────────────────────────────────── + +def build_chunk_text(course: dict) -> str: + """Assemble the text that gets embedded for a catalog course.""" + parts = [ + f"Course: {course['course_code']} - {course['title']}", + f"School: {course['school'].upper()}", + ] + if course.get("credits"): + parts.append(f"Credits: {course['credits']}") + if course.get("description"): + parts.append(f"Description: {course['description']}") + if course.get("prerequisites"): + parts.append(f"Prerequisites: {course['prerequisites']}") + if course.get("instructors"): + parts.append(f"Instructor(s): {', '.join(course['instructors'])}") + if course.get("semester_offered"): + parts.append(f"Offered: {', '.join(course['semester_offered'])}") + return "\n".join(parts) + + +def chunk_id(course_id: str, chunk_text: str) -> str: + """Stable SHA-256 ID — same input always produces the same ID (dedup key).""" + raw = f"{course_id}::{chunk_text}" + return hashlib.sha256(raw.encode()).hexdigest() + + +def embed_batch(texts: list[str]) -> list[list[float]]: + """Embed up to EMBED_BATCH texts in one Gemini call, truncated to 768-dim.""" + response = _gemini.models.embed_content( + model=EMBED_MODEL, + contents=texts, + config=genai_types.EmbedContentConfig(output_dimensionality=768), + ) + return [list(e.values) for e in response.embeddings] + + +# ── Main ─────────────────────────────────────────────────────────────────────── + +def main() -> None: + with open(CATALOG_FILE, encoding="utf-8") as f: + catalog: list[dict] = json.load(f) + + print(f"Loaded {len(catalog):,} courses from {CATALOG_FILE.name}") + + # Build chunk records + records: list[dict] = [] + for course in catalog: + course_id = course["course_code"] # e.g. "CAS CS 330" + chunk_text = build_chunk_text(course) + cid = chunk_id(course_id, chunk_text) + records.append({ + "id": cid, + "course_id": course_id, + "doc_id": None, + "uploader_id": None, + "chunk_index": 0, + "chunk_text": chunk_text, + "chunk_hash": cid, + "embedding": None, # filled in below + "category": "catalog", + "semester": SEMESTER_TAG, + "section_id": None, + "school": course.get("school", ""), + }) + + print(f"Built {len(records):,} chunk records — embedding now...") + + # Embed in batches of EMBED_BATCH + texts = [r["chunk_text"] for r in records] + embeddings: list[list[float]] = [] + + for i in range(0, len(texts), EMBED_BATCH): + batch = texts[i : i + EMBED_BATCH] + try: + vecs = embed_batch(batch) + embeddings.extend(vecs) + except Exception as exc: + exc_str = str(exc) + # Parse retryDelay from Gemini 429 response if present + import re as _re + m = _re.search(r'retryDelay.*?(\d+)s', exc_str) + wait = int(m.group(1)) + 5 if m else 35 + print(f" Embed error at [{i}:{i+EMBED_BATCH}]: retrying in {wait}s") + time.sleep(wait) + try: + vecs = embed_batch(batch) + embeddings.extend(vecs) + except Exception as exc2: + print(f" FAILED [{i}:{i+EMBED_BATCH}]: {exc2} — inserting without embedding") + embeddings.extend([None] * len(batch)) + + done = min(i + EMBED_BATCH, len(texts)) + print(f" embedded {done:,}/{len(texts):,}", flush=True) + time.sleep(RATE_DELAY) + + # Attach embeddings to records + for rec, vec in zip(records, embeddings): + rec["embedding"] = vec + + # Upsert to Supabase in batches of BATCH_SIZE + print(f"\nUpserting to course_chunks...") + db = table("course_chunks") + inserted = 0 + + for i in range(0, len(records), BATCH_SIZE): + batch = records[i : i + BATCH_SIZE] + try: + db.upsert(batch, on_conflict="id") + inserted += len(batch) + print(f" upserted {inserted:,}/{len(records):,}", flush=True) + except Exception as exc: + print(f" Upsert error at [{i}:{i+BATCH_SIZE}]: {exc}") + + print(f"\nDone: {inserted:,} chunks upserted to course_chunks.") + + +if __name__ == "__main__": + main() diff --git a/backend/scripts/scrape_bu_catalog.py b/backend/scripts/scrape_bu_catalog.py new file mode 100644 index 00000000..b7112b48 --- /dev/null +++ b/backend/scripts/scrape_bu_catalog.py @@ -0,0 +1,376 @@ +#!/usr/bin/env python3 +""" +Layer 0: BU Course Catalog Scraper +Crawls bu.edu/academics/{school}/courses/ for all 22 schools, +extracts course data, and writes to backend/data/bu_catalog_{semester}.json + +Run from repo root: + python backend/scripts/scrape_bu_catalog.py + +Resume: re-running skips already-scraped URLs automatically. +""" + +import asyncio +import json +import re +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + +import httpx +from bs4 import BeautifulSoup + +# -- Config --------------------------------------------------------------------- + +SCHOOLS = [ + "cas", + "com", + "eng", + "cfa", + "cgs", + "cds", + "khc", + "gms", + "grs", + "sdm", + "met", + "questrom", + "sar", + "sha", + "law", + "sph", + "ssw", + "sth", + "wheelock", + "frederick-s-pardee-school-of-global-studies", +] + +BASE_URL = "https://www.bu.edu" +SEMESTER_TAG = "fall_2026" +CONCURRENCY = 8 # parallel course-page fetches +PAGE_DELAY = 0.4 # seconds between requests per worker + +OUTPUT_DIR = Path(__file__).parent.parent / "data" +OUTPUT_FILE = OUTPUT_DIR / f"bu_catalog_{SEMESTER_TAG}.json" +SUMMARY_FILE = OUTPUT_DIR / "scrape_summary.json" + +HEADERS = { + "User-Agent": "SaplingEduBot/1.0 (educational research tool; contact jackhe@honorsocietyofcinematicarts.org)", + "Accept": "text/html,application/xhtml+xml", +} + +# -- State ---------------------------------------------------------------------- + +_sem = None # initialized in main() +_errors: list[dict] = [] + +# -- HTTP ----------------------------------------------------------------------- + +async def fetch(client: httpx.AsyncClient, url: str, retries: int = 2) -> Optional[str]: + async with _sem: + for attempt in range(retries + 1): + try: + r = await client.get(url, headers=HEADERS, timeout=20.0, follow_redirects=True) + await asyncio.sleep(PAGE_DELAY) + if r.status_code == 200: + return r.text + if r.status_code == 404: + return None + # Other HTTP errors — retry + if attempt < retries: + await asyncio.sleep(5) + except (httpx.TimeoutException, httpx.ConnectError) as exc: + if attempt < retries: + await asyncio.sleep(5) + else: + _errors.append({"url": url, "error": str(exc)}) + return None + +# -- Listing page parser -------------------------------------------------------- + +def parse_listing(html: str, school: str) -> list[str]: + """Return absolute course-detail URLs from one listing page.""" + soup = BeautifulSoup(html, "html.parser") + # Match course slugs but exclude pure-numeric pagination slugs like /courses/2/ + pattern = re.compile(rf"^/academics/{re.escape(school)}/courses/(?!\d+$)([^/]+)/$") + seen: set[str] = set() + urls: list[str] = [] + for a in soup.find_all("a", href=pattern): + href = a["href"] + if href not in seen: + seen.add(href) + urls.append(BASE_URL + href) + return urls + +# -- Course detail page parser -------------------------------------------------- +# Confirmed BU course page structure: +#

Boston University Academics

<- skip (nav) +#

Introduction to Analysis of Algorithms

<- TITLE +#

CAS CS 330

<- COURSE CODE +#
Units:
4
<- CREDITS +#

Prerequisites: X. - Description. Effective Fall YYYY, ...

<- split on " - " +#

FALL 2025Schedule

<- semester labels (strip "Schedule") + +_CODE_RE = re.compile(r'\b([A-Z]{2,4}\s+[A-Z]{1,4}\s+[A-Z]?\d{3}[A-Z]?)\b') +_SEM_RE = re.compile(r'(FALL|SPRING|SPRG|SUMMER|SUMM|WINTER|WINT)\s+\d{4}', re.I) +# Strip BU Hub boilerplate from description tail +_HUB_RE = re.compile(r'\s*Effective (Fall|Spring|Summer|Winter)\s+\d{4}.*', re.S | re.I) +_BOILER = re.compile( + r'Boston University is accredited|javascript|cookie|copyright|privacy|' + r'listed here|guarantee|portal|register|accredited by|terms of use', re.I +) + + +def _extract_code_and_title(soup: BeautifulSoup, slug: str) -> tuple[str, str]: + # Title: second

(first is the site nav "Boston University Academics") + title = "" + h1s = soup.find_all("h1") + for h1 in h1s: + t = h1.get_text(strip=True) + if t and "Boston University" not in t: + title = t + break + + # Code: first

that matches a course code pattern + code = "" + for h2 in soup.find_all("h2"): + t = h2.get_text(strip=True) + if _CODE_RE.search(t): + code = t + break + + if not code: + code = slug.upper().replace("-", " ") + if not title: + pt = soup.find("title") + title = pt.get_text(strip=True).split("|")[0].strip() if pt else "" + + return code, title + + +def _extract_credits(soup: BeautifulSoup) -> Optional[int]: + # Structure:
Units:
4
+ for dt in soup.find_all("dt"): + if re.search(r'units?', dt.get_text(), re.I): + dd = dt.find_next_sibling("dd") + if dd: + m = re.search(r'\d+(?:\.\d+)?', dd.get_text()) + if m: + val = float(m.group()) + return int(val) if val == int(val) else val + return None + + +def _extract_prereq_and_desc(soup: BeautifulSoup) -> tuple[str, str]: + # BU embeds description in prereq paragraph: "Prerequisites: X. - Description. Effective..." + for p in soup.find_all("p"): + text = p.get_text(" ", strip=True) + if re.match(r'(Undergraduate\s+)?Prerequisites?:', text, re.I): + # Split on " - " to separate prereq list from course description + parts = re.split(r'\s+-\s+', text, maxsplit=1) + prereq = parts[0].strip() + desc = _HUB_RE.sub("", parts[1]).strip() if len(parts) > 1 else "" + return prereq, desc + # No prereq paragraph — look for standalone description paragraph + for p in soup.find_all("p"): + text = p.get_text(" ", strip=True) + if len(text) > 80 and not _BOILER.search(text): + return "", _HUB_RE.sub("", text).strip() + return "", "" + + +def _extract_schedule(soup: BeautifulSoup) -> tuple[list[str], list[str]]: + # Semester labels are in

FALL 2025Schedule

(text = "FALL 2025" + "Schedule") + semesters: list[str] = [] + for h4 in soup.find_all("h4"): + m = _SEM_RE.search(h4.get_text(strip=True)) + if m: + label = m.group(0).strip().title() + if label not in semesters: + semesters.append(label) + + # Instructors from all section tables + instructors: list[str] = [] + for table in soup.find_all("table"): + headers = [th.get_text(strip=True).lower() for th in table.find_all("th")] + if "instructor" not in headers: + continue + idx = headers.index("instructor") + for row in table.find_all("tr")[1:]: + cells = row.find_all("td") + if len(cells) > idx: + name = cells[idx].get_text(strip=True) + if name and name not in ("TBA", "Staff", "") and name not in instructors: + instructors.append(name) + + return semesters, instructors + + +def _extract_schedule(soup: BeautifulSoup) -> tuple[list[str], list[str]]: + semesters: list[str] = [] + instructors: list[str] = [] + sem_pattern = re.compile( + r'^(FALL|SPRING|SPRG|SUMMER|SUMM|SUM|WINTER|WINT)\s+\d{4}$', re.I + ) + for table in soup.find_all("table"): + headers = [th.get_text(strip=True) for th in table.find_all("th")] + for h in headers: + if sem_pattern.match(h.strip()) and h not in semesters: + semesters.append(h.strip().title()) + + # Instructor column + lower_headers = [h.lower() for h in headers] + if "instructor" in lower_headers: + idx = lower_headers.index("instructor") + for row in table.find_all("tr")[1:]: + cells = row.find_all("td") + if len(cells) > idx: + name = cells[idx].get_text(strip=True) + if name and name not in ("TBA", "Staff", "") and name not in instructors: + instructors.append(name) + return semesters, instructors + + +def parse_course(html: str, url: str, school: str) -> Optional[dict]: + soup = BeautifulSoup(html, "html.parser") + slug = url.rstrip("/").split("/")[-1] + + code, title = _extract_code_and_title(soup, slug) + if not code: + return None + + prerequisites, description = _extract_prereq_and_desc(soup) + credits = _extract_credits(soup) + semesters, instructors = _extract_schedule(soup) + + return { + "course_code": code, + "course_slug": slug, + "title": title, + "school": school, + "description": description, + "credits": credits, + "prerequisites": prerequisites, + "semester_offered": semesters, + "instructors": instructors, + "source_url": url, + "scraped_at": datetime.now(timezone.utc).isoformat(), + "semester_tag": SEMESTER_TAG, + } + +# -- School scraper ------------------------------------------------------------- + +async def scrape_school( + client: httpx.AsyncClient, + school: str, + seen: set[str], + on_batch=None, # callable(batch: list[dict]) — called every 100 courses +) -> list[dict]: + # Phase 1: collect all course URLs by walking paginated listing. + # BU wraps pagination (serves page 1 again past the last page) instead of + # 404-ing, so we stop as soon as a page yields zero URLs we haven't seen + # in this school's current crawl. + crawl_seen: set[str] = set() # URLs found in this school's listing walk + page = 1 + while True: + listing_url = ( + f"{BASE_URL}/academics/{school}/courses/" + if page == 1 + else f"{BASE_URL}/academics/{school}/courses/{page}/" + ) + html = await fetch(client, listing_url) + if not html: + break + urls = parse_listing(html, school) + fresh = [u for u in urls if u not in crawl_seen] + if not fresh: + break # pagination wrapped or past last page + crawl_seen.update(fresh) + page += 1 + if page > 250: # absolute safety limit + break + + new_urls = [u for u in crawl_seen if u not in seen] + print(f" [{school}] {page - 1} listing pages | {len(crawl_seen)} total | {len(new_urls)} new", flush=True) + + # Phase 2: fetch + parse course detail pages in batches of 100. + # on_batch is called after each batch so the caller can save a checkpoint. + BATCH = 100 + courses: list[dict] = [] + + async def _fetch_one(url: str) -> Optional[dict]: + html = await fetch(client, url) + if not html: + _errors.append({"url": url, "error": "empty response"}) + return None + result = parse_course(html, url, school) + if result is None: + _errors.append({"url": url, "error": "parse failed"}) + return result + + for i in range(0, len(new_urls), BATCH): + batch_results = await asyncio.gather(*[_fetch_one(u) for u in new_urls[i:i + BATCH]]) + batch_courses = [r for r in batch_results if r is not None] + courses.extend(batch_courses) + if on_batch: + on_batch(batch_courses) + + return courses + +# -- Main ----------------------------------------------------------------------- + +async def main() -> None: + global _sem + _sem = asyncio.Semaphore(CONCURRENCY) + + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + + # Resume: load whatever was scraped in a previous run + existing: dict[str, dict] = {} + if OUTPUT_FILE.exists(): + with open(OUTPUT_FILE, encoding="utf-8") as f: + for c in json.load(f): + existing[c["source_url"]] = c + print(f"Resuming — {len(existing)} courses already in {OUTPUT_FILE.name}") + + seen_urls: set[str] = set(existing.keys()) + all_courses: list[dict] = list(existing.values()) + start = datetime.now(timezone.utc) + + def save_checkpoint(batch: list[dict]) -> None: + all_courses.extend(batch) + seen_urls.update(c["source_url"] for c in batch) + with open(OUTPUT_FILE, "w", encoding="utf-8") as f: + json.dump(all_courses, f, indent=2, ensure_ascii=False) + print(f" checkpoint: {len(all_courses)} courses saved", flush=True) + + async with httpx.AsyncClient(limits=httpx.Limits(max_connections=20)) as client: + for school in SCHOOLS: + print(f"\n>> {school}", flush=True) + before = len(all_courses) + school_courses = await scrape_school(client, school, seen_urls, on_batch=save_checkpoint) + print(f" [{school}] +{len(all_courses) - before} -> total {len(all_courses)}", flush=True) + + elapsed = (datetime.now(timezone.utc) - start).total_seconds() + + summary = { + "total_courses": len(all_courses), + "total_errors": len(_errors), + "errors": _errors[:100], + "elapsed_seconds": round(elapsed), + "completed_at": datetime.now(timezone.utc).isoformat(), + "semester_tag": SEMESTER_TAG, + "output_file": str(OUTPUT_FILE), + } + with open(SUMMARY_FILE, "w", encoding="utf-8") as f: + json.dump(summary, f, indent=2) + + print(f"\n{'='*50}") + print(f"Done: {len(all_courses):,} courses | {len(_errors)} errors | {round(elapsed/60)} min") + print(f" Output -> {OUTPUT_FILE}") + print(f" Summary -> {SUMMARY_FILE}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/backend/services/rag_service.py b/backend/services/rag_service.py new file mode 100644 index 00000000..58b8005c --- /dev/null +++ b/backend/services/rag_service.py @@ -0,0 +1,62 @@ +""" +RAG retrieval service. + +Embeds a query with Gemini gemini-embedding-001 and calls the +match_course_chunks Supabase RPC to return the top-k semantically +similar course chunks. +""" + +import os + +from google import genai +from google.genai import types as genai_types + +from db.connection import rpc + +_client = genai.Client(api_key=os.getenv("GEMINI_API_KEY", "")) +_EMBED_MODEL = "gemini-embedding-001" +_OUTPUT_DIM = 768 + + +def _embed(text: str) -> list[float]: + resp = _client.models.embed_content( + model=_EMBED_MODEL, + contents=[text], + config=genai_types.EmbedContentConfig(output_dimensionality=_OUTPUT_DIM), + ) + return list(resp.embeddings[0].values) + + +def retrieve_chunks( + query: str, + course_id: str | None = None, + k: int = 5, + min_similarity: float = 0.55, +) -> list[dict]: + """Return up to k chunks similar to query, optionally filtered by course_id. + + Each result: {"course_id": str, "chunk_text": str, "similarity": float} + """ + try: + embedding = _embed(query) + params: dict = { + "query_embedding": embedding, + "match_count": k, + "filter_course_id": course_id, + } + rows = rpc("match_course_chunks", params) + return [r for r in rows if r.get("similarity", 0) >= min_similarity] + except Exception as e: + print(f"[RAG] retrieve_chunks failed: {e}") + return [] + + +def format_rag_context(chunks: list[dict]) -> str: + """Format retrieved chunks into a text block for prompt injection.""" + if not chunks: + return "" + lines = ["RETRIEVED COURSE CONTEXT (semantically relevant to this question):"] + for i, chunk in enumerate(chunks, 1): + sim = chunk.get("similarity", 0) + lines.append(f"\n[{i}] (relevance {sim:.2f})\n{chunk.get('chunk_text', '')}") + return "\n".join(lines) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 616a4cf6..029e011f 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -14330,7 +14330,6 @@ "version": "1.29.1", "resolved": "https://registry.npmjs.org/react-force-graph-3d/-/react-force-graph-3d-1.29.1.tgz", "integrity": "sha512-5Vp+PGpYnO+zLwgK2NvNqdXHvsWLrFzpDfJW1vUA1twjo9SPvXqfUYQrnRmAbD+K2tOxkZw1BkbH31l5b4TWHg==", - "license": "MIT", "dependencies": { "3d-force-graph": "^1.79", "prop-types": "15", diff --git a/frontend/src/components/KnowledgeGraph2D.tsx b/frontend/src/components/KnowledgeGraph2D.tsx index dfd3de1d..82822d7b 100644 --- a/frontend/src/components/KnowledgeGraph2D.tsx +++ b/frontend/src/components/KnowledgeGraph2D.tsx @@ -515,7 +515,7 @@ function KnowledgeGraph2DImpl({ {n.name} )} - {!n.is_subject_root && r > 10 && ( + {!n.is_subject_root && ( {([ - ["mastered", "var(--state-mastery)"], - ["learning", "var(--state-progress)"], - ["struggling", "var(--state-struggle)"], - ["unexplored", "var(--state-neutral)"], + ["Mastered", "var(--state-mastery)"], + ["Learning", "var(--state-progress)"], + ["Struggling", "var(--state-struggle)"], + ["Unexplored", "var(--state-neutral)"], ] as const).map(([t, color]) => (