From afc97df61da2f15702e81a223f8277a4138c7402 Mon Sep 17 00:00:00 2001 From: AndresL230 <190146319+AndresL230@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:53:27 -0400 Subject: [PATCH 1/3] feat(quiz): adaptive difficulty, /api/quiz/config, stable error envelope (#540) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Workstream A of the pre-revamp quiz repair batch (epic #537): A1 — 'adaptive' is a real difficulty. The route accepts it, the agent picks the per-question mix (ADAPTIVE MODE prompt rules), the attempt row records the request (CHECK extended by migration), and the response reports requested_difficulty + resolved_difficulty (mode, ties break harder) so the client can say what was actually chosen. A2 — GET /api/quiz/config is the single source of truth for selector values; the Pydantic model reads the same constants. Cap decision is measured, not guessed: scripts/bench_quiz_question_cap.py shows 15- and 20-question schemas are rejected outright by gemini-2.5-flash-lite ("too many states for serving"), so 10 is a hard ceiling. QuizPanel now builds its selects from the endpoint (static fallback mirrors it; the dead "15 questions" option is gone). A3 — every quiz-route 4xx/5xx returns {error: {code, message, detail?, request_id}} with codes in services/quiz_errors.py::QuizErrorCode; the legacy top-level detail key is kept so the current client still works. Co-Authored-By: Claude Fable 5 --- backend/agents/quiz.py | 9 + backend/db/e2e_checks/quiz.py | 2 +- ...4809_quiz_attempts_adaptive_difficulty.sql | 14 + backend/main.py | 47 ++- backend/models/__init__.py | 7 +- backend/routes/quiz.py | 128 ++++++-- backend/scripts/bench_quiz_question_cap.py | 128 ++++++++ backend/services/quiz_config.py | 54 +++ backend/services/quiz_errors.py | 107 ++++++ backend/tests/test_quiz_preflight_a.py | 309 ++++++++++++++++++ frontend/src/components/QuizPanel.test.tsx | 3 + frontend/src/components/QuizPanel.tsx | 62 +++- frontend/src/lib/api.ts | 13 +- 13 files changed, 844 insertions(+), 39 deletions(-) create mode 100644 backend/db/migrations/20260812204809_quiz_attempts_adaptive_difficulty.sql create mode 100644 backend/scripts/bench_quiz_question_cap.py create mode 100644 backend/services/quiz_config.py create mode 100644 backend/services/quiz_errors.py create mode 100644 backend/tests/test_quiz_preflight_a.py diff --git a/backend/agents/quiz.py b/backend/agents/quiz.py index 01252cd9..2fd28c24 100644 --- a/backend/agents/quiz.py +++ b/backend/agents/quiz.py @@ -113,6 +113,15 @@ class Quiz(BaseModel): "- Don't drop high-mastery, recently-reviewed concepts entirely; " " include 1 question on a strong-and-fresh concept to keep the " " quiz from feeling punishing.\n\n" + "ADAPTIVE MODE (#540): when the user message says the quiz is in " + "adaptive mode, there is no user-requested difficulty — you choose " + "each question's difficulty yourself. Base the mix on mastery and " + "`recent_attempts.accuracy`: struggling/low-accuracy concepts get " + "easy-leaning questions, strong/high-accuracy ones get hard-leaning " + "questions; with no history at all, center the mix on medium. The " + "±1-step limits below do not apply in adaptive mode, but every " + "question still carries a concrete easy|medium|hard difficulty — " + "'adaptive' is never a per-question value.\n\n" "Adaptive-difficulty rules (use `recent_attempts.accuracy`):\n" "- If the most recent 2-3 attempts on this concept averaged < " " 0.5 accuracy, drop the difficulty mix one step from what the " diff --git a/backend/db/e2e_checks/quiz.py b/backend/db/e2e_checks/quiz.py index 10cff8dc..66f050ef 100644 --- a/backend/db/e2e_checks/quiz.py +++ b/backend/db/e2e_checks/quiz.py @@ -32,7 +32,7 @@ def run() -> None: # ── 1. Generate a quiz to obtain a real quiz_id ─────────────────────────── # GenerateQuizBody: {user_id, concept_node_id, num_questions, difficulty, use_shared_context} - # difficulty must be in {"easy", "medium", "hard"} (CHECK constraint from 0025). + # difficulty ∈ {"easy","medium","hard","adaptive"} (0025 CHECK + the #540 adaptive migration). gen_r = client.post( "/api/quiz/generate", json={ diff --git a/backend/db/migrations/20260812204809_quiz_attempts_adaptive_difficulty.sql b/backend/db/migrations/20260812204809_quiz_attempts_adaptive_difficulty.sql new file mode 100644 index 00000000..eab193f2 --- /dev/null +++ b/backend/db/migrations/20260812204809_quiz_attempts_adaptive_difficulty.sql @@ -0,0 +1,14 @@ +-- #540 A1: 'adaptive' becomes a real request-side difficulty. The attempt row +-- records what the student asked for (the response's resolved_difficulty +-- reports what generation actually produced), so the 0025 CHECK +-- (easy|medium|hard) must admit 'adaptive'. +-- +-- Idempotent: DROP IF EXISTS + re-ADD. The constraint name is the PG default +-- for 0025's inline CHECK on quiz_attempts(difficulty). + +ALTER TABLE quiz_attempts + DROP CONSTRAINT IF EXISTS quiz_attempts_difficulty_check; + +ALTER TABLE quiz_attempts + ADD CONSTRAINT quiz_attempts_difficulty_check + CHECK (difficulty IN ('easy', 'medium', 'hard', 'adaptive')); diff --git a/backend/main.py b/backend/main.py index 41497a02..0e756c01 100644 --- a/backend/main.py +++ b/backend/main.py @@ -26,6 +26,7 @@ from routes.admin import router as admin_router from routes.admin_analytics import router as admin_analytics_router from routes.newsletter import router as newsletter_router +from services import quiz_config, quiz_errors from services.logfire_scrubber import EXTRA_PATTERNS, scrub_value from services.request_context import RequestIDMiddleware, current_request_id from services.storage_service import ( @@ -186,9 +187,22 @@ def _drop_request_arguments(_request, _attributes): @app.exception_handler(StarletteHTTPException) async def http_exception_handler(request: Request, exc: StarletteHTTPException): rid = getattr(request.state, "request_id", None) or current_request_id() + if quiz_errors.is_quiz_path(request.url.path): + # #540 A3: quiz routes speak the coded envelope. QuizAPIError raise + # sites carry a precise code; plain HTTPExceptions (require_self, + # etc.) fall back to a status-derived one. + content = quiz_errors.quiz_error_body( + exc.status_code, + exc.detail, + rid, + code=getattr(exc, "code", None), + machine_detail=getattr(exc, "machine_detail", None), + ) + else: + content = {"detail": exc.detail, "request_id": rid} return JSONResponse( status_code=exc.status_code, - content={"detail": exc.detail, "request_id": rid}, + content=content, headers={"X-Request-ID": rid} if rid else {}, ) @@ -196,9 +210,29 @@ async def http_exception_handler(request: Request, exc: StarletteHTTPException): @app.exception_handler(RequestValidationError) async def validation_exception_handler(request: Request, exc: RequestValidationError): rid = getattr(request.state, "request_id", None) or current_request_id() + if quiz_errors.is_quiz_path(request.url.path): + # A bad num_questions is the one validation failure the old UI could + # actually produce (it offered 15 against le=10) — give it a precise + # code + a sentence; everything else is generic QUIZ_VALIDATION_ERROR. + locs = {str(part) for e in exc.errors() for part in e.get("loc", ())} + if "num_questions" in locs: + code = quiz_errors.QuizErrorCode.QUIZ_COUNT_OUT_OF_RANGE + message = ( + f"Quizzes can have between {quiz_config.QUIZ_MIN_QUESTIONS} " + f"and {quiz_config.QUIZ_MAX_QUESTIONS} questions." + ) + else: + code = quiz_errors.QuizErrorCode.QUIZ_VALIDATION_ERROR + message = "That quiz request wasn't valid — please try again." + content = quiz_errors.quiz_error_body( + 422, exc.errors(), rid, code=code, message=message, + machine_detail=exc.errors(), + ) + else: + content = {"detail": exc.errors(), "request_id": rid} return JSONResponse( status_code=422, - content={"detail": exc.errors(), "request_id": rid}, + content=content, headers={"X-Request-ID": rid} if rid else {}, ) @@ -207,9 +241,16 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE async def unhandled_exception_handler(request: Request, exc: Exception): logging.getLogger("main").exception("Unhandled exception") rid = getattr(request.state, "request_id", None) or current_request_id() + if quiz_errors.is_quiz_path(request.url.path): + content = quiz_errors.quiz_error_body( + 500, "Internal server error.", rid, + code=quiz_errors.QuizErrorCode.QUIZ_INTERNAL_ERROR, + ) + else: + content = {"detail": "Internal server error.", "request_id": rid} return JSONResponse( status_code=500, - content={"detail": "Internal server error.", "request_id": rid}, + content=content, headers={"X-Request-ID": rid} if rid else {}, ) diff --git a/backend/models/__init__.py b/backend/models/__init__.py index d91b01b5..9faa4d68 100644 --- a/backend/models/__init__.py +++ b/backend/models/__init__.py @@ -1,6 +1,8 @@ from typing import Optional, Union, List, Literal from pydantic import BaseModel, Field +from services.quiz_config import QUIZ_MIN_QUESTIONS, QUIZ_MAX_QUESTIONS + # ── Learn ───────────────────────────────────────────────────────────────────── @@ -46,7 +48,10 @@ class ActionBody(BaseModel): class GenerateQuizBody(BaseModel): user_id: str = "user_andres" concept_node_id: str - num_questions: int = Field(default=5, ge=1, le=10) + # Bounds come from services/quiz_config.py — the same constants + # GET /api/quiz/config serves, so the client can't offer a value + # this model rejects (#540 A2). + num_questions: int = Field(default=5, ge=QUIZ_MIN_QUESTIONS, le=QUIZ_MAX_QUESTIONS) difficulty: str = "medium" use_shared_context: bool = True # Mirrors the Learn-route fast/smart toggle so quiz generation has diff --git a/backend/routes/quiz.py b/backend/routes/quiz.py index 2dbb5364..2f880450 100644 --- a/backend/routes/quiz.py +++ b/backend/routes/quiz.py @@ -20,6 +20,11 @@ from routes.learn import _get_catalog_chunk from services import events_service from services.auth_guard import require_self +from services.quiz_config import ( + REQUESTED_DIFFICULTIES, + quiz_config_payload, +) +from services.quiz_errors import QuizAPIError, QuizErrorCode from services.profiles import get_display_name from services.encryption import encrypt_json, decrypt_json_column from services.graph_service import apply_graph_update @@ -35,8 +40,10 @@ PROMPTS_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "prompts") -# quiz_attempts.difficulty CHECK enum (0025). -VALID_DIFFICULTIES = {"easy", "medium", "hard"} +# Request-side difficulties live in services/quiz_config.py (#540 A2): +# the concrete trio matches the quiz_attempts.difficulty CHECK (0025, +# extended with 'adaptive' by the #540 migration); 'adaptive' hands the +# per-question mix decision to the agent (A1). def _load_prompt(name: str) -> str: @@ -62,6 +69,28 @@ def _load_prompt(name: str) -> str: _OPTION_LABELS = ["A", "B", "C", "D", "E", "F"] +# Rank order for tie-breaking the overall difficulty report. +_DIFFICULTY_RANK = {"easy": 0, "medium": 1, "hard": 2} + + +def _resolved_difficulty(wire_questions: list[dict]) -> str: + """The overall difficulty generation actually produced (#540 A1). + + Mode of the per-question difficulties; ties break to the harder value + so the report never understates what the student is about to face. + Defaults to 'medium' when nothing usable is present (can't happen for + agent output — QuizQuestion.difficulty is a concrete Literal — but + this also runs on stored legacy rows). + """ + counts: dict[str, int] = {} + for q in wire_questions: + d = q.get("difficulty") + if d in _DIFFICULTY_RANK: + counts[d] = counts.get(d, 0) + 1 + if not counts: + return "medium" + return max(counts, key=lambda d: (counts[d], _DIFFICULTY_RANK[d])) + def _agent_question_to_wire(q: QuizQuestion, qid: int) -> dict | None: """Map an agent QuizQuestion to the legacy wire-format dict, or @@ -237,8 +266,24 @@ async def _quiz_via_agent( # Keep this message routing-only; the workflow + adaptive rules # live in the system prompt. We just hand the agent the inputs it # needs and trust the prompt to drive tool calls. + if difficulty == "adaptive": + # #540 A1: no target difficulty — the agent picks the whole mix + # from mastery + recent accuracy (ADAPTIVE MODE in the system + # prompt). Every emitted question still carries a concrete + # easy|medium|hard; the route reports the overall pick back to + # the client as `resolved_difficulty`. + difficulty_clause = ( + f"Generate {num_questions} questions in ADAPTIVE MODE: you " + f"choose each question's difficulty (easy, medium, or hard) " + f"from the student's mastery and recent accuracy, per the " + f"adaptive-mode rules in your system prompt." + ) + else: + difficulty_clause = ( + f"Generate {num_questions} {difficulty} questions for the student." + ) routing_msg = ( - f"Generate {num_questions} {difficulty} questions for the student. " + f"{difficulty_clause} " f"The target concept is '{concept_name}' " f"(concept_node_id={concept_node_id}). Follow the workflow in your " f"system prompt; pass concept_node_id='{concept_node_id}' to " @@ -290,23 +335,40 @@ async def _quiz_via_agent( ) return wire_questions +@router.get("/config") +def quiz_config(): + """Selector options for the quiz UI (#540 A2). Single source of truth: + the same constants bound the Pydantic request model, so a client that + builds its selects from this payload can never send a value the route + rejects. No user data, no auth needed.""" + return quiz_config_payload() + + @router.post("/generate") async def generate_quiz(body: GenerateQuizBody, request: Request): require_self(body.user_id, request) - # quiz_attempts.difficulty is CHECK-constrained (0025); reject drift before - # we run the agent or write an attempt row. - if body.difficulty not in VALID_DIFFICULTIES: - raise HTTPException( + # The concrete trio is CHECK-constrained on quiz_attempts (0025 + + # the #540 'adaptive' extension); reject drift before we run the + # agent or write an attempt row. + if body.difficulty not in REQUESTED_DIFFICULTIES: + raise QuizAPIError( status_code=400, - detail=f"Invalid difficulty '{body.difficulty}'. " - f"Must be one of {sorted(VALID_DIFFICULTIES)}.", + code=QuizErrorCode.QUIZ_DIFFICULTY_INVALID, + message=( + "That difficulty isn't available. Choose easy, medium, " + "hard, or adaptive." + ), ) node_rows = table("graph_nodes").select( "*", filters={"id": f"eq.{body.concept_node_id}", "user_id": f"eq.{body.user_id}"}, ) if not node_rows: - raise HTTPException(status_code=404, detail="Concept node not found") + raise QuizAPIError( + status_code=404, + code=QuizErrorCode.QUIZ_CONCEPT_NOT_FOUND, + message="We couldn't find that concept in your knowledge graph.", + ) node = node_rows[0] course_id = node.get("course_id") or None concept_name = node.get("concept_name") or "" @@ -339,15 +401,17 @@ async def generate_quiz(body: GenerateQuizBody, request: Request): # The raw-Gemini legacy fallback was retired in #145; degrade to 502 # rather than serving a quiz from a second LLM path. logger.warning("Quiz agent guardrails tripped; returning 502", exc_info=e) - raise HTTPException( + raise QuizAPIError( status_code=502, - detail="Quiz generation is temporarily unavailable. Please try again.", + code=QuizErrorCode.QUIZ_GENERATION_FAILED, + message="Quiz generation is temporarily unavailable. Please try again.", ) from e except Exception as e: logger.exception("Unexpected quiz-agent failure; returning 502") - raise HTTPException( + raise QuizAPIError( status_code=502, - detail="Quiz generation is temporarily unavailable. Please try again.", + code=QuizErrorCode.QUIZ_GENERATION_FAILED, + message="Quiz generation is temporarily unavailable. Please try again.", ) from e quiz_id = str(uuid.uuid4()) @@ -372,14 +436,28 @@ async def generate_quiz(body: GenerateQuizBody, request: Request): "difficulty": body.difficulty, }, ) - return {"quiz_id": quiz_id, "questions": questions} + # #540 A1: echo what generation actually chose. requested_difficulty + # is what the student asked for (may be 'adaptive'); + # resolved_difficulty is the overall mix the agent produced (always + # concrete) — so the client can say "we picked hard for you" instead + # of repeating the request back. + return { + "quiz_id": quiz_id, + "questions": questions, + "requested_difficulty": body.difficulty, + "resolved_difficulty": _resolved_difficulty(questions), + } @router.post("/submit") def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request: Request): attempt_rows = table("quiz_attempts").select("*", filters={"id": f"eq.{body.quiz_id}"}) if not attempt_rows: - raise HTTPException(status_code=404, detail="Quiz not found") + raise QuizAPIError( + status_code=404, + code=QuizErrorCode.QUIZ_ATTEMPT_NOT_FOUND, + message="We couldn't find that quiz.", + ) attempt = attempt_rows[0] user_id = attempt["user_id"] @@ -395,8 +473,10 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request # 200: quiz_attempts stores no mastery_before/after, so faithfully # reconstructing the first response would need a migration. if attempt.get("completed_at"): - raise HTTPException( - status_code=409, detail="Quiz attempt has already been submitted" + raise QuizAPIError( + status_code=409, + code=QuizErrorCode.QUIZ_ATTEMPT_ALREADY_COMPLETED, + message="This quiz has already been submitted.", ) # The read above is only the fast path — two CONCURRENT submits (a # double-click on the final submit) would both pass it. The atomic claim @@ -410,8 +490,10 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request filters={"id": f"eq.{body.quiz_id}", "completed_at": "is.null"}, ) if not claimed: - raise HTTPException( - status_code=409, detail="Quiz attempt has already been submitted" + raise QuizAPIError( + status_code=409, + code=QuizErrorCode.QUIZ_ATTEMPT_ALREADY_COMPLETED, + message="This quiz has already been submitted.", ) concept_node_id = attempt["concept_node_id"] @@ -450,7 +532,11 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request filters={"id": f"eq.{concept_node_id}", "user_id": f"eq.{user_id}"}, ) if not node_rows: - raise HTTPException(status_code=404, detail="Concept node not found") + raise QuizAPIError( + status_code=404, + code=QuizErrorCode.QUIZ_CONCEPT_NOT_FOUND, + message="We couldn't find that concept in your knowledge graph.", + ) node = node_rows[0] mastery_before = node["mastery_score"] mastery_after = max(0.0, min(1.0, mastery_before + (score * 0.03) - ((total - score) * 0.02))) diff --git a/backend/scripts/bench_quiz_question_cap.py b/backend/scripts/bench_quiz_question_cap.py new file mode 100644 index 00000000..89989b0f --- /dev/null +++ b/backend/scripts/bench_quiz_question_cap.py @@ -0,0 +1,128 @@ +"""Offline benchmark behind the #540 A2 question-cap decision. + +Measures wall-clock latency + token usage of quiz generation on +gemini-2.5-flash-lite at schema caps 10 / 15 / 20, using structurally +identical copies of agents/quiz.py's Quiz schema (same QuizQuestion +fields, no tools). Two questions it answers: + +1. Does a >10 cap even SERVE? agents/quiz.py:77-80 records that the + original 20-cap schema tripped Gemini's constrained-decoding + "too many states for serving" on flash-lite and had to be cut to 10. +2. If it serves, what do 15- and 20-question generations cost in + latency and tokens versus 10? + +Run from backend/ with GEMINI_API_KEY in .env: + + venv/bin/python -m scripts.bench_quiz_question_cap + +Offline benchmark only — never import from application code (same rule +as scripts/_raw_gemini.py). Results are recorded in the RESULTS block +below and referenced by services/quiz_config.py's cap comment. + +RESULTS (2026-08-12, gemini-2.5-flash-lite, 2 runs per cap): + cap=10: 4.6s / 5.6s (median 5.1s), in=260 out=1445-1557 tokens, + delivered 10/10 both runs. + cap=15: BOTH runs rejected before generation — HTTP 400 + INVALID_ARGUMENT "schema produces a constraint that has too + many states for serving". + cap=20: same 400 rejection, both runs. + +Conclusion: 10 is the hard ceiling for a single structured call on +flash-lite — 15+ is not slower, it is unservable. Raising the cap means +a model-tier change or batched generation, which is #537 revamp scope. +""" + +import asyncio +import statistics +import sys +import time +from typing import Literal + +from dotenv import load_dotenv + +load_dotenv() + +from pydantic import BaseModel, Field # noqa: E402 +from pydantic_ai import Agent # noqa: E402 + + +class BenchQuizQuestion(BaseModel): + """Structural copy of agents/quiz.py::QuizQuestion (kept in sync by hand; + this is a bench, not a contract).""" + + question: str + type: Literal["multiple_choice"] + difficulty: Literal["easy", "medium", "hard"] + options: list[str] = Field(min_length=4, max_length=4) + correct_answer: str + explanation: str + concept: str + + +PROMPT = ( + "You generate multiple-choice quizzes. 4 options each, exactly one " + "correct; correct_answer must appear verbatim in options; explanation " + "is 1-3 sentences." +) + +CONCEPT_MSG = ( + "Generate {n} medium questions on the concept 'gradient descent' for an " + "undergraduate machine-learning student. The concept field of every " + "question must be 'gradient descent'." +) + +RUNS_PER_CAP = 2 +CAPS = (10, 15, 20) +MODEL = "gemini-2.5-flash-lite" + + +def _agent_for_cap(cap: int) -> Agent: + class BenchQuiz(BaseModel): + questions: list[BenchQuizQuestion] = Field(min_length=1, max_length=cap) + + return Agent( + model=f"google-gla:{MODEL}", + output_type=BenchQuiz, + system_prompt=PROMPT, + ) + + +async def bench_cap(cap: int) -> None: + agent = _agent_for_cap(cap) + latencies, in_tokens, out_tokens, counts = [], [], [], [] + for run in range(RUNS_PER_CAP): + t0 = time.perf_counter() + try: + result = await agent.run(CONCEPT_MSG.format(n=cap)) + except Exception as e: # serving rejection is a *result* here, not a bug + print(f"cap={cap} run={run + 1}: FAILED — {type(e).__name__}: {e}") + continue + dt = time.perf_counter() - t0 + usage = result.usage() + latencies.append(dt) + in_tokens.append(usage.input_tokens or 0) + out_tokens.append(usage.output_tokens or 0) + counts.append(len(result.output.questions)) + print( + f"cap={cap} run={run + 1}: {dt:.1f}s, " + f"in={usage.input_tokens} out={usage.output_tokens} " + f"questions={len(result.output.questions)}" + ) + if latencies: + print( + f"cap={cap} SUMMARY: median {statistics.median(latencies):.1f}s, " + f"mean in={statistics.mean(in_tokens):.0f} " + f"out={statistics.mean(out_tokens):.0f} " + f"delivered={counts}" + ) + else: + print(f"cap={cap} SUMMARY: no successful runs (schema likely rejected)") + + +async def main() -> None: + for cap in CAPS: + await bench_cap(cap) + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) diff --git a/backend/services/quiz_config.py b/backend/services/quiz_config.py new file mode 100644 index 00000000..2c01acf4 --- /dev/null +++ b/backend/services/quiz_config.py @@ -0,0 +1,54 @@ +"""Quiz selector configuration — the single source of truth (#540 A2). + +Both the Pydantic request model (`models.GenerateQuizBody`) and the +`GET /api/quiz/config` endpoint read these constants, so the client can +build its selectors from the endpoint and never again offer a value the +route rejects (the pre-#540 UI offered "15 questions" against a le=10 +cap, and an "Adaptive" difficulty the route 400'd). + +Standalone constants module: no imports from models/routes/services so +it can be imported from anywhere without cycles. +""" + +QUIZ_MIN_QUESTIONS = 1 + +# The cap is a Gemini structured-output constraint before it is a product +# choice. Measured 2026-08-12 (#540 A2, scripts/bench_quiz_question_cap.py): +# on gemini-2.5-flash-lite a 10-question schema serves at ~5.1s median +# (~260 in / ~1500 out tokens), while 15- and 20-question schemas are +# REJECTED outright — HTTP 400 INVALID_ARGUMENT "too many states for +# serving" — before any generation happens. 10 is therefore the hard +# ceiling for a single structured call on the quiz task's model tier +# (matches agents/quiz.py:77-80); raising it requires a model-tier change +# or batched generation, which is #537 revamp scope. +QUIZ_MAX_QUESTIONS = 10 + +# The selector values the product offers (config endpoint → client). +# Must all sit within [QUIZ_MIN_QUESTIONS, QUIZ_MAX_QUESTIONS]. +QUIZ_NUM_QUESTION_OPTIONS = (3, 5, 10) + +# Concrete difficulties — what the agent can emit per question and what +# quiz_attempts.difficulty accepted before the adaptive migration. +CONCRETE_DIFFICULTIES = ("easy", "medium", "hard") + +# Request-side difficulties: 'adaptive' asks the agent to pick the +# per-question mix itself (A1); the response reports what it chose via +# `resolved_difficulty`. +REQUESTED_DIFFICULTIES = CONCRETE_DIFFICULTIES + ("adaptive",) + +# MCQ-only today — mirrors agents/quiz.py::QuizQuestionType. Grows when +# the #537 revamp adds real short-answer grading. +QUIZ_QUESTION_TYPES = ("mcq",) + + +def quiz_config_payload() -> dict: + """The GET /api/quiz/config response body.""" + return { + "num_questions": { + "min": QUIZ_MIN_QUESTIONS, + "max": QUIZ_MAX_QUESTIONS, + "options": list(QUIZ_NUM_QUESTION_OPTIONS), + }, + "difficulties": list(REQUESTED_DIFFICULTIES), + "question_types": list(QUIZ_QUESTION_TYPES), + } diff --git a/backend/services/quiz_errors.py b/backend/services/quiz_errors.py new file mode 100644 index 00000000..94bf4159 --- /dev/null +++ b/backend/services/quiz_errors.py @@ -0,0 +1,107 @@ +"""Stable machine-readable error contract for the quiz routes (#540 A3). + +Every 4xx/5xx under /api/quiz/* returns: + + { + "error": { + "code": "", # stable machine string for the client + "message": "", + "detail": ..., # optional machine detail (e.g. Pydantic errors) + "request_id": "" # correlation for support; never inside message + }, + "detail": ..., # legacy key — current QuizPanel reads data?.detail + "request_id": "" + } + +The envelope is applied by main.py's exception handlers, scoped to quiz +paths only (`is_quiz_path`), so every raise inside the routes — including +shared dependencies like require_self — comes out enveloped without each +raise site needing to know about the format. Raise `QuizAPIError` to attach +a precise code; plain HTTPExceptions fall back to a status-derived code. + +The frontend maps codes from this one enum; add codes here, never inline. +""" + +from enum import Enum + +from fastapi import HTTPException + + +class QuizErrorCode(str, Enum): + QUIZ_DIFFICULTY_INVALID = "QUIZ_DIFFICULTY_INVALID" + QUIZ_COUNT_OUT_OF_RANGE = "QUIZ_COUNT_OUT_OF_RANGE" + QUIZ_VALIDATION_ERROR = "QUIZ_VALIDATION_ERROR" + QUIZ_CONCEPT_NOT_FOUND = "QUIZ_CONCEPT_NOT_FOUND" + QUIZ_ATTEMPT_NOT_FOUND = "QUIZ_ATTEMPT_NOT_FOUND" + QUIZ_ATTEMPT_ALREADY_COMPLETED = "QUIZ_ATTEMPT_ALREADY_COMPLETED" + QUIZ_NOT_AUTHORIZED = "QUIZ_NOT_AUTHORIZED" + QUIZ_GENERATION_FAILED = "QUIZ_GENERATION_FAILED" + QUIZ_INTERNAL_ERROR = "QUIZ_INTERNAL_ERROR" + + +# Fallback code when an ordinary HTTPException (no explicit code) escapes a +# quiz route — e.g. require_self's 401/403. +_STATUS_FALLBACK: dict[int, QuizErrorCode] = { + 401: QuizErrorCode.QUIZ_NOT_AUTHORIZED, + 403: QuizErrorCode.QUIZ_NOT_AUTHORIZED, + 404: QuizErrorCode.QUIZ_ATTEMPT_NOT_FOUND, + 409: QuizErrorCode.QUIZ_ATTEMPT_ALREADY_COMPLETED, + 422: QuizErrorCode.QUIZ_VALIDATION_ERROR, + 502: QuizErrorCode.QUIZ_GENERATION_FAILED, +} + + +class QuizAPIError(HTTPException): + """HTTPException carrying a stable code + student-readable message. + + `detail` stays the legacy-compatible string (the message), so clients + reading `data?.detail` see the same sentence as `error.message`. + """ + + def __init__( + self, + status_code: int, + code: QuizErrorCode, + message: str, + machine_detail=None, + ): + super().__init__(status_code=status_code, detail=message) + self.code = code + self.machine_detail = machine_detail + + +def is_quiz_path(path: str) -> bool: + # Exact prefix match: '/api/quizzes' or '/api/quiz-x' must NOT envelope. + return path == "/api/quiz" or path.startswith("/api/quiz/") + + +def quiz_error_body( + status_code: int, + legacy_detail, + request_id: str | None, + code: QuizErrorCode | None = None, + message: str | None = None, + machine_detail=None, +) -> dict: + """Build the enveloped payload; keeps the legacy top-level keys.""" + resolved_code = code or _STATUS_FALLBACK.get( + status_code, + QuizErrorCode.QUIZ_INTERNAL_ERROR, + ) + resolved_message = message or ( + legacy_detail + if isinstance(legacy_detail, str) + else "Something went wrong with this quiz request." + ) + error: dict = { + "code": resolved_code.value, + "message": resolved_message, + "request_id": request_id, + } + if machine_detail is not None: + error["detail"] = machine_detail + return { + "error": error, + "detail": legacy_detail, + "request_id": request_id, + } diff --git a/backend/tests/test_quiz_preflight_a.py b/backend/tests/test_quiz_preflight_a.py new file mode 100644 index 00000000..0d5792bc --- /dev/null +++ b/backend/tests/test_quiz_preflight_a.py @@ -0,0 +1,309 @@ +""" +Workstream A of the pre-revamp quiz repair batch (#540, epic #537). + +Covers: +- GET /api/quiz/config — single source of truth for selector options +- POST /api/quiz/generate accepts difficulty='adaptive' (A1) +- resolved_difficulty / requested_difficulty echo what generation chose +- Stable error envelope { error: { code, message, detail?, request_id } } + on every quiz-route 4xx/5xx, with the legacy `detail` key kept so the + current QuizPanel client keeps working (A3) +""" +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch +from fastapi.testclient import TestClient + +from main import app +from agents.quiz import Quiz, QuizQuestion + +client = TestClient(app) + + +def _question(difficulty: str, n: int = 1) -> QuizQuestion: + return QuizQuestion( + question=f"Q{n}?", + type="multiple_choice", + difficulty=difficulty, + options=[f"a{n}", f"b{n}", f"c{n}", f"d{n}"], + correct_answer=f"a{n}", + explanation="x", + concept="X", + ) + + +def _quiz(*difficulties: str) -> Quiz: + return Quiz(questions=[_question(d, i + 1) for i, d in enumerate(difficulties)]) + + +def _generate_table_factory(): + def factory(name): + mock = MagicMock() + if name == "graph_nodes": + mock.select.return_value = [{ + "id": "node1", + "course_id": "course1", + "concept_name": "Loops", + "mastery_score": 0.5, + }] + elif name == "quiz_attempts": + mock.insert.return_value = [{"id": "quiz-generated"}] + else: + mock.select.return_value = [] + mock.insert.return_value = [] + return mock + + return factory + + +def _generate(body_extra: dict) -> object: + return client.post("/api/quiz/generate", json={ + "user_id": "user_andres", + "concept_node_id": "node1", + "num_questions": 3, + "use_shared_context": False, + **body_extra, + }) + + +# ── GET /api/quiz/config (A2) ──────────────────────────────────────────────── + + +class TestQuizConfigEndpoint: + """One source of truth: the client builds its selectors from this payload + and can never again offer a value the route rejects.""" + + def test_config_returns_selector_options(self): + r = client.get("/api/quiz/config") + assert r.status_code == 200 + data = r.json() + nq = data["num_questions"] + assert isinstance(nq["min"], int) + assert isinstance(nq["max"], int) + assert isinstance(nq["options"], list) and nq["options"] + assert all(nq["min"] <= v <= nq["max"] for v in nq["options"]) + assert data["difficulties"] == ["easy", "medium", "hard", "adaptive"] + assert data["question_types"] == ["mcq"] + + def test_config_matches_pydantic_model_bounds(self): + """The Pydantic cap and the config endpoint must read the same named + constant — if they drift this fails.""" + from models import GenerateQuizBody + + field = GenerateQuizBody.model_fields["num_questions"] + ge = next(m.ge for m in field.metadata if hasattr(m, "ge")) + le = next(m.le for m in field.metadata if hasattr(m, "le")) + data = client.get("/api/quiz/config").json() + assert data["num_questions"]["min"] == ge + assert data["num_questions"]["max"] == le + + def test_config_matches_named_constants(self): + from services.quiz_config import ( + QUIZ_MIN_QUESTIONS, + QUIZ_MAX_QUESTIONS, + QUIZ_NUM_QUESTION_OPTIONS, + ) + + data = client.get("/api/quiz/config").json() + assert data["num_questions"]["min"] == QUIZ_MIN_QUESTIONS + assert data["num_questions"]["max"] == QUIZ_MAX_QUESTIONS + assert data["num_questions"]["options"] == list(QUIZ_NUM_QUESTION_OPTIONS) + + +# ── difficulty='adaptive' (A1) ─────────────────────────────────────────────── + + +class TestGenerateAdaptiveDifficulty: + def test_adaptive_generates_and_echoes_resolved_difficulty(self): + run_mock = AsyncMock( + return_value=SimpleNamespace(output=_quiz("medium", "hard", "hard")) + ) + captured = {} + + def factory(name): + mock = MagicMock() + if name == "graph_nodes": + mock.select.return_value = [{ + "id": "node1", + "course_id": "course1", + "concept_name": "Loops", + "mastery_score": 0.5, + }] + elif name == "quiz_attempts": + def _capture(payload): + captured["payload"] = payload + return [{"id": payload["id"]}] + mock.insert.side_effect = _capture + else: + mock.select.return_value = [] + return mock + + with ( + patch("routes.quiz.table", side_effect=factory), + patch("routes.quiz.quiz_agent.run", new=run_mock), + ): + r = _generate({"difficulty": "adaptive"}) + + assert r.status_code == 200 + data = r.json() + assert data["requested_difficulty"] == "adaptive" + # Mode of (medium, hard, hard) → hard. + assert data["resolved_difficulty"] == "hard" + # Per-question difficulty stays on each wire question. + assert [q["difficulty"] for q in data["questions"]] == [ + "medium", "hard", "hard", + ] + # The attempt row records what the student asked for. + assert captured["payload"]["difficulty"] == "adaptive" + + def test_adaptive_routing_message_instructs_agent(self): + """The agent must be told to pick the mix itself — and that every + emitted question still carries a concrete difficulty.""" + run_mock = AsyncMock( + return_value=SimpleNamespace(output=_quiz("easy", "medium", "medium")) + ) + with ( + patch("routes.quiz.table", side_effect=_generate_table_factory()), + patch("routes.quiz.quiz_agent.run", new=run_mock), + ): + r = _generate({"difficulty": "adaptive"}) + assert r.status_code == 200 + msg = run_mock.call_args[0][0] + assert "adaptive" in msg.lower() + # The literal string 'adaptive' must not be requested as a + # per-question difficulty value (the output schema is concrete). + assert "3 adaptive questions" not in msg + + def test_concrete_request_also_reports_resolved_difficulty(self): + """Even a concrete request echoes what generation actually chose — + the agent may legitimately shift the mix ±1 step.""" + run_mock = AsyncMock( + return_value=SimpleNamespace(output=_quiz("easy", "easy", "medium")) + ) + with ( + patch("routes.quiz.table", side_effect=_generate_table_factory()), + patch("routes.quiz.quiz_agent.run", new=run_mock), + ): + r = _generate({"difficulty": "medium"}) + assert r.status_code == 200 + data = r.json() + assert data["requested_difficulty"] == "medium" + assert data["resolved_difficulty"] == "easy" + + def test_resolved_difficulty_tie_breaks_harder(self): + from routes.quiz import _resolved_difficulty + + assert _resolved_difficulty([{"difficulty": "easy"}, + {"difficulty": "hard"}]) == "hard" + assert _resolved_difficulty([{"difficulty": "medium"}]) == "medium" + assert _resolved_difficulty([{"difficulty": "easy"}, + {"difficulty": "easy"}, + {"difficulty": "medium"}]) == "easy" + # Unknown/missing difficulty values are ignored, not fatal. + assert _resolved_difficulty([{}]) == "medium" + + +# ── Stable error envelope (A3) ─────────────────────────────────────────────── + + +def _assert_envelope(r, status: int, code: str): + assert r.status_code == status + body = r.json() + err = body["error"] + assert err["code"] == code + assert isinstance(err["message"], str) and err["message"] + # request_id rides in the payload for support, not inside message. + assert "request_id" in err + assert err["request_id"] not in err["message"] + # Legacy key kept so the current client's `data?.detail` reads work. + assert "detail" in body + return body + + +class TestQuizErrorEnvelope: + def test_invalid_difficulty_400(self): + agent_run = AsyncMock() + with ( + patch("routes.quiz.table", side_effect=_generate_table_factory()), + patch("routes.quiz.quiz_agent.run", new=agent_run), + ): + r = _generate({"difficulty": "impossible"}) + _assert_envelope(r, 400, "QUIZ_DIFFICULTY_INVALID") + agent_run.assert_not_called() + + def test_concept_not_found_404(self): + def factory(name): + mock = MagicMock() + mock.select.return_value = [] + return mock + + with patch("routes.quiz.table", side_effect=factory): + r = _generate({"difficulty": "easy"}) + _assert_envelope(r, 404, "QUIZ_CONCEPT_NOT_FOUND") + + def test_attempt_not_found_404(self): + with patch("routes.quiz.table") as t: + t.return_value.select.return_value = [] + r = client.post("/api/quiz/submit", json={ + "quiz_id": "missing", "answers": [], + }) + _assert_envelope(r, 404, "QUIZ_ATTEMPT_NOT_FOUND") + + def test_already_completed_409(self): + def factory(name): + mock = MagicMock() + if name == "quiz_attempts": + mock.select.return_value = [{ + "id": "quiz1", + "user_id": "user_andres", + "concept_node_id": "node1", + "difficulty": "medium", + "questions_json": [], + "completed_at": "2026-08-01T12:00:00", + }] + else: + mock.select.return_value = [] + return mock + + with patch("routes.quiz.table", side_effect=factory): + r = client.post("/api/quiz/submit", json={ + "quiz_id": "quiz1", "answers": [], + }) + _assert_envelope(r, 409, "QUIZ_ATTEMPT_ALREADY_COMPLETED") + + def test_count_out_of_range_422(self): + r = _generate({"difficulty": "medium", "num_questions": 15}) + body = _assert_envelope(r, 422, "QUIZ_COUNT_OUT_OF_RANGE") + # Pydantic's machine-readable errors stay available under detail. + assert isinstance(body["detail"], list) + + def test_generation_failure_502(self): + with ( + patch("routes.quiz.table", side_effect=_generate_table_factory()), + patch( + "routes.quiz.quiz_agent.run", + new=AsyncMock(side_effect=RuntimeError("boom")), + ), + ): + r = _generate({"difficulty": "easy"}) + _assert_envelope(r, 502, "QUIZ_GENERATION_FAILED") + + def test_non_quiz_routes_keep_legacy_shape(self): + """The envelope is scoped to /api/quiz/*; everything else keeps the + plain {detail, request_id} contract.""" + r = client.get("/api/does-not-exist") + assert r.status_code == 404 + assert "error" not in r.json() + + def test_error_codes_are_a_single_enum(self): + from services.quiz_errors import QuizErrorCode + + values = {c.value for c in QuizErrorCode} + for expected in ( + "QUIZ_DIFFICULTY_INVALID", + "QUIZ_COUNT_OUT_OF_RANGE", + "QUIZ_GENERATION_FAILED", + "QUIZ_ATTEMPT_ALREADY_COMPLETED", + "QUIZ_ATTEMPT_NOT_FOUND", + "QUIZ_CONCEPT_NOT_FOUND", + ): + assert expected in values diff --git a/frontend/src/components/QuizPanel.test.tsx b/frontend/src/components/QuizPanel.test.tsx index c707e399..9df9f874 100644 --- a/frontend/src/components/QuizPanel.test.tsx +++ b/frontend/src/components/QuizPanel.test.tsx @@ -42,6 +42,9 @@ vi.mock("./CustomSelect", () => ({ vi.mock("@/lib/api", () => ({ generateQuiz: vi.fn(), submitQuiz: vi.fn(), + // Reject so the panel exercises its static fallback lists — the + // config fetch is best-effort by design (#540 A2). + fetchQuizConfig: vi.fn().mockRejectedValue(new Error("offline")), })); import { generateQuiz } from "@/lib/api"; diff --git a/frontend/src/components/QuizPanel.tsx b/frontend/src/components/QuizPanel.tsx index 7b77a350..d6dda16f 100644 --- a/frontend/src/components/QuizPanel.tsx +++ b/frontend/src/components/QuizPanel.tsx @@ -1,10 +1,10 @@ "use client"; -import React, { useMemo, useState } from "react"; +import React, { useEffect, useMemo, useState } from "react"; import { useRouter } from "next/navigation"; import { CustomSelect } from "./CustomSelect"; import { useToast } from "./ToastProvider"; -import { generateQuiz, submitQuiz } from "@/lib/api"; +import { fetchQuizConfig, generateQuiz, submitQuiz, type QuizConfig } from "@/lib/api"; import { conceptOptionsForCourse, courseOptions, @@ -55,18 +55,43 @@ interface QuizPanelProps { onExit: () => void; } -const COUNT_OPTIONS = [ +// #540 A2: the backend's GET /api/quiz/config is the source of truth for +// these selects; the static lists below are only the pre-fetch fallback and +// mirror backend/services/quiz_config.py. The old list offered "15 +// questions" against a 10-question cap (guaranteed 422) and the route used +// to reject "adaptive" — both fixed server-side in #540. +const FALLBACK_COUNT_OPTIONS = [ + { value: "3", label: "3 questions" }, { value: "5", label: "5 questions" }, { value: "10", label: "10 questions" }, - { value: "15", label: "15 questions" }, ]; -const DIFFICULTY_OPTIONS = [ - { value: "easy", label: "Easy" }, - { value: "medium", label: "Medium" }, - { value: "hard", label: "Hard" }, - { value: "adaptive", label: "Adaptive" }, -]; +const DIFFICULTY_LABELS: Record = { + easy: "Easy", + medium: "Medium", + hard: "Hard", + adaptive: "Adaptive", +}; + +const FALLBACK_DIFFICULTY_OPTIONS = Object.entries(DIFFICULTY_LABELS).map( + ([value, label]) => ({ value, label }), +); + +function countOptionsFrom(config: QuizConfig | null) { + if (!config?.num_questions?.options?.length) return FALLBACK_COUNT_OPTIONS; + return config.num_questions.options.map(n => ({ + value: String(n), + label: `${n} questions`, + })); +} + +function difficultyOptionsFrom(config: QuizConfig | null) { + if (!config?.difficulties?.length) return FALLBACK_DIFFICULTY_OPTIONS; + return config.difficulties.map(d => ({ + value: d, + label: DIFFICULTY_LABELS[d] ?? d.charAt(0).toUpperCase() + d.slice(1), + })); +} export function QuizPanel({ userId, concepts, courses, initialConceptId, onExit }: QuizPanelProps) { const router = useRouter(); @@ -98,6 +123,19 @@ export function QuizPanel({ userId, concepts, courses, initialConceptId, onExit const [count, setCount] = useState("5"); const [difficulty, setDifficulty] = useState("medium"); + // Selector values come from the backend (#540 A2); fall back to the + // static mirror until the fetch lands (or if it fails). + const [quizConfig, setQuizConfig] = useState(null); + useEffect(() => { + let cancelled = false; + fetchQuizConfig() + .then(cfg => { if (!cancelled) setQuizConfig(cfg); }) + .catch(() => { /* fallback lists stay in place */ }); + return () => { cancelled = true; }; + }, []); + const countOptions = useMemo(() => countOptionsFrom(quizConfig), [quizConfig]); + const difficultyOptions = useMemo(() => difficultyOptionsFrom(quizConfig), [quizConfig]); + const [quizId, setQuizId] = useState(null); const [questions, setQuestions] = useState([]); const [answers, setAnswers] = useState([]); @@ -249,11 +287,11 @@ export function QuizPanel({ userId, concepts, courses, initialConceptId, onExit
Count
- +
Difficulty
- +
diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index cd45e99a..909be269 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -429,8 +429,19 @@ export const resumeSession = (sessionId: string) => }>(`/api/learn/sessions/${sessionId}/resume`); // Quiz +export interface QuizConfig { + num_questions: { min: number; max: number; options: number[] }; + difficulties: string[]; + question_types: string[]; +} + +// #540 A2: the backend is the single source of truth for selector values — +// QuizPanel builds its count/difficulty selects from this so the UI can +// never again offer a value the route rejects (e.g. the old "15 questions"). +export const fetchQuizConfig = () => fetchJSON('/api/quiz/config'); + export const generateQuiz = (userId: string, conceptNodeId: string, numQuestions: number, difficulty: string, useSharedContext = true) => - fetchJSON<{ quiz_id: string; questions: any[] }>('/api/quiz/generate', { + fetchJSON<{ quiz_id: string; questions: any[]; requested_difficulty?: string; resolved_difficulty?: string }>('/api/quiz/generate', { method: 'POST', body: JSON.stringify({ user_id: userId, concept_node_id: conceptNodeId, num_questions: numQuestions, difficulty, use_shared_context: useSharedContext }), }); From 1ea4c725c1d41228894eab2a6f3863a9472bfb63 Mon Sep 17 00:00:00 2001 From: AndresL230 <190146319+AndresL230@users.noreply.github.com> Date: Wed, 12 Aug 2026 17:31:57 -0400 Subject: [PATCH 2/3] =?UTF-8?q?fix(quiz):=20address=20#547=20review=20?= =?UTF-8?q?=E2=80=94=20envelope=20precision,=20adaptive=20history=20semant?= =?UTF-8?q?ics,=20config-driven=20coverage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review findings (xhigh, 15 defects): - 422s on num_questions only get QUIZ_COUNT_OUT_OF_RANGE for actual bounds violations; type errors stay QUIZ_VALIDATION_ERROR. - _STATUS_FALLBACK narrowed to auth/validation; uncoded 404/405s get the new generic QUIZ_HTTP_ERROR instead of impersonating domain states, and a non-QuizErrorCode `code` attr can no longer crash the handler. - The envelope-vs-legacy branch now lives once in quiz_errors.error_content; main.py's three handlers all call it. - The system prompt defines stored difficulty='adaptive' history rows (judge by accuracy) so the quiz-history tool doesn't feed the stepping rules an undefined token. - The adaptive migration drops the old CHECK by introspection (name-robust) and documents migrate-before-deploy ordering. - bench_quiz_question_cap exits 1 when the baseline cap measured nothing. - QUIZ_QUESTION_TYPES uses the schema token 'multiple_choice', not 'mcq'. - _DIFFICULTY_RANK derived from CONCRETE_DIFFICULTIES. - QuizPanel: humanizeError in both catches (the envelope's sentence now actually reaches the student) and an "Adaptive · " chip in the active phase; vitest covers the config-driven select path both ways; a promoted #540 journey pins config-mirroring selects + the adaptive wire round trip in a real browser. Skipped (documented): moving difficulty validation into a Pydantic Literal — the 400 + QUIZ_DIFFICULTY_INVALID contract is published in #540 and changing it to a 422 buys no client anything today; revisit in #537. Co-Authored-By: Claude Fable 5 --- backend/agents/quiz.py | 5 +- ...4809_quiz_attempts_adaptive_difficulty.sql | 28 +++++-- backend/main.py | 65 +++++++--------- backend/routes/quiz.py | 7 +- backend/scripts/bench_quiz_question_cap.py | 25 ++++-- backend/services/quiz_config.py | 8 +- backend/services/quiz_errors.py | 70 ++++++++++++++--- backend/tests/test_quiz_preflight_a.py | 76 ++++++++++++++++++- frontend/e2e/quiz.spec.ts | 61 +++++++++++++++ frontend/src/components/QuizPanel.test.tsx | 69 +++++++++++++++-- frontend/src/components/QuizPanel.tsx | 29 +++++-- 11 files changed, 365 insertions(+), 78 deletions(-) diff --git a/backend/agents/quiz.py b/backend/agents/quiz.py index 2fd28c24..550cbace 100644 --- a/backend/agents/quiz.py +++ b/backend/agents/quiz.py @@ -121,7 +121,10 @@ class Quiz(BaseModel): "questions; with no history at all, center the mix on medium. The " "±1-step limits below do not apply in adaptive mode, but every " "question still carries a concrete easy|medium|hard difficulty — " - "'adaptive' is never a per-question value.\n\n" + "'adaptive' is never a per-question value. Note that PAST attempts " + "in `recent_attempts` may carry difficulty 'adaptive' (that " + "attempt's mix was agent-chosen); treat such an attempt's " + "difficulty as unspecified and judge it by its accuracy alone.\n\n" "Adaptive-difficulty rules (use `recent_attempts.accuracy`):\n" "- If the most recent 2-3 attempts on this concept averaged < " " 0.5 accuracy, drop the difficulty mix one step from what the " diff --git a/backend/db/migrations/20260812204809_quiz_attempts_adaptive_difficulty.sql b/backend/db/migrations/20260812204809_quiz_attempts_adaptive_difficulty.sql index eab193f2..92ffdd75 100644 --- a/backend/db/migrations/20260812204809_quiz_attempts_adaptive_difficulty.sql +++ b/backend/db/migrations/20260812204809_quiz_attempts_adaptive_difficulty.sql @@ -3,11 +3,29 @@ -- reports what generation actually produced), so the 0025 CHECK -- (easy|medium|hard) must admit 'adaptive'. -- --- Idempotent: DROP IF EXISTS + re-ADD. The constraint name is the PG default --- for 0025's inline CHECK on quiz_attempts(difficulty). - -ALTER TABLE quiz_attempts - DROP CONSTRAINT IF EXISTS quiz_attempts_difficulty_check; +-- DEPLOY ORDER: widening the CHECK is backward-compatible — apply this +-- migration BEFORE deploying the code that accepts 'adaptive'. In the gap the +-- other way around, an adaptive generate runs the full LLM call and then 500s +-- on the INSERT. (The promote runner migrates before merging; for staging run +-- `python -m db.migrate` before the deploy picks up the code.) +-- +-- The DROP is by introspection, not by an assumed default name: environments +-- with out-of-band table history (staging has had some) can carry the same +-- CHECK under a different name, and a name-keyed `DROP IF EXISTS` would +-- silently no-op and leave the narrow constraint alive beside the new one. +DO $$ +DECLARE c record; +BEGIN + FOR c IN + SELECT conname + FROM pg_constraint + WHERE conrelid = 'quiz_attempts'::regclass + AND contype = 'c' + AND pg_get_constraintdef(oid) ILIKE '%difficulty%' + LOOP + EXECUTE format('ALTER TABLE quiz_attempts DROP CONSTRAINT %I', c.conname); + END LOOP; +END $$; ALTER TABLE quiz_attempts ADD CONSTRAINT quiz_attempts_difficulty_check diff --git a/backend/main.py b/backend/main.py index 0e756c01..0d8c9eff 100644 --- a/backend/main.py +++ b/backend/main.py @@ -184,22 +184,23 @@ def _drop_request_arguments(_request, _attributes): app.add_middleware(RequestIDMiddleware) +# #540 A3: on /api/quiz/* paths, quiz_errors.error_content wraps errors in +# the coded envelope (QuizAPIError raise sites carry precise codes; plain +# HTTPExceptions fall back to a status-derived one); everywhere else it +# returns the legacy {detail, request_id} shape unchanged. + + @app.exception_handler(StarletteHTTPException) async def http_exception_handler(request: Request, exc: StarletteHTTPException): rid = getattr(request.state, "request_id", None) or current_request_id() - if quiz_errors.is_quiz_path(request.url.path): - # #540 A3: quiz routes speak the coded envelope. QuizAPIError raise - # sites carry a precise code; plain HTTPExceptions (require_self, - # etc.) fall back to a status-derived one. - content = quiz_errors.quiz_error_body( - exc.status_code, - exc.detail, - rid, - code=getattr(exc, "code", None), - machine_detail=getattr(exc, "machine_detail", None), - ) - else: - content = {"detail": exc.detail, "request_id": rid} + content = quiz_errors.error_content( + request.url.path, + exc.status_code, + exc.detail, + rid, + code=getattr(exc, "code", None), + machine_detail=getattr(exc, "machine_detail", None), + ) return JSONResponse( status_code=exc.status_code, content=content, @@ -210,26 +211,16 @@ async def http_exception_handler(request: Request, exc: StarletteHTTPException): @app.exception_handler(RequestValidationError) async def validation_exception_handler(request: Request, exc: RequestValidationError): rid = getattr(request.state, "request_id", None) or current_request_id() - if quiz_errors.is_quiz_path(request.url.path): - # A bad num_questions is the one validation failure the old UI could - # actually produce (it offered 15 against le=10) — give it a precise - # code + a sentence; everything else is generic QUIZ_VALIDATION_ERROR. - locs = {str(part) for e in exc.errors() for part in e.get("loc", ())} - if "num_questions" in locs: - code = quiz_errors.QuizErrorCode.QUIZ_COUNT_OUT_OF_RANGE - message = ( - f"Quizzes can have between {quiz_config.QUIZ_MIN_QUESTIONS} " - f"and {quiz_config.QUIZ_MAX_QUESTIONS} questions." - ) - else: - code = quiz_errors.QuizErrorCode.QUIZ_VALIDATION_ERROR - message = "That quiz request wasn't valid — please try again." - content = quiz_errors.quiz_error_body( - 422, exc.errors(), rid, code=code, message=message, - machine_detail=exc.errors(), + code, message = quiz_errors.validation_error_code(exc.errors()) + if code is quiz_errors.QuizErrorCode.QUIZ_COUNT_OUT_OF_RANGE: + message = ( + f"Quizzes can have between {quiz_config.QUIZ_MIN_QUESTIONS} " + f"and {quiz_config.QUIZ_MAX_QUESTIONS} questions." ) - else: - content = {"detail": exc.errors(), "request_id": rid} + content = quiz_errors.error_content( + request.url.path, 422, exc.errors(), rid, + code=code, message=message, machine_detail=exc.errors(), + ) return JSONResponse( status_code=422, content=content, @@ -241,13 +232,9 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE async def unhandled_exception_handler(request: Request, exc: Exception): logging.getLogger("main").exception("Unhandled exception") rid = getattr(request.state, "request_id", None) or current_request_id() - if quiz_errors.is_quiz_path(request.url.path): - content = quiz_errors.quiz_error_body( - 500, "Internal server error.", rid, - code=quiz_errors.QuizErrorCode.QUIZ_INTERNAL_ERROR, - ) - else: - content = {"detail": "Internal server error.", "request_id": rid} + content = quiz_errors.error_content( + request.url.path, 500, "Internal server error.", rid, + ) return JSONResponse( status_code=500, content=content, diff --git a/backend/routes/quiz.py b/backend/routes/quiz.py index 2f880450..7cc66144 100644 --- a/backend/routes/quiz.py +++ b/backend/routes/quiz.py @@ -21,6 +21,7 @@ from services import events_service from services.auth_guard import require_self from services.quiz_config import ( + CONCRETE_DIFFICULTIES, REQUESTED_DIFFICULTIES, quiz_config_payload, ) @@ -69,8 +70,10 @@ def _load_prompt(name: str) -> str: _OPTION_LABELS = ["A", "B", "C", "D", "E", "F"] -# Rank order for tie-breaking the overall difficulty report. -_DIFFICULTY_RANK = {"easy": 0, "medium": 1, "hard": 2} +# Rank order for tie-breaking the overall difficulty report — derived from +# the config tuple so a difficulty added there can't be silently dropped by +# _resolved_difficulty's counting. +_DIFFICULTY_RANK = {d: i for i, d in enumerate(CONCRETE_DIFFICULTIES)} def _resolved_difficulty(wire_questions: list[dict]) -> str: diff --git a/backend/scripts/bench_quiz_question_cap.py b/backend/scripts/bench_quiz_question_cap.py index 89989b0f..4bcfb53a 100644 --- a/backend/scripts/bench_quiz_question_cap.py +++ b/backend/scripts/bench_quiz_question_cap.py @@ -87,7 +87,10 @@ class BenchQuiz(BaseModel): ) -async def bench_cap(cap: int) -> None: +async def bench_cap(cap: int) -> int: + """Run the bench for one cap; returns the number of successful runs + (a serving rejection is a legitimate *result* for the >10 caps, but + the caller treats zero successes at the baseline cap as failure).""" agent = _agent_for_cap(cap) latencies, in_tokens, out_tokens, counts = [], [], [], [] for run in range(RUNS_PER_CAP): @@ -117,11 +120,21 @@ async def bench_cap(cap: int) -> None: ) else: print(f"cap={cap} SUMMARY: no successful runs (schema likely rejected)") - - -async def main() -> None: - for cap in CAPS: - await bench_cap(cap) + return len(latencies) + + +async def main() -> int: + successes = {cap: await bench_cap(cap) for cap in CAPS} + # The baseline cap (the one we ship) must have measured something — + # otherwise this run produced no data (bad key, outage) and must not + # exit 0 as if the RESULTS block could be refreshed from it. The >10 + # caps legitimately measure zero successes (serving rejection IS the + # result), so only the baseline gates the exit code. + baseline = CAPS[0] + if successes[baseline] == 0: + print(f"BENCH FAILED: baseline cap={baseline} measured no successful runs") + return 1 + return 0 if __name__ == "__main__": diff --git a/backend/services/quiz_config.py b/backend/services/quiz_config.py index 2c01acf4..264caa56 100644 --- a/backend/services/quiz_config.py +++ b/backend/services/quiz_config.py @@ -36,9 +36,11 @@ # `resolved_difficulty`. REQUESTED_DIFFICULTIES = CONCRETE_DIFFICULTIES + ("adaptive",) -# MCQ-only today — mirrors agents/quiz.py::QuizQuestionType. Grows when -# the #537 revamp adds real short-answer grading. -QUIZ_QUESTION_TYPES = ("mcq",) +# MCQ-only today — the SAME token as agents/quiz.py::QuizQuestionType and +# the per-question wire `type` field, so a client keying on this config can +# compare against real payloads. Grows when the #537 revamp adds real +# short-answer grading. +QUIZ_QUESTION_TYPES = ("multiple_choice",) def quiz_config_payload() -> dict: diff --git a/backend/services/quiz_errors.py b/backend/services/quiz_errors.py index 94bf4159..c297b9e0 100644 --- a/backend/services/quiz_errors.py +++ b/backend/services/quiz_errors.py @@ -37,17 +37,23 @@ class QuizErrorCode(str, Enum): QUIZ_NOT_AUTHORIZED = "QUIZ_NOT_AUTHORIZED" QUIZ_GENERATION_FAILED = "QUIZ_GENERATION_FAILED" QUIZ_INTERNAL_ERROR = "QUIZ_INTERNAL_ERROR" + # Uncoded HTTP errors that aren't one of the semantic states above — + # router 404s/405s on version-skewed clients, library-raised + # HTTPExceptions, anything without an explicit QuizErrorCode. A client + # must never mistake these for a domain state like "attempt not found". + QUIZ_HTTP_ERROR = "QUIZ_HTTP_ERROR" # Fallback code when an ordinary HTTPException (no explicit code) escapes a -# quiz route — e.g. require_self's 401/403. +# quiz route. Deliberately narrow: only statuses whose meaning is unambiguous +# regardless of which code path raised them (auth guards, Pydantic). Domain +# states (404 concept/attempt, 409 replay, 502 generation) are NOT here — +# their raise sites all carry explicit codes, and a router-level 404/405 must +# come out as the generic QUIZ_HTTP_ERROR, not impersonate a domain state. _STATUS_FALLBACK: dict[int, QuizErrorCode] = { 401: QuizErrorCode.QUIZ_NOT_AUTHORIZED, 403: QuizErrorCode.QUIZ_NOT_AUTHORIZED, - 404: QuizErrorCode.QUIZ_ATTEMPT_NOT_FOUND, - 409: QuizErrorCode.QUIZ_ATTEMPT_ALREADY_COMPLETED, 422: QuizErrorCode.QUIZ_VALIDATION_ERROR, - 502: QuizErrorCode.QUIZ_GENERATION_FAILED, } @@ -84,17 +90,23 @@ def quiz_error_body( machine_detail=None, ) -> dict: """Build the enveloped payload; keeps the legacy top-level keys.""" - resolved_code = code or _STATUS_FALLBACK.get( - status_code, - QuizErrorCode.QUIZ_INTERNAL_ERROR, - ) + if not isinstance(code, QuizErrorCode): + # The handler duck-types `code` off the exception; a library + # exception could carry an int or arbitrary string there. Treat + # anything that isn't ours as absent — never crash the handler. + code = None + if code is None: + if status_code >= 500: + code = QuizErrorCode.QUIZ_INTERNAL_ERROR + else: + code = _STATUS_FALLBACK.get(status_code, QuizErrorCode.QUIZ_HTTP_ERROR) resolved_message = message or ( legacy_detail if isinstance(legacy_detail, str) else "Something went wrong with this quiz request." ) error: dict = { - "code": resolved_code.value, + "code": code.value, "message": resolved_message, "request_id": request_id, } @@ -105,3 +117,43 @@ def quiz_error_body( "detail": legacy_detail, "request_id": request_id, } + + +def error_content( + path: str, + status_code: int, + legacy_detail, + request_id: str | None, + code: QuizErrorCode | None = None, + message: str | None = None, + machine_detail=None, +) -> dict: + """The one branch point between the quiz envelope and the legacy + ``{detail, request_id}`` shape — main.py's three exception handlers all + call this so the contract lives in exactly one place.""" + if is_quiz_path(path): + return quiz_error_body( + status_code, legacy_detail, request_id, + code=code, message=message, machine_detail=machine_detail, + ) + return {"detail": legacy_detail, "request_id": request_id} + + +def validation_error_code(errors: list[dict]) -> tuple["QuizErrorCode", str | None]: + """Pick the (code, message) for a quiz-route RequestValidationError. + + QUIZ_COUNT_OUT_OF_RANGE only for an actual bounds violation on + num_questions — a type error ("five", 5.5) on the same field is NOT + out-of-range, and mislabelling it would send a clamping client into a + retry loop on the identical 422. The message is filled by the caller + (it owns the min/max constants). + """ + for e in errors: + if "num_questions" in {str(part) for part in e.get("loc", ())}: + etype = str(e.get("type", "")) + if etype.startswith(("greater_than", "less_than")): + return QuizErrorCode.QUIZ_COUNT_OUT_OF_RANGE, None + return ( + QuizErrorCode.QUIZ_VALIDATION_ERROR, + "That quiz request wasn't valid — please try again.", + ) diff --git a/backend/tests/test_quiz_preflight_a.py b/backend/tests/test_quiz_preflight_a.py index 0d5792bc..f950b01b 100644 --- a/backend/tests/test_quiz_preflight_a.py +++ b/backend/tests/test_quiz_preflight_a.py @@ -11,6 +11,8 @@ """ from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch + +from fastapi import HTTPException from fastapi.testclient import TestClient from main import app @@ -82,7 +84,10 @@ def test_config_returns_selector_options(self): assert isinstance(nq["options"], list) and nq["options"] assert all(nq["min"] <= v <= nq["max"] for v in nq["options"]) assert data["difficulties"] == ["easy", "medium", "hard", "adaptive"] - assert data["question_types"] == ["mcq"] + # Same token as the agent schema (agents/quiz.py::QuizQuestionType) — + # inventing a parallel "mcq" vocabulary would recreate the exact + # client-offers-what-the-server-rejects class this endpoint fixes. + assert data["question_types"] == ["multiple_choice"] def test_config_matches_pydantic_model_bounds(self): """The Pydantic cap and the config endpoint must read the same named @@ -155,6 +160,15 @@ def _capture(payload): # The attempt row records what the student asked for. assert captured["payload"]["difficulty"] == "adaptive" + def test_prompt_defines_stored_adaptive_history_rows(self): + """quiz_attempts.difficulty can now hold 'adaptive'; that value flows + verbatim into recent_attempts via the quiz-history tool, so the + system prompt must define what it means (judge by accuracy) or the + stepping rules operate on an undefined token.""" + from agents.quiz import _SYSTEM_PROMPT + + assert "may carry difficulty 'adaptive'" in _SYSTEM_PROMPT + def test_adaptive_routing_message_instructs_agent(self): """The agent must be told to pick the mix itself — and that every emitted question still carries a concrete difficulty.""" @@ -294,6 +308,66 @@ def test_non_quiz_routes_keep_legacy_shape(self): assert r.status_code == 404 assert "error" not in r.json() + def test_type_error_on_num_questions_is_not_the_count_code(self): + """A non-numeric num_questions fails int parsing, not the bounds — + labelling it QUIZ_COUNT_OUT_OF_RANGE would send a clamping client + into a retry loop on the identical 422.""" + r = _generate({"difficulty": "medium", "num_questions": "five"}) + assert r.status_code == 422 + assert r.json()["error"]["code"] == "QUIZ_VALIDATION_ERROR" + + def test_router_404_gets_generic_code_not_attempt_not_found(self): + """A no-such-endpoint 404 under /api/quiz/ must not impersonate + QUIZ_ATTEMPT_NOT_FOUND — a code-branching client would discard its + active quiz state over a version-skewed route.""" + r = client.get("/api/quiz/definitely-not-a-route") + assert r.status_code == 404 + assert r.json()["error"]["code"] == "QUIZ_HTTP_ERROR" + + def test_method_not_allowed_gets_generic_code(self): + r = client.get("/api/quiz/generate") # POST-only route + assert r.status_code == 405 + assert r.json()["error"]["code"] == "QUIZ_HTTP_ERROR" + + def test_uncoded_403_maps_to_not_authorized(self): + """The status-fallback path for shared deps: a plain, code-less + HTTPException(403) — exactly what require_self raises (conftest's + autouse auth stub replaces the real guard, so we raise the same + exception through the route body) — must come out enveloped as + QUIZ_NOT_AUTHORIZED with the legacy detail preserved.""" + boom = HTTPException(status_code=403, detail="Forbidden: not your account") + + def factory(name): + mock = MagicMock() + mock.select.side_effect = boom + return mock + + with patch("routes.quiz.table", side_effect=factory): + r = client.post("/api/quiz/submit", json={ + "quiz_id": "quiz1", "answers": [], + }) + body = _assert_envelope(r, 403, "QUIZ_NOT_AUTHORIZED") + assert body["detail"] == "Forbidden: not your account" + + def test_non_enum_code_attribute_does_not_crash_the_handler(self): + """An HTTPException subclass carrying a non-QuizErrorCode `code` + (int, plain string) must fall back to a status code — not raise + AttributeError inside the handler and mask the error as a bare 500.""" + boom = HTTPException(status_code=404, detail="library says no") + boom.code = 123 # not a QuizErrorCode + + def factory(name): + mock = MagicMock() + mock.select.side_effect = boom + return mock + + with patch("routes.quiz.table", side_effect=factory): + r = client.post("/api/quiz/submit", json={ + "quiz_id": "quiz1", "answers": [], + }) + assert r.status_code == 404 + assert r.json()["error"]["code"] == "QUIZ_HTTP_ERROR" + def test_error_codes_are_a_single_enum(self): from services.quiz_errors import QuizErrorCode diff --git a/frontend/e2e/quiz.spec.ts b/frontend/e2e/quiz.spec.ts index 3397edd6..4c00fe63 100644 --- a/frontend/e2e/quiz.spec.ts +++ b/frontend/e2e/quiz.spec.ts @@ -316,3 +316,64 @@ test("quiz resubmit: replaying a completed submission returns 409 and re-applies 10, ); }); + +/** + * Journey (#540 A1/A2): the select phase is config-driven and adaptive + * difficulty reports what generation actually chose. + * + * The class of bug this pins: the old static UI lists offered values the + * route rejects ("15 questions" against le=10, "Adaptive" against a + * concrete-only difficulty check). The selects must now mirror + * GET /api/quiz/config, and an adaptive generate must echo + * requested_difficulty="adaptive" + a concrete resolved_difficulty — + * "medium" here, pinned to the all-medium fixture questions in + * agents/function_handlers_e2e.py (KEEP IN SYNC). + */ +test("selects mirror /api/quiz/config; adaptive reports its resolved difficulty (#540)", async ({ + page, +}) => { + const cfgResponse = await page.request.get("/api/quiz/config"); + expect(cfgResponse.ok()).toBe(true); + const cfg = await cfgResponse.json(); + expect(cfg.num_questions.options.length).toBeGreaterThan(0); + for (const n of cfg.num_questions.options) { + expect(n).toBeGreaterThanOrEqual(cfg.num_questions.min); + expect(n).toBeLessThanOrEqual(cfg.num_questions.max); + } + expect(cfg.difficulties).toContain("adaptive"); + + await page.addInitScript(() => { + window.localStorage.setItem("sapling_disclaimer_ack", "true"); + }); + await page.goto(`/quiz?concept=${NODE_ID}`); + await expect(page.getByTestId("quiz-start")).toBeEnabled(); + + // The count select offers exactly the config's options — an out-of-range + // value here is the #540 regression. + await page.getByRole("button", { name: "Number of questions" }).click(); + const countOptions = page.getByRole("option"); + await expect(countOptions).toHaveText( + cfg.num_questions.options.map((n: number) => `${n} questions`), + ); + // Close by committing the current default (5 questions). + await page.getByRole("option", { name: "5 questions" }).click(); + + // Pick Adaptive and start; capture the real wire round trip. + await page.getByRole("button", { name: "Difficulty" }).click(); + await page.getByRole("option", { name: "Adaptive" }).click(); + const responsePromise = page.waitForResponse( + r => r.url().includes("/api/quiz/generate") && r.request().method() === "POST", + ); + await page.getByTestId("quiz-start").click(); + const response = await responsePromise; + expect(response.status()).toBe(200); + expect(response.request().postDataJSON().difficulty).toBe("adaptive"); + const body = await response.json(); + expect(body.requested_difficulty).toBe("adaptive"); + expect(body.resolved_difficulty).toBe("medium"); + + // The pick is surfaced to the student, not just returned on the wire. + await expect(page.getByTestId("quiz-resolved-difficulty")).toHaveText( + /adaptive · medium/i, + ); +}); diff --git a/frontend/src/components/QuizPanel.test.tsx b/frontend/src/components/QuizPanel.test.tsx index 9df9f874..ec939b3d 100644 --- a/frontend/src/components/QuizPanel.test.tsx +++ b/frontend/src/components/QuizPanel.test.tsx @@ -12,7 +12,7 @@ */ import React from "react"; -import { describe, it, expect, vi, afterEach } from "vitest"; +import { beforeEach, describe, it, expect, vi, afterEach } from "vitest"; import { render, screen, cleanup, fireEvent, waitFor } from "@testing-library/react"; // Captured toast spies — hoisted so the vi.mock factory can close over them. @@ -34,23 +34,44 @@ vi.mock("next/navigation", () => ({ // course + concept (via the real quizSelection helpers), so a passive stub is // enough and keeps the DOM simple. vi.mock("./CustomSelect", () => ({ - CustomSelect: ({ value }: { value?: string }) => ( -
{value}
+ CustomSelect: ({ + value, + options, + ariaLabel, + }: { + value?: string; + options?: { value: string; label: string }[]; + ariaLabel?: string; + }) => ( + // Expose the option values so tests can assert what the panel would + // actually offer (the #540 config-driven lists), not just the selection. +
o.value).join(",")} + > + {value} +
), })); vi.mock("@/lib/api", () => ({ generateQuiz: vi.fn(), submitQuiz: vi.fn(), - // Reject so the panel exercises its static fallback lists — the - // config fetch is best-effort by design (#540 A2). - fetchQuizConfig: vi.fn().mockRejectedValue(new Error("offline")), + fetchQuizConfig: vi.fn(), })); -import { generateQuiz } from "@/lib/api"; +import { fetchQuizConfig, generateQuiz } from "@/lib/api"; import { QuizPanel } from "./QuizPanel"; const mockGenerateQuiz = vi.mocked(generateQuiz); +const mockFetchQuizConfig = vi.mocked(fetchQuizConfig); + +beforeEach(() => { + // Default: config unavailable → the panel exercises its static fallback + // lists. The config-driven test overrides this per-test (#540 A2). + mockFetchQuizConfig.mockRejectedValue(new Error("offline")); +}); const CONCEPTS = [ { id: "c1", name: "Gradient Descent", course_id: "course-1", course_code: "CS101" }, @@ -131,3 +152,37 @@ describe("QuizPanel — start() happy path", () => { expect(toast.error).not.toHaveBeenCalled(); }); }); + +describe("QuizPanel — config-driven selectors (#540 A2)", () => { + it("builds the count and difficulty selects from the fetched config", async () => { + // Non-default values so a fallback-list render can't accidentally pass. + mockFetchQuizConfig.mockResolvedValue({ + num_questions: { min: 1, max: 10, options: [2, 4] }, + difficulties: ["easy", "adaptive"], + question_types: ["multiple_choice"], + }); + renderPanel(); + + await waitFor(() => expect(mockFetchQuizConfig).toHaveBeenCalledTimes(1)); + const counts = await screen.findByLabelText("Number of questions"); + await waitFor(() => expect(counts).toHaveAttribute("data-options", "2,4")); + expect(screen.getByLabelText("Difficulty")).toHaveAttribute( + "data-options", + "easy,adaptive", + ); + }); + + it("falls back to the static mirror when the config fetch fails", async () => { + // beforeEach already rejects the fetch. + renderPanel(); + await waitFor(() => expect(mockFetchQuizConfig).toHaveBeenCalledTimes(1)); + expect(screen.getByLabelText("Number of questions")).toHaveAttribute( + "data-options", + "3,5,10", + ); + expect(screen.getByLabelText("Difficulty")).toHaveAttribute( + "data-options", + "easy,medium,hard,adaptive", + ); + }); +}); diff --git a/frontend/src/components/QuizPanel.tsx b/frontend/src/components/QuizPanel.tsx index d6dda16f..0339619d 100644 --- a/frontend/src/components/QuizPanel.tsx +++ b/frontend/src/components/QuizPanel.tsx @@ -5,6 +5,7 @@ import { useRouter } from "next/navigation"; import { CustomSelect } from "./CustomSelect"; import { useToast } from "./ToastProvider"; import { fetchQuizConfig, generateQuiz, submitQuiz, type QuizConfig } from "@/lib/api"; +import { humanizeError } from "@/lib/errorMessage"; import { conceptOptionsForCourse, courseOptions, @@ -137,6 +138,7 @@ export function QuizPanel({ userId, concepts, courses, initialConceptId, onExit const difficultyOptions = useMemo(() => difficultyOptionsFrom(quizConfig), [quizConfig]); const [quizId, setQuizId] = useState(null); + const [resolvedDifficulty, setResolvedDifficulty] = useState(null); const [questions, setQuestions] = useState([]); const [answers, setAnswers] = useState([]); const [qIndex, setQIndex] = useState(0); @@ -165,13 +167,18 @@ export function QuizPanel({ userId, concepts, courses, initialConceptId, onExit } setQuizId(res.quiz_id); setQuestions(nextQuestions); + // #540 A1: what generation actually chose — shown when the student + // asked for "adaptive" so the pick isn't a black box. + setResolvedDifficulty(res.resolved_difficulty ?? null); setAnswers([]); setQIndex(0); setCurrentSelection(null); setLastCorrect(null); setPhase("active"); } catch (err) { - toast.error(err instanceof Error ? err.message : "Failed to generate quiz."); + // The quiz routes return the #540 A3 envelope; humanizeError digs the + // readable sentence out of the thrown body instead of dumping JSON. + toast.error(humanizeError(err, "Failed to generate quiz.")); } finally { setLoading(false); } @@ -204,7 +211,7 @@ export function QuizPanel({ userId, concepts, courses, initialConceptId, onExit setResults(res); setPhase("results"); } catch (err) { - toast.error(err instanceof Error ? err.message : "Failed to submit quiz."); + toast.error(humanizeError(err, "Failed to submit quiz.")); } finally { setLoading(false); } @@ -287,11 +294,11 @@ export function QuizPanel({ userId, concepts, courses, initialConceptId, onExit
Count
- +
Difficulty
- +
@@ -307,7 +314,19 @@ export function QuizPanel({ userId, concepts, courses, initialConceptId, onExit <>
Question {qIndex + 1} of {questions.length}
-
{currentQuestion.difficulty}
+
+ {difficulty === "adaptive" && resolvedDifficulty && ( +
+ Adaptive · {resolvedDifficulty} +
+ )} +
{currentQuestion.difficulty}
+
{currentQuestion.question}
From 1bd35124139241b1390b6855a358d6c2c983dacd Mon Sep 17 00:00:00 2001 From: AndresL230 <190146319+AndresL230@users.noreply.github.com> Date: Wed, 12 Aug 2026 17:35:10 -0400 Subject: [PATCH 3/3] =?UTF-8?q?test(e2e):=20tolerate=20the=20CustomSelect?= =?UTF-8?q?=20=E2=9C=93=20marker=20in=20the=20#540=20selector=20journey?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- frontend/e2e/quiz.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/e2e/quiz.spec.ts b/frontend/e2e/quiz.spec.ts index 4c00fe63..1d008298 100644 --- a/frontend/e2e/quiz.spec.ts +++ b/frontend/e2e/quiz.spec.ts @@ -352,8 +352,9 @@ test("selects mirror /api/quiz/config; adaptive reports its resolved difficulty // value here is the #540 regression. await page.getByRole("button", { name: "Number of questions" }).click(); const countOptions = page.getByRole("option"); + // The selected option carries a trailing ✓ marker — match on the label. await expect(countOptions).toHaveText( - cfg.num_questions.options.map((n: number) => `${n} questions`), + cfg.num_questions.options.map((n: number) => new RegExp(`^${n} questions`)), ); // Close by committing the current default (5 questions). await page.getByRole("option", { name: "5 questions" }).click();