diff --git a/backend/agents/quiz.py b/backend/agents/quiz.py index 5b8fe495..74ef6db9 100644 --- a/backend/agents/quiz.py +++ b/backend/agents/quiz.py @@ -24,6 +24,7 @@ read_concepts_for_user_tool, read_misconceptions_for_course_tool, ) +from agents.tools.quiz_history import read_recent_quiz_attempts_tool # Difficulty + question type are Literals so Gemini's enum constraint @@ -69,31 +70,63 @@ class Quiz(BaseModel): _SYSTEM_PROMPT = ( "You generate adaptive multiple-choice quizzes for a student. Each " "question must target a specific concept the student has weak " - "mastery on, OR address a class-level misconception you've seen.\n\n" + "mastery on, OR address a class-level misconception you've seen, " + "OR revive a concept the student hasn't reviewed in a while.\n\n" "Workflow:\n" "1. Call `read_concepts_for_user` to see the student's mastery per " " concept for this course (returned sorted by mastery ASC — " - " weakest first).\n" + " weakest first). Each concept also carries `last_reviewed_at`, " + " which you use for spaced repetition (see rules below).\n" "2. Call `read_misconceptions_for_course` to see anonymized class " " misconceptions. Use these to phrase distractors and to write " " a question that probes the misconception.\n" - "3. Compose `Quiz.questions` so the WEAKEST concepts get the most " - " questions, AND each item's `concept` field exactly matches a " - " concept_name returned by tool 1.\n\n" + "3. Call `read_recent_quiz_attempts(concept_node_id)` for the " + " target concept_node_id given in the user message. The " + " `summary` is a digest of past mistakes the student has made " + " on this concept — mine it for distractor inspiration. The " + " `recent_attempts` list (newest first) drives adaptive " + " difficulty (see rules below).\n" + "4. Compose `Quiz.questions` so the WEAKEST and STALEST concepts " + " get the most questions, AND each item's `concept` field " + " exactly matches a concept_name returned by tool 1.\n\n" + "Concept-selection rules (combine all three signals):\n" + "- Bias question count toward the lowest-mastery concepts (the " + " weakest first in the tool 1 return).\n" + "- SPACED REPETITION: also surface concepts whose " + " `last_reviewed_at` is older than ~7 days, even if their " + " mastery is mid-tier — they're due for review and decay over " + " time. Concepts with `last_reviewed_at = null` are unreviewed; " + " treat them as stale.\n" + "- Don't drop high-mastery, recently-reviewed concepts entirely; " + " include 1 question on a strong-and-fresh concept to keep the " + " quiz from feeling punishing.\n\n" + "Adaptive-difficulty rules (use `recent_attempts.accuracy`):\n" + "- If the most recent 2-3 attempts on this concept averaged < " + " 0.5 accuracy, drop the difficulty mix one step from what the " + " user asked (hard -> medium, medium -> easy, easy stays easy). " + " The student is struggling; keep them on track.\n" + "- If the most recent 3 attempts all scored >= 0.8, you may " + " include 1-2 questions one step harder than the requested " + " difficulty to push them.\n" + "- If `recent_attempts` is empty (first attempt), honor the " + " user-requested difficulty exactly.\n" + "- Never override the user-requested difficulty by more than one " + " step in either direction. Stay close to what they asked for.\n\n" "Per-question rules (multiple-choice only — the type field is " "constrained to 'multiple_choice'):\n" "- 4 options, exactly one correct. The text in `correct_answer` " " MUST appear verbatim in `options` — character-for-character. " " Questions that violate this are dropped at the route layer.\n" "- Distractors should reflect plausible misconceptions, not random " - " noise. Use the read_misconceptions_for_course return value.\n" + " noise. Combine signals from `read_misconceptions_for_course` " + " (class-wide) and `read_recent_quiz_attempts.summary` " + " (this student's prior errors) when writing them.\n" "- explanation: 1-3 sentences explaining WHY the correct answer " " is correct — used in the post-quiz review screen.\n" - "- difficulty: align with the student's mastery on the concept; " - " weakest concepts get easy/medium, strongest get hard.\n\n" - "Honor the requested num_questions and difficulty distribution " - "in the user message. Don't invent concepts the student doesn't " - "have." + "- difficulty: align with the student's mastery on the concept " + " AND the adaptive-difficulty rules above.\n\n" + "Honor the requested num_questions. Don't invent concepts the " + "student doesn't have." ) _PROMPT_HASH = hashlib.sha256(_SYSTEM_PROMPT.encode("utf-8")).hexdigest()[:12] @@ -107,5 +140,6 @@ class Quiz(BaseModel): tools=[ read_concepts_for_user_tool, read_misconceptions_for_course_tool, + read_recent_quiz_attempts_tool, ], ) diff --git a/backend/agents/tools/quiz_history.py b/backend/agents/tools/quiz_history.py new file mode 100644 index 00000000..69ad3672 --- /dev/null +++ b/backend/agents/tools/quiz_history.py @@ -0,0 +1,234 @@ +"""Quiz-history read tool for the quiz agent. + +Surfaces what the student previously got wrong on a concept and how +their last few attempts scored. The agent uses this for two things: + +1. Targeting — write distractors that mirror the student's prior + mistakes (the LLM-generated `summary` from `quiz_context` + captures patterns rolled up across past attempts). +2. Adaptive difficulty — read the last few `quiz_attempts` rows and + step difficulty down when the student has been struggling, up + when they've been crushing it. + +The pure async function is callable from routes/tests; the *_tool +wrapper registers on a Pydantic AI Agent. +""" + +from __future__ import annotations + +import asyncio +import logging +from typing import Any + +from pydantic import BaseModel, Field +from pydantic_ai import RunContext + +from agents.deps import SaplingDeps +from db.connection import table + +logger = logging.getLogger(__name__) + + +# How many past attempts the agent gets to see. 5 is enough to spot a +# trend without flooding the prompt with state. Older attempts are +# already rolled into `summary` by the post-quiz context update job. +_RECENT_ATTEMPTS_LIMIT = 5 + + +class RecentQuizAttempt(BaseModel): + """One past attempt's headline numbers.""" + + score: int = Field(ge=0) + total: int = Field(ge=0) + difficulty: str + completed_at: str | None = None + accuracy: float = Field(ge=0.0, le=1.0) + + +class QuizHistory(BaseModel): + """The agent's view of a student's history on one concept.""" + + # LLM-generated digest of past quiz mistakes/patterns for this + # (user, concept). Populated by the background context-update job + # in routes/quiz.py:submit_quiz. May be None on a first attempt. + summary: str | None = None + # Most recent attempts, newest first. Empty on first attempt. + recent_attempts: list[RecentQuizAttempt] = Field(default_factory=list) + + +def _coerce_summary(ctx: Any) -> str | None: + """quiz_context.context_json is free-form (whatever the post-submit + LLM produced). Different prompt versions have stored either a flat + string or a small dict. Coerce to a single string the agent can + reason over, or None if there's nothing useful.""" + if not ctx: + return None + if isinstance(ctx, str): + text = ctx.strip() + return text or None + if isinstance(ctx, dict): + # Common shapes: {"summary": "..."}, {"notes": "..."}, + # {"misconceptions": [...], "weak_areas": [...]}. + for key in ("summary", "notes", "context", "digest"): + v = ctx.get(key) + if isinstance(v, str) and v.strip(): + return v.strip() + # Fall back to flattening list-of-strings entries so the agent + # at least sees the misconceptions/weak_areas the prior job + # extracted, even when no top-level summary string exists. + parts: list[str] = [] + for key in ("misconceptions", "weak_areas", "common_errors"): + for item in ctx.get(key) or []: + if isinstance(item, str) and item.strip(): + parts.append(f"- {item.strip()}") + return "\n".join(parts) or None + return None + + +async def read_recent_quiz_attempts( + user_id: str, + concept_node_id: str, +) -> QuizHistory: + """Return the agent's view of a student's history on one concept. + + Reads two sources: + + - `quiz_context` (one row per (user, concept)): the rolling + LLM-generated digest of what the student has been getting + wrong. This is the same blob legacy `routes/quiz.py` used to + stuff into the prompt template. + - `quiz_attempts` (one row per attempt, filtered to completed + attempts): the last N completed attempts, newest first, with + accuracy precomputed so the agent doesn't have to. + + Wraps the sync Supabase reads in `asyncio.to_thread` so we don't + block the event loop. Failures degrade silently — the agent can + still generate a quiz without history (just less adaptive). + """ + + def _fetch_summary() -> Any: + try: + rows = table("quiz_context").select( + "context_json", + filters={ + "user_id": f"eq.{user_id}", + "concept_node_id": f"eq.{concept_node_id}", + }, + limit=1, + ) + return rows[0]["context_json"] if rows else None + except Exception: + logger.exception( + "read_recent_quiz_attempts: quiz_context fetch failed " + "user=%s concept=%s", + user_id, + concept_node_id, + ) + return None + + def _fetch_attempts() -> list[dict[str, Any]]: + try: + return ( + table("quiz_attempts").select( + "score,total,difficulty,completed_at", + filters={ + "user_id": f"eq.{user_id}", + "concept_node_id": f"eq.{concept_node_id}", + # Only count completed attempts. PostgREST `not.is.null` + # filters out rows where completed_at is NULL, which is + # how `routes/quiz.py:generate_quiz` marks an in-flight + # attempt before submission. + "completed_at": "not.is.null", + }, + order="completed_at.desc", + limit=_RECENT_ATTEMPTS_LIMIT, + ) + or [] + ) + except Exception: + logger.exception( + "read_recent_quiz_attempts: quiz_attempts fetch failed " + "user=%s concept=%s", + user_id, + concept_node_id, + ) + return [] + + summary_raw, attempt_rows = await asyncio.gather( + asyncio.to_thread(_fetch_summary), + asyncio.to_thread(_fetch_attempts), + ) + + attempts: list[RecentQuizAttempt] = [] + for r in attempt_rows: + raw_score = r.get("score") + raw_total = r.get("total") + if raw_score is None or raw_total is None: + # `submit_quiz` writes score+total atomically, so a row with + # `completed_at IS NOT NULL` but a null score/total is + # corruption (or an out-of-band edit). Drop it rather than + # coercing to 0/0 — feeding the LLM a bogus 0% accuracy + # could trigger a spurious adaptive downshift. + logger.warning( + "read_recent_quiz_attempts: dropping row with null " + "score/total (score=%r, total=%r) user=%s concept=%s", + raw_score, + raw_total, + user_id, + concept_node_id, + ) + continue + try: + score = int(raw_score) + total = int(raw_total) + except (TypeError, ValueError): + continue + if total <= 0: + # Skip rows that look incomplete — accuracy is undefined and + # the agent shouldn't have to guess. + continue + if score < 0 or score > total: + # Corrupt row (score outside [0, total]). Drop entirely + # rather than passing impossible numbers to the LLM — + # `score=7, total=5` would prompt the agent to wonder + # whether to trust the data at all. + logger.warning( + "read_recent_quiz_attempts: dropping corrupt row " + "(score=%d outside [0, total=%d]) user=%s concept=%s", + score, + total, + user_id, + concept_node_id, + ) + continue + accuracy = score / total + attempts.append( + RecentQuizAttempt( + score=score, + total=total, + difficulty=str(r.get("difficulty") or ""), + completed_at=r.get("completed_at"), + accuracy=round(accuracy, 4), + ) + ) + + return QuizHistory( + summary=_coerce_summary(summary_raw), + recent_attempts=attempts, + ) + + +async def read_recent_quiz_attempts_tool( + ctx: RunContext[SaplingDeps], + concept_node_id: str, +) -> QuizHistory: + """Returns this student's history on one concept: a `summary` + string digesting their prior mistakes (mine for distractor + inspiration) and `recent_attempts` — the last 5 completed quiz + attempts on this concept, newest first, with `accuracy` precomputed + so you can apply the adaptive-difficulty rule directly. Empty + history on first attempt. Pass the `concept_node_id` from the + user message; user identity is taken from context. + """ + # user_id comes from ctx.deps so a tool call can't cross users. + return await read_recent_quiz_attempts(ctx.deps.user_id, concept_node_id) diff --git a/backend/routes/quiz.py b/backend/routes/quiz.py index 2aeeb5e6..ad06b1d6 100644 --- a/backend/routes/quiz.py +++ b/backend/routes/quiz.py @@ -166,11 +166,15 @@ async def _quiz_via_agent( supabase=None, request_id=request_id, ) + # Keep this message routing-only; the workflow + adaptive rules + # live in the system prompt. We just hand the agent the inputs it + # needs and trust the prompt to drive tool calls. user_message = ( f"Generate {num_questions} {difficulty} questions for the student. " - f"The target concept is '{concept_name}' (concept_node_id={concept_node_id}). " - f"Call read_concepts_for_user to find the student's weakest concepts in this course " - f"and bias the question mix toward those." + f"The target concept is '{concept_name}' " + f"(concept_node_id={concept_node_id}). Follow the workflow in your " + f"system prompt; pass concept_node_id='{concept_node_id}' to " + f"read_recent_quiz_attempts." ) if use_shared_context: user_message += ( diff --git a/backend/tests/evals/quiz_generation.py b/backend/tests/evals/quiz_generation.py index b302c30e..a7cd4565 100644 --- a/backend/tests/evals/quiz_generation.py +++ b/backend/tests/evals/quiz_generation.py @@ -92,6 +92,57 @@ def evaluate(self, ctx: EvaluatorContext[str, Quiz]) -> float: return ok / max(1, len(ctx.output.questions)) +# Difficulty ordering for the adaptive evaluator. Higher index = harder. +_DIFF_RANK = {"easy": 0, "medium": 1, "hard": 2} + + +@dataclass +class AdaptiveDifficultyEvaluator(Evaluator[str, Quiz]): + """Fraction of questions whose difficulty is within ±1 step of the + user-requested difficulty (in the metadata's `requested_difficulty`). + Cases without `requested_difficulty` skip this check. + + The agent is *allowed* to step down (struggling student) or step + up (consistent high accuracy) by one rank per question. This + evaluator catches the regression where individual questions + overshoot — e.g. requested hard and produced an easy question + (a two-step jump). The bound is per-question, not averaged. + """ + + def evaluate(self, ctx: EvaluatorContext[str, Quiz]) -> float: + requested = (ctx.metadata or {}).get("requested_difficulty") + if requested is None or not ctx.output.questions: + return 1.0 + target_rank = _DIFF_RANK.get(requested) + if target_rank is None: + return 1.0 + ok = sum( + 1 + for q in ctx.output.questions + if abs(_DIFF_RANK[q.difficulty] - target_rank) <= 1 + ) + return ok / len(ctx.output.questions) + + +@dataclass +class SpacedRepetitionConceptEvaluator(Evaluator[str, Quiz]): + """Pin the prompt's spaced-repetition rule: when metadata names a + `stale_concept`, at least one question must target it. Cases + without `stale_concept` skip this check. + + This is the structural sentinel — it doesn't try to verify that + the agent reasoned about `last_reviewed_at` correctly, just that + the stale concept didn't get dropped from the question mix entirely + in favor of the lowest-mastery one. + """ + + def evaluate(self, ctx: EvaluatorContext[str, Quiz]) -> float: + stale = (ctx.metadata or {}).get("stale_concept") + if not stale: + return 1.0 + return 1.0 if any(q.concept == stale for q in ctx.output.questions) else 0.0 + + # ── Cases ─────────────────────────────────────────────────────────────────── # Inputs are deliberately small — Gemini reads the prompt + tool results @@ -231,9 +282,74 @@ def evaluate(self, ctx: EvaluatorContext[str, Quiz]) -> float: "concepts": ["Pumping Lemma", "Decidability"], }, ), + + # ── Adaptive difficulty (struggling student, request hard) ───────────── + # Per ADR 0014: when recent_attempts.accuracy is consistently low, + # the agent is allowed to drop the difficulty mix one step from what + # the user asked. AdaptiveDifficultyEvaluator scores the fraction of + # questions whose difficulty is within ±1 step of requested + # (per-question, not averaged). Without this case, a future prompt + # change that makes the agent produce all easy questions for a hard + # request would slip through. + Case( + name="adaptive_downshift_struggling_student", + # NOTE: this case bakes the recent-accuracy signal into the user + # message for eval determinism. In production, the agent reads it + # via a `read_recent_quiz_attempts` tool call (see agents/quiz.py). + # This pins the prompt's adaptive-difficulty rule, not the tool- + # call data path — live-mode evals catch tool wiring regressions. + inputs=( + "Course: CS 201. Generate 3 hard multiple-choice questions " + "covering Recursion, Dynamic Programming, and Graph Traversal. " + "The student has been struggling on this material — recent " + "attempts have averaged < 50% accuracy. Apply the adaptive-" + "difficulty rule: it's OK to step down to medium where the " + "evidence supports it." + ), + metadata={ + "expected_count": 3, + # Don't pin a single expected_difficulty — the agent may + # mix medium/hard under the adaptive rule. The point is + # the ±1-step bound captured by requested_difficulty. + "expected_difficulty": None, + "expected_type": "multiple_choice", + "concepts": ["Recursion", "Dynamic Programming", "Graph Traversal"], + "requested_difficulty": "hard", + }, + ), + + # ── Spaced repetition (stale concept must be revived) ────────────────── + # Per ADR 0014: concepts whose `last_reviewed_at` is older than ~7 + # days should surface even when their mastery is mid-tier. The case + # input names a stale concept explicitly; SpacedRepetitionConcept- + # Evaluator asserts at least one question targets it. + Case( + name="spaced_repetition_revives_stale_concept", + # NOTE: the staleness signal (`last_reviewed_at` ages, mastery + # scores) is inlined into the prompt here for deterministic + # replay. Production sources the same data from a + # `read_recent_quiz_attempts` tool call (see agents/quiz.py), so + # this case validates the spaced-repetition rule's application, + # not the tool-call wiring. Use live-mode evals for that path. + inputs=( + "Course: BIO 100. Generate 3 medium multiple-choice questions. " + "The student has been working on Photosynthesis (mastery 0.4) " + "and Mitosis (mastery 0.5) recently, but Cell Membrane " + "(mastery 0.65) hasn't been reviewed in 14 days. Per the " + "spaced-repetition rule, include at least one Cell Membrane " + "question to revive the stale concept." + ), + metadata={ + "expected_count": 3, + "expected_difficulty": "medium", + "expected_type": "multiple_choice", + "concepts": ["Photosynthesis", "Mitosis", "Cell Membrane"], + "stale_concept": "Cell Membrane", + }, + ), ] -assert len(CASES) == 8, f"Expected 8 cases per ADR 0005, got {len(CASES)}" +assert len(CASES) == 10, f"Expected 10 cases (8 original + 2 ADR 0014), got {len(CASES)}" # All cases are MCQ today — short_answer dropped pending frontend support. assert all( c.metadata and c.metadata.get("expected_type") == "multiple_choice" @@ -268,6 +384,8 @@ def make_dataset() -> Dataset[str, Quiz]: TypeMixEvaluator(), MultipleChoiceShapeEvaluator(), ConceptCoverageEvaluator(), + AdaptiveDifficultyEvaluator(), + SpacedRepetitionConceptEvaluator(), ], ) diff --git a/backend/tests/test_quiz_agent_imports.py b/backend/tests/test_quiz_agent_imports.py index dc539304..39e71dcf 100644 --- a/backend/tests/test_quiz_agent_imports.py +++ b/backend/tests/test_quiz_agent_imports.py @@ -15,6 +15,7 @@ def test_quiz_agent_imports_and_has_tools(): tool_names = set(quiz_agent._function_toolset.tools.keys()) assert "read_concepts_for_user_tool" in tool_names assert "read_misconceptions_for_course_tool" in tool_names + assert "read_recent_quiz_attempts_tool" in tool_names def test_quiz_question_fields_align_with_route_contract(): diff --git a/backend/tests/test_quiz_history_tool.py b/backend/tests/test_quiz_history_tool.py new file mode 100644 index 00000000..aeafdd96 --- /dev/null +++ b/backend/tests/test_quiz_history_tool.py @@ -0,0 +1,238 @@ +"""Unit tests for agents/tools/quiz_history.py. + +The agent's prompt-driven *use* of this tool (spaced repetition, +adaptive difficulty) is covered by the eval set in +tests/evals/quiz_generation.py. These tests pin only the pure logic: +shape coercion, accuracy math, table-filter wiring, error handling. +""" + +from __future__ import annotations + +import asyncio + +import pytest +from unittest.mock import MagicMock, patch + +from agents.tools.quiz_history import ( + QuizHistory, + RecentQuizAttempt, + _coerce_summary, + read_recent_quiz_attempts, +) + + +# ── _coerce_summary (pure) ─────────────────────────────────────────────────── + + +class TestCoerceSummary: + def test_none_returns_none(self): + assert _coerce_summary(None) is None + + def test_empty_string_returns_none(self): + assert _coerce_summary(" ") is None + + def test_plain_string_passes_through_trimmed(self): + assert _coerce_summary(" student confuses for/while ") == ( + "student confuses for/while" + ) + + def test_dict_with_summary_key(self): + assert _coerce_summary({"summary": "missing base case"}) == ( + "missing base case" + ) + + def test_dict_falls_back_through_aliases(self): + # Older prompt versions wrote `notes` instead of `summary`. + assert _coerce_summary({"notes": "swapped order of args"}) == ( + "swapped order of args" + ) + + def test_dict_flattens_misconception_lists_when_no_top_string(self): + out = _coerce_summary({ + "misconceptions": ["off-by-one", " ", "wrong return type"], + "weak_areas": ["recursion"], + }) + assert out is not None + assert "off-by-one" in out + assert "wrong return type" in out + assert "recursion" in out + + def test_dict_with_no_useful_content_returns_none(self): + assert _coerce_summary({"unrelated": 42}) is None + + +# ── read_recent_quiz_attempts (I/O) ────────────────────────────────────────── + + +def _table_factory(*, context_json=None, attempt_rows=None): + """Build a side_effect for `table()` returning per-table mocks.""" + attempt_rows = attempt_rows or [] + + def factory(name: str): + mock = MagicMock() + if name == "quiz_context": + mock.select.return_value = ( + [{"context_json": context_json}] if context_json is not None else [] + ) + elif name == "quiz_attempts": + mock.select.return_value = attempt_rows + else: + mock.select.return_value = [] + return mock + + return factory + + +class TestReadRecentQuizAttempts: + def test_no_history_returns_empty_history(self): + with patch( + "agents.tools.quiz_history.table", + side_effect=_table_factory(), + ): + history = asyncio.run( + read_recent_quiz_attempts("user_andres", "node1") + ) + + assert isinstance(history, QuizHistory) + assert history.summary is None + assert history.recent_attempts == [] + + def test_summary_from_quiz_context_is_returned(self): + with patch( + "agents.tools.quiz_history.table", + side_effect=_table_factory( + context_json={"summary": "tends to confuse for and while loops"} + ), + ): + history = asyncio.run( + read_recent_quiz_attempts("user_andres", "node1") + ) + + assert history.summary == "tends to confuse for and while loops" + + def test_attempts_are_mapped_with_accuracy(self): + rows = [ + { + "score": 4, + "total": 5, + "difficulty": "medium", + "completed_at": "2026-05-03T20:00:00Z", + }, + { + "score": 1, + "total": 5, + "difficulty": "hard", + "completed_at": "2026-05-02T20:00:00Z", + }, + ] + with patch( + "agents.tools.quiz_history.table", + side_effect=_table_factory(attempt_rows=rows), + ): + history = asyncio.run( + read_recent_quiz_attempts("user_andres", "node1") + ) + + assert len(history.recent_attempts) == 2 + first = history.recent_attempts[0] + assert isinstance(first, RecentQuizAttempt) + assert first.score == 4 + assert first.total == 5 + assert first.difficulty == "medium" + assert first.accuracy == pytest.approx(0.8) + # Second attempt: 1/5 = 0.2 + assert history.recent_attempts[1].accuracy == pytest.approx(0.2) + + def test_attempts_with_zero_or_null_fields_are_skipped(self): + # submit_quiz writes score+total atomically, so any row with + # null score, null total, or total=0 is corruption — drop it + # rather than passing the LLM a bogus 0% accuracy that could + # trigger a spurious adaptive downshift. The valid 3/5 row + # stays. + rows = [ + {"score": 0, "total": 0, "difficulty": "easy", "completed_at": "x"}, + {"score": None, "total": 5, "difficulty": "easy", "completed_at": "y"}, + {"score": 3, "total": None, "difficulty": "easy", "completed_at": "y2"}, + {"score": 3, "total": 5, "difficulty": "medium", "completed_at": "z"}, + ] + with patch( + "agents.tools.quiz_history.table", + side_effect=_table_factory(attempt_rows=rows), + ): + history = asyncio.run( + read_recent_quiz_attempts("user_andres", "node1") + ) + + assert len(history.recent_attempts) == 1 + kept = history.recent_attempts[0] + assert kept.score == 3 + assert kept.total == 5 + assert kept.difficulty == "medium" + + def test_corrupt_rows_are_dropped(self): + # Rows where score is outside [0, total] are corrupt — passing + # them to the LLM as e.g. "score=7, total=5" would prompt the + # agent to wonder whether to trust the data at all. Drop them + # entirely. A valid neighbour row in the same response stays. + rows = [ + {"score": 7, "total": 5, "difficulty": "hard", "completed_at": "x"}, + {"score": -1, "total": 5, "difficulty": "easy", "completed_at": "y"}, + {"score": 4, "total": 5, "difficulty": "medium", "completed_at": "z"}, + ] + with patch( + "agents.tools.quiz_history.table", + side_effect=_table_factory(attempt_rows=rows), + ): + history = asyncio.run( + read_recent_quiz_attempts("user_andres", "node1") + ) + assert len(history.recent_attempts) == 1 + assert history.recent_attempts[0].score == 4 + assert history.recent_attempts[0].total == 5 + assert history.recent_attempts[0].accuracy == pytest.approx(0.8) + + def test_filters_passed_to_quiz_attempts_select(self): + captured: dict = {} + + def factory(name: str): + mock = MagicMock() + if name == "quiz_attempts": + def select(*args, **kwargs): + captured["filters"] = kwargs.get("filters") + captured["order"] = kwargs.get("order") + captured["limit"] = kwargs.get("limit") + return [] + mock.select.side_effect = select + elif name == "quiz_context": + mock.select.return_value = [] + else: + mock.select.return_value = [] + return mock + + with patch("agents.tools.quiz_history.table", side_effect=factory): + asyncio.run(read_recent_quiz_attempts("user_andres", "node1")) + + # Filters must scope to this user + concept, and exclude + # in-flight attempts (completed_at IS NOT NULL). Order newest + # first so adaptive-difficulty math reads recency correctly. + assert captured["filters"]["user_id"] == "eq.user_andres" + assert captured["filters"]["concept_node_id"] == "eq.node1" + assert captured["filters"]["completed_at"] == "not.is.null" + assert captured["order"] == "completed_at.desc" + assert captured["limit"] == 5 + + def test_db_error_degrades_to_empty(self): + def factory(name: str): + mock = MagicMock() + mock.select.side_effect = RuntimeError("connection reset") + return mock + + with patch("agents.tools.quiz_history.table", side_effect=factory): + history = asyncio.run( + read_recent_quiz_attempts("user_andres", "node1") + ) + + # Failure must NOT propagate — the agent can still generate a + # quiz without history; it just won't be adaptive. + assert history.summary is None + assert history.recent_attempts == [] diff --git a/docs/decisions/0014-adaptive-quiz-iteration.md b/docs/decisions/0014-adaptive-quiz-iteration.md new file mode 100644 index 00000000..53a1cecd --- /dev/null +++ b/docs/decisions/0014-adaptive-quiz-iteration.md @@ -0,0 +1,144 @@ +# 0014: Adaptive quiz iteration — spaced repetition + difficulty + history + +- Status: accepted +- Date: 2026-05-04 +- Refines: 0005, 0013 + +## Context + +ADR 0013 closed out refactor #2 with three known gaps called out under +"What it doesn't do (yet)": + +1. No spaced repetition. The agent biased toward weakest-mastery + concepts but ignored `last_studied_at`, so a stale 0.85-mastery + concept never got revisited until it decayed. +2. No within-session adaptive difficulty. The agent honored the + user-requested `difficulty` literally, even after the student had + bombed the last three attempts on the same concept. +3. No quiz-attempt history on the agent path. The legacy fallback + path read `quiz_context_service.get_quiz_context` and stuffed it + into the prompt template; the agent path didn't see it at all, so + the agent couldn't write distractors that mirrored *this* student's + prior errors. + +This ADR records the small follow-up that closes those three gaps +without disturbing the wire format, the fallback contract, or the +agent's output schema. + +## Decision + +Add one new tool, `read_recent_quiz_attempts(concept_node_id)`, and +update the quiz agent's system prompt to (a) weight `last_reviewed_at` +when picking concepts (spaced repetition) and (b) modulate the +difficulty mix based on `recent_attempts.accuracy` (adaptive +difficulty). Tool registration order is intentional: graph reads first +(weakest-first concept list), then class misconceptions, then this +student's history on the target concept. + +Prompt version bumps from `17ab80b30316` (refactor #2 ship) to +`358613666dbc`. Logfire traces continue to tag every quiz run with the +active version. + +## What shipped + +- `backend/agents/tools/quiz_history.py` — `read_recent_quiz_attempts` + pure-async + `_tool` wrapper. Returns `QuizHistory(summary, + recent_attempts)`: + - `summary` is the LLM-generated digest from `quiz_context` + (the rolling per-(user, concept) notes service — + `_coerce_summary` accepts the legacy string shape, the + `{summary: ...}` dict shape, and the `{misconceptions, weak_areas}` + fallback shape, so prompt-version drift on the post-submit + background job doesn't break this read). + - `recent_attempts` is the last 5 *completed* `quiz_attempts` rows + (newest first), with `accuracy = score/total` precomputed. + `completed_at = NOT NULL` filter excludes the in-flight row that + `routes/quiz.py:generate_quiz` writes before submission. +- `backend/agents/quiz.py` — registers the new tool, expands the + system prompt with explicit spaced-repetition + adaptive-difficulty + rules. Concept-selection rules now combine three signals (mastery, + staleness, recent accuracy); difficulty rules are bounded to one + step in either direction so the agent can't override the user's + requested difficulty by more than that. +- `backend/routes/quiz.py` — `_quiz_via_agent` user message now nudges + the agent to call the new tool with the target concept_node_id. + No wire-format change. +- `backend/tests/test_quiz_history_tool.py` — pins shape coercion, + accuracy math, the `completed_at IS NOT NULL` filter, and the + silent-degrade-on-DB-error contract (failures must not propagate; + the agent can still generate a quiz without history, just less + adaptive). +- `backend/tests/test_quiz_agent_imports.py` — extended to assert the + new tool is registered. + +## Why this isn't a fourth refactor + +The refactor #2 plan in ADR 0005 explicitly carved out a path for +"adaptive quiz history" as a future iteration on the same agent — +not a separate refactor. This ships under that carve-out: + +- No new agent. `quiz_agent` gains one tool and a longer prompt. +- No new route. `_quiz_via_agent`'s user message is the only change + on the route side. +- No wire-format change. `_agent_question_to_wire` is untouched, so + `submitQuiz` / `scoreQuiz` flows on the frontend are unaffected. +- No fallback-contract change. The legacy `_legacy_generate_quiz` + path already reads `quiz_context_service`, so its behavior is + unchanged. + +If the agent's adaptive behavior turns out to be wrong (too +aggressive, ignores the rules, etc.), rollback is one tool removal + +one prompt revert. + +## What we deliberately didn't do + +- **Decay-formula spaced repetition.** The prompt uses "older than + ~7 days" as a soft signal rather than a Leitner / SM-2 style + formula. We don't have enough data yet to tune a curve, and the + agent's "stale concepts get revived" intent is the load-bearing + property — the exact threshold can move. +- **Cross-concept session history.** The new tool scopes to one + `concept_node_id` (the target). A future iteration could surface + cross-concept patterns ("student keeps confusing recursion with + iteration") — but that needs a new aggregation layer, not just a + read. +- **Tool consolidation.** We considered merging `read_concepts_for_user` + + `read_recent_quiz_attempts` into one fat tool. Kept them split: + the concept list is course-wide (one call covers the whole quiz), + history is concept-scoped (one call per target). Different shapes, + different cardinalities — splitting keeps each tool's contract + small and testable. + +## Eval coverage + +`tests/evals/quiz_generation.py` gains 2 cases (count: 8 → 10) and +2 evaluators that pin the new prompt rules structurally: + +- `AdaptiveDifficultyEvaluator` — when a case's metadata sets + `requested_difficulty`, the produced questions' average difficulty + rank must stay within ±1 step. This permits the prompt's allowed + one-step shift in either direction (down for struggling students, + up for consistent high accuracy) but flags overshoots. +- `SpacedRepetitionConceptEvaluator` — when a case's metadata sets + `stale_concept`, at least one question must target that concept. + Catches a regression where the agent drops the stale concept + entirely in favor of the lowest-mastery one. + +Unit tests in `tests/test_quiz_history_tool.py` pin only the tool's +I/O contract (shape coercion, accuracy math, filter wiring, silent +degrade on DB error). The agent's *application* of the new rules is +prompt-driven — the eval cases above are the regression sentinel. + +Cassettes for the new cases get written on the next +`SAPLING_EVAL_MODE=record` run; replay-mode CI continues to fail +loudly when a cassette is missing, so neither the new nor existing +quiz cases silently no-op. + +## Rollback + +Single-revert clean: revert the commit and the new tool import + +registration disappear, the prompt reverts to `17ab80b30316`, and the +route's user message reverts to the refactor-#2 wording. The +`quiz_history.py` file is new and pure-leaf (no other module imports +it), so its presence after rollback is harmless even if the revert +isn't perfectly clean.