From 1a47fec7a76c15356effecab97dacb657ae7d0c9 Mon Sep 17 00:00:00 2001
From: Jose Cruz <jose.colorstack@gmail.com>
Date: Mon, 4 May 2026 22:10:11 -0400
Subject: [PATCH 1/5] =?UTF-8?q?feat(quiz):=20adaptive=20quiz=20iteration?=
 =?UTF-8?q?=20=E2=80=94=20spaced=20repetition=20+=20history=20+=20difficul?=
 =?UTF-8?q?ty?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Closes the three "doesn't do (yet)" gaps from ADR 0013 with one new
tool and a prompt update. No wire-format change, no fallback-contract
change.

- agents/tools/quiz_history.py — read_recent_quiz_attempts surfaces
  the per-(user, concept) digest from quiz_context plus the last 5
  completed attempts (newest first, accuracy precomputed). DB
  failures degrade silently to empty history.
- agents/quiz.py — registers the new tool, expands prompt with
  spaced-repetition rules (weight last_reviewed_at on graph_nodes)
  and adaptive-difficulty rules (modulate by recent_attempts.accuracy
  with a one-step bound). Prompt hash bumps to 358613666dbc.
- routes/quiz.py — _quiz_via_agent user message nudges the agent to
  call the new tool with the target concept_node_id.
- docs/decisions/0014-adaptive-quiz-iteration.md — captures the
  decision, what we deliberately didn't do, and the rollback path.
- Tests: new test_quiz_history_tool.py pins shape coercion, the
  completed_at IS NOT NULL filter, and silent-degrade contract;
  test_quiz_agent_imports.py asserts the new tool is registered.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 backend/agents/quiz.py                        |  56 ++++-
 backend/agents/tools/quiz_history.py          | 200 ++++++++++++++++
 backend/routes/quiz.py                        |   8 +-
 backend/tests/test_quiz_agent_imports.py      |   1 +
 backend/tests/test_quiz_history_tool.py       | 224 ++++++++++++++++++
 .../decisions/0014-adaptive-quiz-iteration.md | 129 ++++++++++
 6 files changed, 605 insertions(+), 13 deletions(-)
 create mode 100644 backend/agents/tools/quiz_history.py
 create mode 100644 backend/tests/test_quiz_history_tool.py
 create mode 100644 docs/decisions/0014-adaptive-quiz-iteration.md

diff --git a/backend/agents/quiz.py b/backend/agents/quiz.py
index 5b8fe495..74ef6db9 100644
--- a/backend/agents/quiz.py
+++ b/backend/agents/quiz.py
@@ -24,6 +24,7 @@
     read_concepts_for_user_tool,
     read_misconceptions_for_course_tool,
 )
+from agents.tools.quiz_history import read_recent_quiz_attempts_tool
 
 
 # Difficulty + question type are Literals so Gemini's enum constraint
@@ -69,31 +70,63 @@ class Quiz(BaseModel):
 _SYSTEM_PROMPT = (
     "You generate adaptive multiple-choice quizzes for a student. Each "
     "question must target a specific concept the student has weak "
-    "mastery on, OR address a class-level misconception you've seen.\n\n"
+    "mastery on, OR address a class-level misconception you've seen, "
+    "OR revive a concept the student hasn't reviewed in a while.\n\n"
     "Workflow:\n"
     "1. Call `read_concepts_for_user` to see the student's mastery per "
     "   concept for this course (returned sorted by mastery ASC — "
-    "   weakest first).\n"
+    "   weakest first). Each concept also carries `last_reviewed_at`, "
+    "   which you use for spaced repetition (see rules below).\n"
     "2. Call `read_misconceptions_for_course` to see anonymized class "
     "   misconceptions. Use these to phrase distractors and to write "
     "   a question that probes the misconception.\n"
-    "3. Compose `Quiz.questions` so the WEAKEST concepts get the most "
-    "   questions, AND each item's `concept` field exactly matches a "
-    "   concept_name returned by tool 1.\n\n"
+    "3. Call `read_recent_quiz_attempts(concept_node_id)` for the "
+    "   target concept_node_id given in the user message. The "
+    "   `summary` is a digest of past mistakes the student has made "
+    "   on this concept — mine it for distractor inspiration. The "
+    "   `recent_attempts` list (newest first) drives adaptive "
+    "   difficulty (see rules below).\n"
+    "4. Compose `Quiz.questions` so the WEAKEST and STALEST concepts "
+    "   get the most questions, AND each item's `concept` field "
+    "   exactly matches a concept_name returned by tool 1.\n\n"
+    "Concept-selection rules (combine all three signals):\n"
+    "- Bias question count toward the lowest-mastery concepts (the "
+    "   weakest first in the tool 1 return).\n"
+    "- SPACED REPETITION: also surface concepts whose "
+    "   `last_reviewed_at` is older than ~7 days, even if their "
+    "   mastery is mid-tier — they're due for review and decay over "
+    "   time. Concepts with `last_reviewed_at = null` are unreviewed; "
+    "   treat them as stale.\n"
+    "- Don't drop high-mastery, recently-reviewed concepts entirely; "
+    "   include 1 question on a strong-and-fresh concept to keep the "
+    "   quiz from feeling punishing.\n\n"
+    "Adaptive-difficulty rules (use `recent_attempts.accuracy`):\n"
+    "- If the most recent 2-3 attempts on this concept averaged < "
+    "   0.5 accuracy, drop the difficulty mix one step from what the "
+    "   user asked (hard -> medium, medium -> easy, easy stays easy). "
+    "   The student is struggling; keep them on track.\n"
+    "- If the most recent 3 attempts all scored >= 0.8, you may "
+    "   include 1-2 questions one step harder than the requested "
+    "   difficulty to push them.\n"
+    "- If `recent_attempts` is empty (first attempt), honor the "
+    "   user-requested difficulty exactly.\n"
+    "- Never override the user-requested difficulty by more than one "
+    "   step in either direction. Stay close to what they asked for.\n\n"
     "Per-question rules (multiple-choice only — the type field is "
     "constrained to 'multiple_choice'):\n"
     "- 4 options, exactly one correct. The text in `correct_answer` "
     "   MUST appear verbatim in `options` — character-for-character. "
     "   Questions that violate this are dropped at the route layer.\n"
     "- Distractors should reflect plausible misconceptions, not random "
-    "   noise. Use the read_misconceptions_for_course return value.\n"
+    "   noise. Combine signals from `read_misconceptions_for_course` "
+    "   (class-wide) and `read_recent_quiz_attempts.summary` "
+    "   (this student's prior errors) when writing them.\n"
     "- explanation: 1-3 sentences explaining WHY the correct answer "
     "   is correct — used in the post-quiz review screen.\n"
-    "- difficulty: align with the student's mastery on the concept; "
-    "   weakest concepts get easy/medium, strongest get hard.\n\n"
-    "Honor the requested num_questions and difficulty distribution "
-    "in the user message. Don't invent concepts the student doesn't "
-    "have."
+    "- difficulty: align with the student's mastery on the concept "
+    "   AND the adaptive-difficulty rules above.\n\n"
+    "Honor the requested num_questions. Don't invent concepts the "
+    "student doesn't have."
 )
 _PROMPT_HASH = hashlib.sha256(_SYSTEM_PROMPT.encode("utf-8")).hexdigest()[:12]
 
@@ -107,5 +140,6 @@ class Quiz(BaseModel):
     tools=[
         read_concepts_for_user_tool,
         read_misconceptions_for_course_tool,
+        read_recent_quiz_attempts_tool,
     ],
 )
diff --git a/backend/agents/tools/quiz_history.py b/backend/agents/tools/quiz_history.py
new file mode 100644
index 00000000..5ff4fcda
--- /dev/null
+++ b/backend/agents/tools/quiz_history.py
@@ -0,0 +1,200 @@
+"""Quiz-history read tool for the quiz agent.
+
+Surfaces what the student previously got wrong on a concept and how
+their last few attempts scored. The agent uses this for two things:
+
+1. Targeting — write distractors that mirror the student's prior
+   mistakes (the LLM-generated `summary` from `quiz_context`
+   captures patterns rolled up across past attempts).
+2. Adaptive difficulty — read the last few `quiz_attempts` rows and
+   step difficulty down when the student has been struggling, up
+   when they've been crushing it.
+
+The pure async function is callable from routes/tests; the *_tool
+wrapper registers on a Pydantic AI Agent.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any
+
+from pydantic import BaseModel, Field
+from pydantic_ai import RunContext
+
+from agents.deps import SaplingDeps
+from db.connection import table
+
+logger = logging.getLogger(__name__)
+
+
+# How many past attempts the agent gets to see. 5 is enough to spot a
+# trend without flooding the prompt with state. Older attempts are
+# already rolled into `summary` by the post-quiz context update job.
+_RECENT_ATTEMPTS_LIMIT = 5
+
+
+class RecentQuizAttempt(BaseModel):
+    """One past attempt's headline numbers."""
+
+    score: int = Field(ge=0)
+    total: int = Field(ge=0)
+    difficulty: str
+    completed_at: str | None = None
+    accuracy: float = Field(ge=0.0, le=1.0)
+
+
+class QuizHistory(BaseModel):
+    """The agent's view of a student's history on one concept."""
+
+    # LLM-generated digest of past quiz mistakes/patterns for this
+    # (user, concept). Populated by the background context-update job
+    # in routes/quiz.py:submit_quiz. May be None on a first attempt.
+    summary: str | None = None
+    # Most recent attempts, newest first. Empty on first attempt.
+    recent_attempts: list[RecentQuizAttempt] = Field(default_factory=list)
+
+
+def _coerce_summary(ctx: Any) -> str | None:
+    """quiz_context.context_json is free-form (whatever the post-submit
+    LLM produced). Different prompt versions have stored either a flat
+    string or a small dict. Coerce to a single string the agent can
+    reason over, or None if there's nothing useful."""
+    if not ctx:
+        return None
+    if isinstance(ctx, str):
+        text = ctx.strip()
+        return text or None
+    if isinstance(ctx, dict):
+        # Common shapes: {"summary": "..."}, {"notes": "..."},
+        # {"misconceptions": [...], "weak_areas": [...]}.
+        for key in ("summary", "notes", "context", "digest"):
+            v = ctx.get(key)
+            if isinstance(v, str) and v.strip():
+                return v.strip()
+        # Fall back to flattening list-of-strings entries so the agent
+        # at least sees the misconceptions/weak_areas the prior job
+        # extracted, even when no top-level summary string exists.
+        parts: list[str] = []
+        for key in ("misconceptions", "weak_areas", "common_errors"):
+            for item in ctx.get(key) or []:
+                if isinstance(item, str) and item.strip():
+                    parts.append(f"- {item.strip()}")
+        return "\n".join(parts) or None
+    return None
+
+
+async def read_recent_quiz_attempts(
+    user_id: str,
+    concept_node_id: str,
+) -> QuizHistory:
+    """Return the agent's view of a student's history on one concept.
+
+    Reads two sources:
+
+    - `quiz_context` (one row per (user, concept)): the rolling
+      LLM-generated digest of what the student has been getting
+      wrong. This is the same blob legacy `routes/quiz.py` used to
+      stuff into the prompt template.
+    - `quiz_attempts` (one row per attempt, filtered to completed
+      attempts): the last N completed attempts, newest first, with
+      accuracy precomputed so the agent doesn't have to.
+
+    Wraps the sync Supabase reads in `asyncio.to_thread` so we don't
+    block the event loop. Failures degrade silently — the agent can
+    still generate a quiz without history (just less adaptive).
+    """
+
+    def _fetch_summary() -> Any:
+        try:
+            rows = table("quiz_context").select(
+                "context_json",
+                filters={
+                    "user_id": f"eq.{user_id}",
+                    "concept_node_id": f"eq.{concept_node_id}",
+                },
+                limit=1,
+            )
+            return rows[0]["context_json"] if rows else None
+        except Exception:
+            logger.exception(
+                "read_recent_quiz_attempts: quiz_context fetch failed "
+                "user=%s concept=%s",
+                user_id,
+                concept_node_id,
+            )
+            return None
+
+    def _fetch_attempts() -> list[dict[str, Any]]:
+        try:
+            return (
+                table("quiz_attempts").select(
+                    "score,total,difficulty,completed_at",
+                    filters={
+                        "user_id": f"eq.{user_id}",
+                        "concept_node_id": f"eq.{concept_node_id}",
+                        # Only count completed attempts. PostgREST `not.is.null`
+                        # filters out rows where completed_at is NULL, which is
+                        # how `routes/quiz.py:generate_quiz` marks an in-flight
+                        # attempt before submission.
+                        "completed_at": "not.is.null",
+                    },
+                    order="completed_at.desc",
+                    limit=_RECENT_ATTEMPTS_LIMIT,
+                )
+                or []
+            )
+        except Exception:
+            logger.exception(
+                "read_recent_quiz_attempts: quiz_attempts fetch failed "
+                "user=%s concept=%s",
+                user_id,
+                concept_node_id,
+            )
+            return []
+
+    summary_raw, attempt_rows = await asyncio.gather(
+        asyncio.to_thread(_fetch_summary),
+        asyncio.to_thread(_fetch_attempts),
+    )
+
+    attempts: list[RecentQuizAttempt] = []
+    for r in attempt_rows:
+        try:
+            score = int(r.get("score") or 0)
+            total = int(r.get("total") or 0)
+        except (TypeError, ValueError):
+            continue
+        if total <= 0:
+            # Skip rows that look incomplete — accuracy is undefined and
+            # the agent shouldn't have to guess.
+            continue
+        accuracy = max(0.0, min(1.0, score / total))
+        attempts.append(
+            RecentQuizAttempt(
+                score=score,
+                total=total,
+                difficulty=str(r.get("difficulty") or ""),
+                completed_at=r.get("completed_at"),
+                accuracy=round(accuracy, 4),
+            )
+        )
+
+    return QuizHistory(
+        summary=_coerce_summary(summary_raw),
+        recent_attempts=attempts,
+    )
+
+
+async def read_recent_quiz_attempts_tool(
+    ctx: RunContext[SaplingDeps],
+    concept_node_id: str,
+) -> QuizHistory:
+    """Pydantic AI tool wrapper.
+
+    `concept_node_id` is supplied by the agent (which receives it in
+    the user message). user_id comes from deps so a tool-call can't
+    cross users.
+    """
+    return await read_recent_quiz_attempts(ctx.deps.user_id, concept_node_id)
diff --git a/backend/routes/quiz.py b/backend/routes/quiz.py
index 2aeeb5e6..8bf829b9 100644
--- a/backend/routes/quiz.py
+++ b/backend/routes/quiz.py
@@ -169,8 +169,12 @@ async def _quiz_via_agent(
     user_message = (
         f"Generate {num_questions} {difficulty} questions for the student. "
         f"The target concept is '{concept_name}' (concept_node_id={concept_node_id}). "
-        f"Call read_concepts_for_user to find the student's weakest concepts in this course "
-        f"and bias the question mix toward those."
+        f"Call read_concepts_for_user to find the student's weakest and stalest "
+        f"concepts in this course and bias the question mix toward those. "
+        f"Then call read_recent_quiz_attempts(concept_node_id='{concept_node_id}') "
+        f"to see what this student has been getting wrong on this concept and how "
+        f"recent attempts have been scoring — use that to set the difficulty mix "
+        f"adaptively per the system prompt."
     )
     if use_shared_context:
         user_message += (
diff --git a/backend/tests/test_quiz_agent_imports.py b/backend/tests/test_quiz_agent_imports.py
index dc539304..39e71dcf 100644
--- a/backend/tests/test_quiz_agent_imports.py
+++ b/backend/tests/test_quiz_agent_imports.py
@@ -15,6 +15,7 @@ def test_quiz_agent_imports_and_has_tools():
     tool_names = set(quiz_agent._function_toolset.tools.keys())
     assert "read_concepts_for_user_tool" in tool_names
     assert "read_misconceptions_for_course_tool" in tool_names
+    assert "read_recent_quiz_attempts_tool" in tool_names
 
 
 def test_quiz_question_fields_align_with_route_contract():
diff --git a/backend/tests/test_quiz_history_tool.py b/backend/tests/test_quiz_history_tool.py
new file mode 100644
index 00000000..0e8d4ecd
--- /dev/null
+++ b/backend/tests/test_quiz_history_tool.py
@@ -0,0 +1,224 @@
+"""Unit tests for agents/tools/quiz_history.py.
+
+The agent's prompt-driven *use* of this tool (spaced repetition,
+adaptive difficulty) is covered by the eval set in
+tests/evals/quiz_generation.py. These tests pin only the pure logic:
+shape coercion, accuracy math, table-filter wiring, error handling.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+import pytest
+from unittest.mock import MagicMock, patch
+
+from agents.tools.quiz_history import (
+    QuizHistory,
+    RecentQuizAttempt,
+    _coerce_summary,
+    read_recent_quiz_attempts,
+)
+
+
+# ── _coerce_summary (pure) ───────────────────────────────────────────────────
+
+
+class TestCoerceSummary:
+    def test_none_returns_none(self):
+        assert _coerce_summary(None) is None
+
+    def test_empty_string_returns_none(self):
+        assert _coerce_summary("   ") is None
+
+    def test_plain_string_passes_through_trimmed(self):
+        assert _coerce_summary("  student confuses for/while  ") == (
+            "student confuses for/while"
+        )
+
+    def test_dict_with_summary_key(self):
+        assert _coerce_summary({"summary": "missing base case"}) == (
+            "missing base case"
+        )
+
+    def test_dict_falls_back_through_aliases(self):
+        # Older prompt versions wrote `notes` instead of `summary`.
+        assert _coerce_summary({"notes": "swapped order of args"}) == (
+            "swapped order of args"
+        )
+
+    def test_dict_flattens_misconception_lists_when_no_top_string(self):
+        out = _coerce_summary({
+            "misconceptions": ["off-by-one", "  ", "wrong return type"],
+            "weak_areas": ["recursion"],
+        })
+        assert out is not None
+        assert "off-by-one" in out
+        assert "wrong return type" in out
+        assert "recursion" in out
+
+    def test_dict_with_no_useful_content_returns_none(self):
+        assert _coerce_summary({"unrelated": 42}) is None
+
+
+# ── read_recent_quiz_attempts (I/O) ──────────────────────────────────────────
+
+
+def _table_factory(*, context_json=None, attempt_rows=None):
+    """Build a side_effect for `table()` returning per-table mocks."""
+    attempt_rows = attempt_rows or []
+
+    def factory(name: str):
+        mock = MagicMock()
+        if name == "quiz_context":
+            mock.select.return_value = (
+                [{"context_json": context_json}] if context_json is not None else []
+            )
+        elif name == "quiz_attempts":
+            mock.select.return_value = attempt_rows
+        else:
+            mock.select.return_value = []
+        return mock
+
+    return factory
+
+
+class TestReadRecentQuizAttempts:
+    def test_no_history_returns_empty_history(self):
+        with patch(
+            "agents.tools.quiz_history.table",
+            side_effect=_table_factory(),
+        ):
+            history = asyncio.run(
+                read_recent_quiz_attempts("user_andres", "node1")
+            )
+
+        assert isinstance(history, QuizHistory)
+        assert history.summary is None
+        assert history.recent_attempts == []
+
+    def test_summary_from_quiz_context_is_returned(self):
+        with patch(
+            "agents.tools.quiz_history.table",
+            side_effect=_table_factory(
+                context_json={"summary": "tends to confuse for and while loops"}
+            ),
+        ):
+            history = asyncio.run(
+                read_recent_quiz_attempts("user_andres", "node1")
+            )
+
+        assert history.summary == "tends to confuse for and while loops"
+
+    def test_attempts_are_mapped_with_accuracy(self):
+        rows = [
+            {
+                "score": 4,
+                "total": 5,
+                "difficulty": "medium",
+                "completed_at": "2026-05-03T20:00:00Z",
+            },
+            {
+                "score": 1,
+                "total": 5,
+                "difficulty": "hard",
+                "completed_at": "2026-05-02T20:00:00Z",
+            },
+        ]
+        with patch(
+            "agents.tools.quiz_history.table",
+            side_effect=_table_factory(attempt_rows=rows),
+        ):
+            history = asyncio.run(
+                read_recent_quiz_attempts("user_andres", "node1")
+            )
+
+        assert len(history.recent_attempts) == 2
+        first = history.recent_attempts[0]
+        assert isinstance(first, RecentQuizAttempt)
+        assert first.score == 4
+        assert first.total == 5
+        assert first.difficulty == "medium"
+        assert first.accuracy == pytest.approx(0.8)
+        # Second attempt: 1/5 = 0.2
+        assert history.recent_attempts[1].accuracy == pytest.approx(0.2)
+
+    def test_attempts_with_zero_or_missing_total_are_skipped(self):
+        rows = [
+            {"score": 0, "total": 0, "difficulty": "easy", "completed_at": "x"},
+            {"score": None, "total": 5, "difficulty": "easy", "completed_at": "y"},
+            {"score": 3, "total": 5, "difficulty": "medium", "completed_at": "z"},
+        ]
+        with patch(
+            "agents.tools.quiz_history.table",
+            side_effect=_table_factory(attempt_rows=rows),
+        ):
+            history = asyncio.run(
+                read_recent_quiz_attempts("user_andres", "node1")
+            )
+
+        # The total=0 row drops; the score=None row coerces to 0/5 → 0.0
+        # and stays. The valid 3/5 row stays. Two valid rows total.
+        assert len(history.recent_attempts) == 2
+        difficulties = [a.difficulty for a in history.recent_attempts]
+        assert "easy" in difficulties  # the score=None one
+        assert "medium" in difficulties
+
+    def test_accuracy_is_clamped_to_unit_interval(self):
+        # Pathological row (score > total). Shouldn't blow the model
+        # validator; clamp to 1.0.
+        rows = [{"score": 7, "total": 5, "difficulty": "hard", "completed_at": "z"}]
+        with patch(
+            "agents.tools.quiz_history.table",
+            side_effect=_table_factory(attempt_rows=rows),
+        ):
+            history = asyncio.run(
+                read_recent_quiz_attempts("user_andres", "node1")
+            )
+        assert history.recent_attempts[0].accuracy == 1.0
+
+    def test_filters_passed_to_quiz_attempts_select(self):
+        captured: dict = {}
+
+        def factory(name: str):
+            mock = MagicMock()
+            if name == "quiz_attempts":
+                def select(*args, **kwargs):
+                    captured["filters"] = kwargs.get("filters")
+                    captured["order"] = kwargs.get("order")
+                    captured["limit"] = kwargs.get("limit")
+                    return []
+                mock.select.side_effect = select
+            elif name == "quiz_context":
+                mock.select.return_value = []
+            else:
+                mock.select.return_value = []
+            return mock
+
+        with patch("agents.tools.quiz_history.table", side_effect=factory):
+            asyncio.run(read_recent_quiz_attempts("user_andres", "node1"))
+
+        # Filters must scope to this user + concept, and exclude
+        # in-flight attempts (completed_at IS NOT NULL). Order newest
+        # first so adaptive-difficulty math reads recency correctly.
+        assert captured["filters"]["user_id"] == "eq.user_andres"
+        assert captured["filters"]["concept_node_id"] == "eq.node1"
+        assert captured["filters"]["completed_at"] == "not.is.null"
+        assert captured["order"] == "completed_at.desc"
+        assert captured["limit"] == 5
+
+    def test_db_error_degrades_to_empty(self):
+        def factory(name: str):
+            mock = MagicMock()
+            mock.select.side_effect = RuntimeError("connection reset")
+            return mock
+
+        with patch("agents.tools.quiz_history.table", side_effect=factory):
+            history = asyncio.run(
+                read_recent_quiz_attempts("user_andres", "node1")
+            )
+
+        # Failure must NOT propagate — the agent can still generate a
+        # quiz without history; it just won't be adaptive.
+        assert history.summary is None
+        assert history.recent_attempts == []
diff --git a/docs/decisions/0014-adaptive-quiz-iteration.md b/docs/decisions/0014-adaptive-quiz-iteration.md
new file mode 100644
index 00000000..42043489
--- /dev/null
+++ b/docs/decisions/0014-adaptive-quiz-iteration.md
@@ -0,0 +1,129 @@
+# 0014: Adaptive quiz iteration — spaced repetition + difficulty + history
+
+- Status: accepted
+- Date: 2026-05-03
+- Refines: 0005, 0013
+
+## Context
+
+ADR 0013 closed out refactor #2 with three known gaps called out under
+"What it doesn't do (yet)":
+
+1. No spaced repetition. The agent biased toward weakest-mastery
+   concepts but ignored `last_studied_at`, so a stale 0.85-mastery
+   concept never got revisited until it decayed.
+2. No within-session adaptive difficulty. The agent honored the
+   user-requested `difficulty` literally, even after the student had
+   bombed the last three attempts on the same concept.
+3. No quiz-attempt history on the agent path. The legacy fallback
+   path read `quiz_context_service.get_quiz_context` and stuffed it
+   into the prompt template; the agent path didn't see it at all, so
+   the agent couldn't write distractors that mirrored *this* student's
+   prior errors.
+
+This ADR records the small follow-up that closes those three gaps
+without disturbing the wire format, the fallback contract, or the
+agent's output schema.
+
+## Decision
+
+Add one new tool, `read_recent_quiz_attempts(concept_node_id)`, and
+update the quiz agent's system prompt to (a) weight `last_reviewed_at`
+when picking concepts (spaced repetition) and (b) modulate the
+difficulty mix based on `recent_attempts.accuracy` (adaptive
+difficulty). Tool registration order is intentional: graph reads first
+(weakest-first concept list), then class misconceptions, then this
+student's history on the target concept.
+
+Prompt version bumps from `17ab80b30316` (refactor #2 ship) to
+`358613666dbc`. Logfire traces continue to tag every quiz run with the
+active version.
+
+## What shipped
+
+- `backend/agents/tools/quiz_history.py` — `read_recent_quiz_attempts`
+  pure-async + `_tool` wrapper. Returns `QuizHistory(summary,
+  recent_attempts)`:
+  - `summary` is the LLM-generated digest from `quiz_context`
+    (the rolling per-(user, concept) notes service —
+    `_coerce_summary` accepts the legacy string shape, the
+    `{summary: ...}` dict shape, and the `{misconceptions, weak_areas}`
+    fallback shape, so prompt-version drift on the post-submit
+    background job doesn't break this read).
+  - `recent_attempts` is the last 5 *completed* `quiz_attempts` rows
+    (newest first), with `accuracy = score/total` precomputed.
+    `completed_at = NOT NULL` filter excludes the in-flight row that
+    `routes/quiz.py:generate_quiz` writes before submission.
+- `backend/agents/quiz.py` — registers the new tool, expands the
+  system prompt with explicit spaced-repetition + adaptive-difficulty
+  rules. Concept-selection rules now combine three signals (mastery,
+  staleness, recent accuracy); difficulty rules are bounded to one
+  step in either direction so the agent can't override the user's
+  requested difficulty by more than that.
+- `backend/routes/quiz.py` — `_quiz_via_agent` user message now nudges
+  the agent to call the new tool with the target concept_node_id.
+  No wire-format change.
+- `backend/tests/test_quiz_history_tool.py` — pins shape coercion,
+  accuracy math, the `completed_at IS NOT NULL` filter, and the
+  silent-degrade-on-DB-error contract (failures must not propagate;
+  the agent can still generate a quiz without history, just less
+  adaptive).
+- `backend/tests/test_quiz_agent_imports.py` — extended to assert the
+  new tool is registered.
+
+## Why this isn't a fourth refactor
+
+The refactor #2 plan in ADR 0005 explicitly carved out a path for
+"adaptive quiz history" as a future iteration on the same agent —
+not a separate refactor. This ships under that carve-out:
+
+- No new agent. `quiz_agent` gains one tool and a longer prompt.
+- No new route. `_quiz_via_agent`'s user message is the only change
+  on the route side.
+- No wire-format change. `_agent_question_to_wire` is untouched, so
+  `submitQuiz` / `scoreQuiz` flows on the frontend are unaffected.
+- No fallback-contract change. The legacy `_legacy_generate_quiz`
+  path already reads `quiz_context_service`, so its behavior is
+  unchanged.
+
+If the agent's adaptive behavior turns out to be wrong (too
+aggressive, ignores the rules, etc.), rollback is one tool removal +
+one prompt revert.
+
+## What we deliberately didn't do
+
+- **Decay-formula spaced repetition.** The prompt uses "older than
+  ~7 days" as a soft signal rather than a Leitner / SM-2 style
+  formula. We don't have enough data yet to tune a curve, and the
+  agent's "stale concepts get revived" intent is the load-bearing
+  property — the exact threshold can move.
+- **Cross-concept session history.** The new tool scopes to one
+  `concept_node_id` (the target). A future iteration could surface
+  cross-concept patterns ("student keeps confusing recursion with
+  iteration") — but that needs a new aggregation layer, not just a
+  read.
+- **Tool consolidation.** We considered merging `read_concepts_for_user`
+  + `read_recent_quiz_attempts` into one fat tool. Kept them split:
+  the concept list is course-wide (one call covers the whole quiz),
+  history is concept-scoped (one call per target). Different shapes,
+  different cardinalities — splitting keeps each tool's contract
+  small and testable.
+
+## Eval coverage
+
+The existing replay-mode eval set in `tests/evals/quiz_generation.py`
+exercises the agent end-to-end. Adaptive-difficulty + spaced-
+repetition behaviors are inherently prompt-driven (the LLM decides
+how aggressively to apply them), so unit tests pin only the tool's
+I/O contract. Live-mode evals (run with `SAPLING_EVAL_MODE=live`) are
+the right place to catch prompt regressions — the unit tests don't
+try.
+
+## Rollback
+
+Single-revert clean: revert the commit and the new tool import +
+registration disappear, the prompt reverts to `17ab80b30316`, and the
+route's user message reverts to the refactor-#2 wording. The
+`quiz_history.py` file is new and pure-leaf (no other module imports
+it), so its presence after rollback is harmless even if the revert
+isn't perfectly clean.

From 801980d1050f59ca1293d156efc610a2544dc7a9 Mon Sep 17 00:00:00 2001
From: Jose Cruz <jose.colorstack@gmail.com>
Date: Mon, 4 May 2026 22:22:29 -0400
Subject: [PATCH 2/5] =?UTF-8?q?fix(quiz):=20address=20PR=20review=20?=
 =?UTF-8?q?=E2=80=94=20eval=20cases,=20drop=20corrupt=20rows,=20prompt=20n?=
 =?UTF-8?q?its?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Closes the four issues flagged on PR #77's self-review.

- tests/evals/quiz_generation.py — adds 2 cases (count: 8 -> 10) and
  2 evaluators that pin the new prompt rules structurally:
  - AdaptiveDifficultyEvaluator: requested vs. produced difficulty
    rank stays within ±1 step. Permits the prompt's allowed adaptive
    shift, flags overshoots.
  - SpacedRepetitionConceptEvaluator: when metadata names a
    `stale_concept`, at least one question must target it.
  Smoke-tested against synthetic Quiz outputs: bounds in → 1.0,
  overshoots → 0.0.
- agents/tools/quiz_history.py — drop rows with score outside
  [0, total] entirely (with a logger.warning) instead of clamping
  accuracy and passing impossible numbers (score=7, total=5) to the
  LLM. Matching test_corrupt_rows_are_dropped replaces the old
  clamp-asserting test.
- agents/tools/quiz_history.py — read_recent_quiz_attempts_tool
  docstring is now LLM-facing ("returns this student's history…")
  instead of engineering-facing. Pydantic AI surfaces this as the
  tool's description to the model.
- routes/quiz.py — _quiz_via_agent user message trimmed to
  routing-only; the workflow + adaptive rules already live in the
  system prompt and don't need to be restated per request.
- docs/decisions/0014-adaptive-quiz-iteration.md — Date corrected
  (2026-05-03 → 2026-05-04). Eval-coverage section updated to
  describe the two new structural sentinels.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 backend/agents/tools/quiz_history.py          |  29 ++++-
 backend/routes/quiz.py                        |  14 +--
 backend/tests/evals/quiz_generation.py        | 110 +++++++++++++++++-
 backend/tests/test_quiz_history_tool.py       |  19 ++-
 .../decisions/0014-adaptive-quiz-iteration.md |  31 +++--
 5 files changed, 176 insertions(+), 27 deletions(-)

diff --git a/backend/agents/tools/quiz_history.py b/backend/agents/tools/quiz_history.py
index 5ff4fcda..2cdb9a6b 100644
--- a/backend/agents/tools/quiz_history.py
+++ b/backend/agents/tools/quiz_history.py
@@ -170,7 +170,21 @@ def _fetch_attempts() -> list[dict[str, Any]]:
             # Skip rows that look incomplete — accuracy is undefined and
             # the agent shouldn't have to guess.
             continue
-        accuracy = max(0.0, min(1.0, score / total))
+        if score < 0 or score > total:
+            # Corrupt row (score outside [0, total]). Drop entirely
+            # rather than passing impossible numbers to the LLM —
+            # `score=7, total=5` would prompt the agent to wonder
+            # whether to trust the data at all.
+            logger.warning(
+                "read_recent_quiz_attempts: dropping corrupt row "
+                "(score=%d outside [0, total=%d]) user=%s concept=%s",
+                score,
+                total,
+                user_id,
+                concept_node_id,
+            )
+            continue
+        accuracy = score / total
         attempts.append(
             RecentQuizAttempt(
                 score=score,
@@ -191,10 +205,13 @@ async def read_recent_quiz_attempts_tool(
     ctx: RunContext[SaplingDeps],
     concept_node_id: str,
 ) -> QuizHistory:
-    """Pydantic AI tool wrapper.
-
-    `concept_node_id` is supplied by the agent (which receives it in
-    the user message). user_id comes from deps so a tool-call can't
-    cross users.
+    """Returns this student's history on one concept: a `summary`
+    string digesting their prior mistakes (mine for distractor
+    inspiration) and `recent_attempts` — the last 5 completed quiz
+    attempts on this concept, newest first, with `accuracy` precomputed
+    so you can apply the adaptive-difficulty rule directly. Empty
+    history on first attempt. Pass the `concept_node_id` from the
+    user message; user identity is taken from context.
     """
+    # user_id comes from ctx.deps so a tool call can't cross users.
     return await read_recent_quiz_attempts(ctx.deps.user_id, concept_node_id)
diff --git a/backend/routes/quiz.py b/backend/routes/quiz.py
index 8bf829b9..ad06b1d6 100644
--- a/backend/routes/quiz.py
+++ b/backend/routes/quiz.py
@@ -166,15 +166,15 @@ async def _quiz_via_agent(
         supabase=None,
         request_id=request_id,
     )
+    # Keep this message routing-only; the workflow + adaptive rules
+    # live in the system prompt. We just hand the agent the inputs it
+    # needs and trust the prompt to drive tool calls.
     user_message = (
         f"Generate {num_questions} {difficulty} questions for the student. "
-        f"The target concept is '{concept_name}' (concept_node_id={concept_node_id}). "
-        f"Call read_concepts_for_user to find the student's weakest and stalest "
-        f"concepts in this course and bias the question mix toward those. "
-        f"Then call read_recent_quiz_attempts(concept_node_id='{concept_node_id}') "
-        f"to see what this student has been getting wrong on this concept and how "
-        f"recent attempts have been scoring — use that to set the difficulty mix "
-        f"adaptively per the system prompt."
+        f"The target concept is '{concept_name}' "
+        f"(concept_node_id={concept_node_id}). Follow the workflow in your "
+        f"system prompt; pass concept_node_id='{concept_node_id}' to "
+        f"read_recent_quiz_attempts."
     )
     if use_shared_context:
         user_message += (
diff --git a/backend/tests/evals/quiz_generation.py b/backend/tests/evals/quiz_generation.py
index b302c30e..4feff16a 100644
--- a/backend/tests/evals/quiz_generation.py
+++ b/backend/tests/evals/quiz_generation.py
@@ -92,6 +92,59 @@ def evaluate(self, ctx: EvaluatorContext[str, Quiz]) -> float:
         return ok / max(1, len(ctx.output.questions))
 
 
+# Difficulty ordering for the adaptive evaluator. Higher index = harder.
+_DIFF_RANK = {"easy": 0, "medium": 1, "hard": 2}
+
+
+@dataclass
+class AdaptiveDifficultyEvaluator(Evaluator[str, Quiz]):
+    """Pin the prompt's adaptive-difficulty bound: the average produced
+    difficulty must be within ±1 step of the user-requested difficulty
+    (in the metadata's `requested_difficulty`). Cases without
+    `requested_difficulty` skip this check.
+
+    The agent is *allowed* to step down (struggling student) or step
+    up (consistent high accuracy) by one rank. This evaluator catches
+    the regression where it overshoots — e.g. requested medium and
+    produced all easy AND all hard. The point is the bound, not the
+    direction.
+    """
+
+    def evaluate(self, ctx: EvaluatorContext[str, Quiz]) -> float:
+        requested = (ctx.metadata or {}).get("requested_difficulty")
+        if requested is None or not ctx.output.questions:
+            return 1.0
+        target_rank = _DIFF_RANK.get(requested)
+        if target_rank is None:
+            return 1.0
+        # Compute average rank across produced questions; the bound
+        # is "within 1 step of requested." This permits the prompt's
+        # one-step adaptive shift in either direction but rejects
+        # anything beyond that.
+        ranks = [_DIFF_RANK.get(q.difficulty, target_rank) for q in ctx.output.questions]
+        avg = sum(ranks) / len(ranks)
+        return 1.0 if abs(avg - target_rank) <= 1.0 else 0.0
+
+
+@dataclass
+class SpacedRepetitionConceptEvaluator(Evaluator[str, Quiz]):
+    """Pin the prompt's spaced-repetition rule: when metadata names a
+    `stale_concept`, at least one question must target it. Cases
+    without `stale_concept` skip this check.
+
+    This is the structural sentinel — it doesn't try to verify that
+    the agent reasoned about `last_reviewed_at` correctly, just that
+    the stale concept didn't get dropped from the question mix entirely
+    in favor of the lowest-mastery one.
+    """
+
+    def evaluate(self, ctx: EvaluatorContext[str, Quiz]) -> float:
+        stale = (ctx.metadata or {}).get("stale_concept")
+        if not stale:
+            return 1.0
+        return 1.0 if any(q.concept == stale for q in ctx.output.questions) else 0.0
+
+
 # ── Cases ───────────────────────────────────────────────────────────────────
 
 # Inputs are deliberately small — Gemini reads the prompt + tool results
@@ -231,9 +284,62 @@ def evaluate(self, ctx: EvaluatorContext[str, Quiz]) -> float:
             "concepts": ["Pumping Lemma", "Decidability"],
         },
     ),
+
+    # ── Adaptive difficulty (struggling student, request hard) ─────────────
+    # Per ADR 0014: when recent_attempts.accuracy is consistently low,
+    # the agent is allowed to drop the difficulty mix one step from what
+    # the user asked. AdaptiveDifficultyEvaluator pins the bound: the
+    # produced average difficulty must stay within ±1 step of requested.
+    # Without this case, a future prompt change that makes the agent
+    # produce all easy questions for a hard request would slip through.
+    Case(
+        name="adaptive_downshift_struggling_student",
+        inputs=(
+            "Course: CS 201. Generate 3 hard multiple-choice questions "
+            "covering Recursion, Dynamic Programming, and Graph Traversal. "
+            "The student has been struggling on this material — recent "
+            "attempts have averaged < 50% accuracy. Apply the adaptive-"
+            "difficulty rule: it's OK to step down to medium where the "
+            "evidence supports it."
+        ),
+        metadata={
+            "expected_count": 3,
+            # Don't pin a single expected_difficulty — the agent may
+            # mix medium/hard under the adaptive rule. The point is
+            # the ±1-step bound captured by requested_difficulty.
+            "expected_difficulty": None,
+            "expected_type": "multiple_choice",
+            "concepts": ["Recursion", "Dynamic Programming", "Graph Traversal"],
+            "requested_difficulty": "hard",
+        },
+    ),
+
+    # ── Spaced repetition (stale concept must be revived) ──────────────────
+    # Per ADR 0014: concepts whose `last_reviewed_at` is older than ~7
+    # days should surface even when their mastery is mid-tier. The case
+    # input names a stale concept explicitly; SpacedRepetitionConcept-
+    # Evaluator asserts at least one question targets it.
+    Case(
+        name="spaced_repetition_revives_stale_concept",
+        inputs=(
+            "Course: BIO 100. Generate 3 medium multiple-choice questions. "
+            "The student has been working on Photosynthesis (mastery 0.4) "
+            "and Mitosis (mastery 0.5) recently, but Cell Membrane "
+            "(mastery 0.65) hasn't been reviewed in 14 days. Per the "
+            "spaced-repetition rule, include at least one Cell Membrane "
+            "question to revive the stale concept."
+        ),
+        metadata={
+            "expected_count": 3,
+            "expected_difficulty": "medium",
+            "expected_type": "multiple_choice",
+            "concepts": ["Photosynthesis", "Mitosis", "Cell Membrane"],
+            "stale_concept": "Cell Membrane",
+        },
+    ),
 ]
 
-assert len(CASES) == 8, f"Expected 8 cases per ADR 0005, got {len(CASES)}"
+assert len(CASES) == 10, f"Expected 10 cases (8 original + 2 ADR 0014), got {len(CASES)}"
 # All cases are MCQ today — short_answer dropped pending frontend support.
 assert all(
     c.metadata and c.metadata.get("expected_type") == "multiple_choice"
@@ -268,6 +374,8 @@ def make_dataset() -> Dataset[str, Quiz]:
             TypeMixEvaluator(),
             MultipleChoiceShapeEvaluator(),
             ConceptCoverageEvaluator(),
+            AdaptiveDifficultyEvaluator(),
+            SpacedRepetitionConceptEvaluator(),
         ],
     )
 
diff --git a/backend/tests/test_quiz_history_tool.py b/backend/tests/test_quiz_history_tool.py
index 0e8d4ecd..e829061a 100644
--- a/backend/tests/test_quiz_history_tool.py
+++ b/backend/tests/test_quiz_history_tool.py
@@ -164,10 +164,16 @@ def test_attempts_with_zero_or_missing_total_are_skipped(self):
         assert "easy" in difficulties  # the score=None one
         assert "medium" in difficulties
 
-    def test_accuracy_is_clamped_to_unit_interval(self):
-        # Pathological row (score > total). Shouldn't blow the model
-        # validator; clamp to 1.0.
-        rows = [{"score": 7, "total": 5, "difficulty": "hard", "completed_at": "z"}]
+    def test_corrupt_rows_are_dropped(self):
+        # Rows where score is outside [0, total] are corrupt — passing
+        # them to the LLM as e.g. "score=7, total=5" would prompt the
+        # agent to wonder whether to trust the data at all. Drop them
+        # entirely. A valid neighbour row in the same response stays.
+        rows = [
+            {"score": 7, "total": 5, "difficulty": "hard", "completed_at": "x"},
+            {"score": -1, "total": 5, "difficulty": "easy", "completed_at": "y"},
+            {"score": 4, "total": 5, "difficulty": "medium", "completed_at": "z"},
+        ]
         with patch(
             "agents.tools.quiz_history.table",
             side_effect=_table_factory(attempt_rows=rows),
@@ -175,7 +181,10 @@ def test_accuracy_is_clamped_to_unit_interval(self):
             history = asyncio.run(
                 read_recent_quiz_attempts("user_andres", "node1")
             )
-        assert history.recent_attempts[0].accuracy == 1.0
+        assert len(history.recent_attempts) == 1
+        assert history.recent_attempts[0].score == 4
+        assert history.recent_attempts[0].total == 5
+        assert history.recent_attempts[0].accuracy == pytest.approx(0.8)
 
     def test_filters_passed_to_quiz_attempts_select(self):
         captured: dict = {}
diff --git a/docs/decisions/0014-adaptive-quiz-iteration.md b/docs/decisions/0014-adaptive-quiz-iteration.md
index 42043489..53a1cecd 100644
--- a/docs/decisions/0014-adaptive-quiz-iteration.md
+++ b/docs/decisions/0014-adaptive-quiz-iteration.md
@@ -1,7 +1,7 @@
 # 0014: Adaptive quiz iteration — spaced repetition + difficulty + history
 
 - Status: accepted
-- Date: 2026-05-03
+- Date: 2026-05-04
 - Refines: 0005, 0013
 
 ## Context
@@ -111,13 +111,28 @@ one prompt revert.
 
 ## Eval coverage
 
-The existing replay-mode eval set in `tests/evals/quiz_generation.py`
-exercises the agent end-to-end. Adaptive-difficulty + spaced-
-repetition behaviors are inherently prompt-driven (the LLM decides
-how aggressively to apply them), so unit tests pin only the tool's
-I/O contract. Live-mode evals (run with `SAPLING_EVAL_MODE=live`) are
-the right place to catch prompt regressions — the unit tests don't
-try.
+`tests/evals/quiz_generation.py` gains 2 cases (count: 8 → 10) and
+2 evaluators that pin the new prompt rules structurally:
+
+- `AdaptiveDifficultyEvaluator` — when a case's metadata sets
+  `requested_difficulty`, the produced questions' average difficulty
+  rank must stay within ±1 step. This permits the prompt's allowed
+  one-step shift in either direction (down for struggling students,
+  up for consistent high accuracy) but flags overshoots.
+- `SpacedRepetitionConceptEvaluator` — when a case's metadata sets
+  `stale_concept`, at least one question must target that concept.
+  Catches a regression where the agent drops the stale concept
+  entirely in favor of the lowest-mastery one.
+
+Unit tests in `tests/test_quiz_history_tool.py` pin only the tool's
+I/O contract (shape coercion, accuracy math, filter wiring, silent
+degrade on DB error). The agent's *application* of the new rules is
+prompt-driven — the eval cases above are the regression sentinel.
+
+Cassettes for the new cases get written on the next
+`SAPLING_EVAL_MODE=record` run; replay-mode CI continues to fail
+loudly when a cassette is missing, so neither the new nor existing
+quiz cases silently no-op.
 
 ## Rollback
 

From 6493988bea00a14c417b18dc6a358fd28ac778d4 Mon Sep 17 00:00:00 2001
From: Jose Cruz <jose.colorstack@gmail.com>
Date: Mon, 4 May 2026 22:27:59 -0400
Subject: [PATCH 3/5] fix(quiz): tighten AdaptiveDifficultyEvaluator + clarify
 eval semantics
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Closes the two issues from PR #77's second review.

- AdaptiveDifficultyEvaluator now scores per-question (fraction
  compliant) instead of average rank. The prompt rule is per-question
  ("Never override the user-requested difficulty by more than one
  step"); the previous avg-based check let a 2-step outlier slip
  through if the rest of the mix balanced it out — e.g. requested
  hard with [easy, hard, hard, hard] used to score 1.0 (avg=1.5,
  within 1 of target=2) and now correctly scores 0.75. Switched to
  subscript _DIFF_RANK[q.difficulty] (Literal-constrained, fallback
  was unreachable).
- The two new ADR-0014 cases now carry NOTE comments explaining that
  recency/staleness state is baked into the user message for replay
  determinism, while production sources it via read_recent_quiz_attempts.
  The cases pin the prompt's rule application; live-mode evals are
  the right place to catch tool-wiring regressions.

Smoke-tested: hard + [easy, hard, hard, hard] -> 0.75; hard + all
medium -> 1.0 (allowed shift); hard + all easy -> 0.0 (overshoot).
46 quiz unit tests still pass.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 backend/tests/evals/quiz_generation.py | 39 ++++++++++++++++----------
 1 file changed, 24 insertions(+), 15 deletions(-)

diff --git a/backend/tests/evals/quiz_generation.py b/backend/tests/evals/quiz_generation.py
index 4feff16a..88342542 100644
--- a/backend/tests/evals/quiz_generation.py
+++ b/backend/tests/evals/quiz_generation.py
@@ -98,16 +98,15 @@ def evaluate(self, ctx: EvaluatorContext[str, Quiz]) -> float:
 
 @dataclass
 class AdaptiveDifficultyEvaluator(Evaluator[str, Quiz]):
-    """Pin the prompt's adaptive-difficulty bound: the average produced
-    difficulty must be within ±1 step of the user-requested difficulty
-    (in the metadata's `requested_difficulty`). Cases without
-    `requested_difficulty` skip this check.
+    """Fraction of questions whose difficulty is within ±1 step of the
+    user-requested difficulty (in the metadata's `requested_difficulty`).
+    Cases without `requested_difficulty` skip this check.
 
     The agent is *allowed* to step down (struggling student) or step
-    up (consistent high accuracy) by one rank. This evaluator catches
-    the regression where it overshoots — e.g. requested medium and
-    produced all easy AND all hard. The point is the bound, not the
-    direction.
+    up (consistent high accuracy) by one rank per question. This
+    evaluator catches the regression where individual questions
+    overshoot — e.g. requested hard and produced an easy question
+    (a two-step jump). The bound is per-question, not averaged.
     """
 
     def evaluate(self, ctx: EvaluatorContext[str, Quiz]) -> float:
@@ -117,13 +116,12 @@ def evaluate(self, ctx: EvaluatorContext[str, Quiz]) -> float:
         target_rank = _DIFF_RANK.get(requested)
         if target_rank is None:
             return 1.0
-        # Compute average rank across produced questions; the bound
-        # is "within 1 step of requested." This permits the prompt's
-        # one-step adaptive shift in either direction but rejects
-        # anything beyond that.
-        ranks = [_DIFF_RANK.get(q.difficulty, target_rank) for q in ctx.output.questions]
-        avg = sum(ranks) / len(ranks)
-        return 1.0 if abs(avg - target_rank) <= 1.0 else 0.0
+        ok = sum(
+            1
+            for q in ctx.output.questions
+            if abs(_DIFF_RANK[q.difficulty] - target_rank) <= 1
+        )
+        return ok / len(ctx.output.questions)
 
 
 @dataclass
@@ -294,6 +292,11 @@ def evaluate(self, ctx: EvaluatorContext[str, Quiz]) -> float:
     # produce all easy questions for a hard request would slip through.
     Case(
         name="adaptive_downshift_struggling_student",
+        # NOTE: this case bakes the recent-accuracy signal into the user
+        # message for eval determinism. In production, the agent reads it
+        # via a `read_recent_quiz_attempts` tool call (see agents/quiz.py).
+        # This pins the prompt's adaptive-difficulty rule, not the tool-
+        # call data path — live-mode evals catch tool wiring regressions.
         inputs=(
             "Course: CS 201. Generate 3 hard multiple-choice questions "
             "covering Recursion, Dynamic Programming, and Graph Traversal. "
@@ -321,6 +324,12 @@ def evaluate(self, ctx: EvaluatorContext[str, Quiz]) -> float:
     # Evaluator asserts at least one question targets it.
     Case(
         name="spaced_repetition_revives_stale_concept",
+        # NOTE: the staleness signal (`last_reviewed_at` ages, mastery
+        # scores) is inlined into the prompt here for deterministic
+        # replay. Production sources the same data from a
+        # `read_recent_quiz_attempts` tool call (see agents/quiz.py), so
+        # this case validates the spaced-repetition rule's application,
+        # not the tool-call wiring. Use live-mode evals for that path.
         inputs=(
             "Course: BIO 100. Generate 3 medium multiple-choice questions. "
             "The student has been working on Photosynthesis (mastery 0.4) "

From 6453e954bcefd68c25d61bd1e4ace1403078c2e7 Mon Sep 17 00:00:00 2001
From: Jose Cruz <jose.colorstack@gmail.com>
Date: Mon, 4 May 2026 22:32:23 -0400
Subject: [PATCH 4/5] docs(quiz): align stale eval-section comment with
 per-question evaluator

The section comment above adaptive_downshift_struggling_student
described the old average-based scoring; commit 6493988 rewrote the
evaluator to per-question fraction but missed this comment. Bring the
prose in line with the code so future readers don't trust an outdated
description.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 backend/tests/evals/quiz_generation.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/backend/tests/evals/quiz_generation.py b/backend/tests/evals/quiz_generation.py
index 88342542..a7cd4565 100644
--- a/backend/tests/evals/quiz_generation.py
+++ b/backend/tests/evals/quiz_generation.py
@@ -286,10 +286,11 @@ def evaluate(self, ctx: EvaluatorContext[str, Quiz]) -> float:
     # ── Adaptive difficulty (struggling student, request hard) ─────────────
     # Per ADR 0014: when recent_attempts.accuracy is consistently low,
     # the agent is allowed to drop the difficulty mix one step from what
-    # the user asked. AdaptiveDifficultyEvaluator pins the bound: the
-    # produced average difficulty must stay within ±1 step of requested.
-    # Without this case, a future prompt change that makes the agent
-    # produce all easy questions for a hard request would slip through.
+    # the user asked. AdaptiveDifficultyEvaluator scores the fraction of
+    # questions whose difficulty is within ±1 step of requested
+    # (per-question, not averaged). Without this case, a future prompt
+    # change that makes the agent produce all easy questions for a hard
+    # request would slip through.
     Case(
         name="adaptive_downshift_struggling_student",
         # NOTE: this case bakes the recent-accuracy signal into the user

From e2112a8223fab1462354a54bdbfe0181ad67846e Mon Sep 17 00:00:00 2001
From: Jose Cruz <jose.colorstack@gmail.com>
Date: Mon, 4 May 2026 22:37:34 -0400
Subject: [PATCH 5/5] fix(quiz): drop rows with null score/total instead of
 coercing to 0

submit_quiz writes score+total atomically (routes/quiz.py:426), so any
row with completed_at IS NOT NULL but a null score or null total is
corruption. The previous coercion (`r.get('score') or 0`) silently
turned that into a 0/5 = 0% accuracy reading, which the LLM could
trust and use to trigger a spurious adaptive downshift on a perfectly
healthy concept.

Drop those rows alongside the existing score-out-of-bounds drop
branch (with the same logger.warning treatment) and tighten the test
to cover null score, null total, and total=0 in one sweep.

Surfaced by an independent code-reviewer pass on PR #77.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 backend/agents/tools/quiz_history.py    | 21 +++++++++++++++++++--
 backend/tests/test_quiz_history_tool.py | 19 ++++++++++++-------
 2 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/backend/agents/tools/quiz_history.py b/backend/agents/tools/quiz_history.py
index 2cdb9a6b..69ad3672 100644
--- a/backend/agents/tools/quiz_history.py
+++ b/backend/agents/tools/quiz_history.py
@@ -161,9 +161,26 @@ def _fetch_attempts() -> list[dict[str, Any]]:
 
     attempts: list[RecentQuizAttempt] = []
     for r in attempt_rows:
+        raw_score = r.get("score")
+        raw_total = r.get("total")
+        if raw_score is None or raw_total is None:
+            # `submit_quiz` writes score+total atomically, so a row with
+            # `completed_at IS NOT NULL` but a null score/total is
+            # corruption (or an out-of-band edit). Drop it rather than
+            # coercing to 0/0 — feeding the LLM a bogus 0% accuracy
+            # could trigger a spurious adaptive downshift.
+            logger.warning(
+                "read_recent_quiz_attempts: dropping row with null "
+                "score/total (score=%r, total=%r) user=%s concept=%s",
+                raw_score,
+                raw_total,
+                user_id,
+                concept_node_id,
+            )
+            continue
         try:
-            score = int(r.get("score") or 0)
-            total = int(r.get("total") or 0)
+            score = int(raw_score)
+            total = int(raw_total)
         except (TypeError, ValueError):
             continue
         if total <= 0:
diff --git a/backend/tests/test_quiz_history_tool.py b/backend/tests/test_quiz_history_tool.py
index e829061a..aeafdd96 100644
--- a/backend/tests/test_quiz_history_tool.py
+++ b/backend/tests/test_quiz_history_tool.py
@@ -143,10 +143,16 @@ def test_attempts_are_mapped_with_accuracy(self):
         # Second attempt: 1/5 = 0.2
         assert history.recent_attempts[1].accuracy == pytest.approx(0.2)
 
-    def test_attempts_with_zero_or_missing_total_are_skipped(self):
+    def test_attempts_with_zero_or_null_fields_are_skipped(self):
+        # submit_quiz writes score+total atomically, so any row with
+        # null score, null total, or total=0 is corruption — drop it
+        # rather than passing the LLM a bogus 0% accuracy that could
+        # trigger a spurious adaptive downshift. The valid 3/5 row
+        # stays.
         rows = [
             {"score": 0, "total": 0, "difficulty": "easy", "completed_at": "x"},
             {"score": None, "total": 5, "difficulty": "easy", "completed_at": "y"},
+            {"score": 3, "total": None, "difficulty": "easy", "completed_at": "y2"},
             {"score": 3, "total": 5, "difficulty": "medium", "completed_at": "z"},
         ]
         with patch(
@@ -157,12 +163,11 @@ def test_attempts_with_zero_or_missing_total_are_skipped(self):
                 read_recent_quiz_attempts("user_andres", "node1")
             )
 
-        # The total=0 row drops; the score=None row coerces to 0/5 → 0.0
-        # and stays. The valid 3/5 row stays. Two valid rows total.
-        assert len(history.recent_attempts) == 2
-        difficulties = [a.difficulty for a in history.recent_attempts]
-        assert "easy" in difficulties  # the score=None one
-        assert "medium" in difficulties
+        assert len(history.recent_attempts) == 1
+        kept = history.recent_attempts[0]
+        assert kept.score == 3
+        assert kept.total == 5
+        assert kept.difficulty == "medium"
 
     def test_corrupt_rows_are_dropped(self):
         # Rows where score is outside [0, total] are corrupt — passing