diff --git a/backend/db/migrations/20260812214402_quiz_responses.sql b/backend/db/migrations/20260812214402_quiz_responses.sql new file mode 100644 index 00000000..afd0a8d8 --- /dev/null +++ b/backend/db/migrations/20260812214402_quiz_responses.sql @@ -0,0 +1,30 @@ +-- #541 C2: per-question quiz responses. This is the table that makes item +-- statistics, timing analysis, and misconception mining possible — one row +-- per answered question, written by POST /api/quiz/attempts/{id}/answer. +-- +-- Everything here is a plaintext analytics scalar (same rationale as +-- quiz_attempts.score/total in #521): indexes and booleans carry no student +-- free text. The question/option TEXT lives encrypted in +-- quiz_attempts.questions_json; this table references it only by position. +-- +-- The UNIQUE is the C1 idempotency contract: one response per +-- (attempt, question); re-answering returns the first recorded response +-- (no revision — the #537 revamp decides if that changes). + +CREATE TABLE quiz_responses ( + id TEXT PRIMARY KEY DEFAULT gen_random_uuid()::text, + attempt_id TEXT NOT NULL REFERENCES quiz_attempts(id) ON DELETE CASCADE, + question_index INTEGER NOT NULL CHECK (question_index >= 0), + selected_index INTEGER NOT NULL CHECK (selected_index >= 0), + is_correct BOOLEAN NOT NULL, + time_ms INTEGER CHECK (time_ms >= 0), + confidence REAL CHECK (confidence >= 0.0 AND confidence <= 1.0), + answered_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT quiz_responses_attempt_question_key UNIQUE (attempt_id, question_index) +); + +-- No separate attempt_id index: Postgres backs the UNIQUE above with a btree +-- whose LEADING column is attempt_id, which already serves both access +-- patterns (submit's per-attempt scan and the answer endpoint's +-- (attempt_id, question_index) lookup). A standalone index would just add a +-- second write to the per-answer hot path. diff --git a/backend/models/__init__.py b/backend/models/__init__.py index 9faa4d68..5bd25a23 100644 --- a/backend/models/__init__.py +++ b/backend/models/__init__.py @@ -60,6 +60,31 @@ class GenerateQuizBody(BaseModel): # through to whatever SAPLING_MODEL_QUIZ resolves to (default # gemini-2.5-flash-lite per ADR 0008). model_pref: Optional[Literal["fast", "smart"]] = None + # DEPRECATED (#541 C3, removal tracked in #546): when true (the default + # the current QuizPanel needs), the response's per-option dicts carry + # `correct` booleans — the full answer key, client-side. Removing the + # key is a hard requirement of the #537 revamp: the new client grades + # through POST /api/quiz/attempts/{id}/answer instead. Every keyed + # response is logged so we can see when usage reaches zero. + include_answer_key: bool = True + + +class AnswerQuestionBody(BaseModel): + """One answer for POST /api/quiz/attempts/{attempt_id}/answer (#541 C1). + + Indexes are 0-based positions into the attempt's stored questions and + the question's options. The wire questions carry 1-based `id`s (what + /submit keys on), so `question_index = id - 1` — two addressing schemes + for one question. `question_id` is the guard against confusing them: + send the id you displayed and the route rejects a mismatch instead of + silently grading the neighbouring question (which the idempotency rule + would then lock in). The response echoes both either way.""" + + question_index: int = Field(ge=0) + selected_index: int = Field(ge=0) + question_id: Optional[int] = None + time_ms: Optional[int] = Field(default=None, ge=0) + confidence: Optional[float] = Field(default=None, ge=0.0, le=1.0) class AnswerItem(BaseModel): diff --git a/backend/routes/quiz.py b/backend/routes/quiz.py index a5a90d0b..7c880420 100644 --- a/backend/routes/quiz.py +++ b/backend/routes/quiz.py @@ -18,7 +18,7 @@ from agents.quiz_context import quiz_context_agent from agents.usage import record_agent_usage from db.connection import table -from models import GenerateQuizBody, SubmitQuizBody +from models import AnswerQuestionBody, GenerateQuizBody, SubmitQuizBody from routes.learn import _get_catalog_chunk from services import events_service from services.auth_guard import require_self @@ -78,6 +78,21 @@ def _load_prompt(name: str) -> str: _DIFFICULTY_RANK = {d: i for i, d in enumerate(CONCRETE_DIFFICULTIES)} +def _strip_answer_key(wire_questions: list[dict]) -> list[dict]: + """Deep-enough copy of the wire questions without per-option `correct` + booleans (#541 C3). Storage always keeps the key — grading is + server-side; only the RESPONSE is stripped.""" + stripped = [] + for q in wire_questions: + q2 = dict(q) + q2["options"] = [ + {k: v for k, v in o.items() if k != "correct"} + for o in q.get("options", []) + ] + stripped.append(q2) + return stripped + + def _resolved_difficulty(wire_questions: list[dict]) -> str: """The overall difficulty generation actually produced (#540 A1). @@ -441,6 +456,20 @@ async def generate_quiz(body: GenerateQuizBody, request: Request): "difficulty": body.difficulty, }, ) + # #541 C3: the answer key (per-option `correct` booleans) ships to the + # client only behind the deprecated include_answer_key flag — default + # true for the current QuizPanel, removed with #546 once the #537 + # client grades via /attempts/{id}/answer. Log every keyed response so + # zero-usage is observable before the default flips. + if body.include_answer_key: + logger.info( + "quiz: generate served the client-side answer key " + "(include_answer_key=true, deprecated — #546) quiz_id=%s", quiz_id, + ) + response_questions = questions + else: + response_questions = _strip_answer_key(questions) + # #540 A1: echo what generation actually chose. requested_difficulty # is what the student asked for (may be 'adaptive'); # resolved_difficulty is the overall mix the agent produced (always @@ -448,12 +477,132 @@ async def generate_quiz(body: GenerateQuizBody, request: Request): # of repeating the request back. return { "quiz_id": quiz_id, - "questions": questions, + "questions": response_questions, "requested_difficulty": body.difficulty, "resolved_difficulty": _resolved_difficulty(questions), } +@router.post("/attempts/{attempt_id}/answer") +def answer_question(attempt_id: str, body: AnswerQuestionBody, request: Request): + """#541 C1: grade one question server-side and record the response. + + Idempotent on (attempt_id, question_index): re-answering returns the + FIRST recorded response (`recorded: false` marks the replay) rather + than overwriting — no revision, decided for the #537 revamp flow. + """ + attempt_rows = table("quiz_attempts").select( + "*", filters={"id": f"eq.{attempt_id}"} + ) + if not attempt_rows: + raise QuizAPIError( + status_code=404, + code=QuizErrorCode.QUIZ_ATTEMPT_NOT_FOUND, + message="We couldn't find that quiz.", + ) + attempt = attempt_rows[0] + require_self(attempt["user_id"], request) + + if attempt.get("completed_at"): + raise QuizAPIError( + status_code=409, + code=QuizErrorCode.QUIZ_ATTEMPT_ALREADY_COMPLETED, + message="This quiz has already been submitted.", + ) + + questions = decrypt_json_column(attempt["questions_json"]) or [] + if body.question_index >= len(questions): + raise QuizAPIError( + status_code=400, + code=QuizErrorCode.QUIZ_QUESTION_INVALID, + message="That question isn't part of this quiz.", + ) + question = questions[body.question_index] + options = question.get("options", []) + if body.selected_index >= len(options): + raise QuizAPIError( + status_code=400, + code=QuizErrorCode.QUIZ_QUESTION_INVALID, + message="That answer choice isn't part of this question.", + ) + # Wire ids are 1-based, question_index is 0-based. When the client sends + # both, they must agree — otherwise passing the displayed id as the index + # silently grades the NEXT question and idempotency locks that in. + if body.question_id is not None and body.question_id != question.get("id"): + raise QuizAPIError( + status_code=400, + code=QuizErrorCode.QUIZ_QUESTION_INVALID, + message="That answer doesn't match the question it was sent for.", + ) + + # The correct option is a property of the question, not of the answer — + # resolve it once. -1 means a malformed item with no correct option, + # which must never grade correct (same rule as submit's #129 fix). + correct_index = next( + (i for i, o in enumerate(options) if o.get("correct")), -1 + ) + + def _is_correct(selected_index: int) -> bool: + return correct_index >= 0 and correct_index == selected_index + + recorded = True + response_row = None + existing = table("quiz_responses").select( + "*", + filters={ + "attempt_id": f"eq.{attempt_id}", + "question_index": f"eq.{body.question_index}", + }, + ) + if existing: + recorded = False + response_row = existing[0] + else: + row = { + "attempt_id": attempt_id, + "question_index": body.question_index, + "selected_index": body.selected_index, + "is_correct": _is_correct(body.selected_index), + "time_ms": body.time_ms, + "confidence": body.confidence, + } + try: + table("quiz_responses").insert(row) + response_row = row + except Exception: + # Lost a race with a concurrent answer for the same index — the + # UNIQUE arbitrates; return whatever won. + recorded = False + raced = table("quiz_responses").select( + "*", + filters={ + "attempt_id": f"eq.{attempt_id}", + "question_index": f"eq.{body.question_index}", + }, + ) + if not raced: + raise + response_row = raced[0] + + next_index = body.question_index + 1 + next_question = ( + _strip_answer_key([questions[next_index]])[0] + if next_index < len(questions) + else None + ) + return { + # Echo both addressing schemes so a client that mixed them up sees + # it immediately rather than discovering it at submit time. + "question_index": body.question_index, + "question_id": question.get("id"), + "is_correct": _is_correct(response_row["selected_index"]), + "correct_index": correct_index, + "explanation": question.get("explanation", ""), + "next_question": next_question, + "recorded": recorded, + } + + @router.post("/submit") def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request: Request): attempt_rows = table("quiz_attempts").select("*", filters={"id": f"eq.{body.quiz_id}"}) @@ -503,12 +652,45 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request concept_node_id = attempt["concept_node_id"] + # #541 C4: responses recorded through /attempts/{id}/answer are the + # source of truth — a payload answer for the same question is ignored + # (the recorded response was graded at answer time; letting the final + # POST override it would reopen the client-side-grading hole C exists + # to close). Questions never answered through C1 fall back to the + # submitted payload, so the current all-at-the-end client keeps working. + recorded_rows = table("quiz_responses").select( + "question_index,selected_index", + filters={"attempt_id": f"eq.{body.quiz_id}"}, + ) or [] + recorded_by_index = {r["question_index"]: r for r in recorded_rows} + answer_map = {str(a.question_id): a.selected_label for a in body.answers} results = [] + # The reconciled answer set — what was ACTUALLY graded, which is what + # answers_json must persist. Storing the raw payload instead left a + # recorded-only submit with a full score beside an empty answer list, + # and a contradicted payload answer stored despite losing to the + # recorded response. + graded_answers: list[dict] = [] score = 0 - for q in questions: + for q_index, q in enumerate(questions): qid = str(q["id"]) - selected = answer_map.get(qid, "") + recorded = recorded_by_index.get(q_index) + if recorded is not None: + sel_idx = recorded.get("selected_index") + options = q.get("options", []) + selected = ( + options[sel_idx]["label"] + if isinstance(sel_idx, int) and 0 <= sel_idx < len(options) + else "" + ) + else: + selected = answer_map.get(qid, "") + if selected: + graded_answers.append({ + "question_id": q["id"], + "selected_label": selected, + }) correct_opt = next((o for o in q["options"] if o.get("correct")), None) correct_label = correct_opt["label"] if correct_opt else "" # #129: a malformed item with NO correct option must never grade as @@ -580,7 +762,10 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request { "score": score, "total": total, - "answers_json": encrypt_json([a.model_dump() for a in body.answers]), + # The reconciled set (recorded responses winning over payload), + # not the raw request — the attempt's stored answers must agree + # with the score computed from them. + "answers_json": encrypt_json(graded_answers), # completed_at was already stamped by the atomic claim above. }, filters={"id": f"eq.{body.quiz_id}"}, diff --git a/backend/services/quiz_errors.py b/backend/services/quiz_errors.py index c297b9e0..7a2b15d1 100644 --- a/backend/services/quiz_errors.py +++ b/backend/services/quiz_errors.py @@ -34,6 +34,9 @@ class QuizErrorCode(str, Enum): QUIZ_CONCEPT_NOT_FOUND = "QUIZ_CONCEPT_NOT_FOUND" QUIZ_ATTEMPT_NOT_FOUND = "QUIZ_ATTEMPT_NOT_FOUND" QUIZ_ATTEMPT_ALREADY_COMPLETED = "QUIZ_ATTEMPT_ALREADY_COMPLETED" + # #541 C1: the answer endpoint got an index that doesn't exist on this + # attempt (question_index past the quiz, selected_index past the options). + QUIZ_QUESTION_INVALID = "QUIZ_QUESTION_INVALID" QUIZ_NOT_AUTHORIZED = "QUIZ_NOT_AUTHORIZED" QUIZ_GENERATION_FAILED = "QUIZ_GENERATION_FAILED" QUIZ_INTERNAL_ERROR = "QUIZ_INTERNAL_ERROR" diff --git a/backend/tests/integration/test_quiz_responses_db.py b/backend/tests/integration/test_quiz_responses_db.py new file mode 100644 index 00000000..a0e6cc7a --- /dev/null +++ b/backend/tests/integration/test_quiz_responses_db.py @@ -0,0 +1,74 @@ +"""#541 C2 real-DB half: quiz_responses storage against local Supabase. + +Proves the migration's shape actually holds in Postgres — the UNIQUE +arbitrates duplicate answers, the FK cascades with the attempt — the +exact class of constraint behavior MagicMock suites cannot see (#529's +lesson). #397 seam: writes through the app layer, raw reads via psycopg. +""" +import uuid + +import pytest + +pytestmark = pytest.mark.integration + +USER = "rich-user-active" + + +def _make_attempt(db_conn) -> str: + from db.connection import table + + node = db_conn.execute( + "SELECT id FROM graph_nodes WHERE user_id = %s ORDER BY id LIMIT 1", + (USER,), + ).fetchone() + assert node is not None + attempt_id = str(uuid.uuid4()) + table("quiz_attempts").insert({ + "id": attempt_id, + "user_id": USER, + "concept_node_id": node["id"], + "difficulty": "adaptive", # also exercises the #540 CHECK widening + "questions_json": [], + }) + return attempt_id + + +def test_unique_arbitrates_duplicate_answers(db_conn): + from db.connection import table + + attempt_id = _make_attempt(db_conn) + table("quiz_responses").insert({ + "attempt_id": attempt_id, "question_index": 0, + "selected_index": 1, "is_correct": True, "time_ms": 1200, + }) + with pytest.raises(Exception): + table("quiz_responses").insert({ + "attempt_id": attempt_id, "question_index": 0, + "selected_index": 0, "is_correct": False, + }) + + rows = db_conn.execute( + "SELECT selected_index, is_correct, time_ms FROM quiz_responses " + "WHERE attempt_id = %s", + (attempt_id,), + ).fetchall() + assert len(rows) == 1 + assert rows[0]["selected_index"] == 1 # the first write won + assert rows[0]["is_correct"] is True + assert rows[0]["time_ms"] == 1200 + + +def test_responses_cascade_with_their_attempt(db_conn): + from db.connection import table + + attempt_id = _make_attempt(db_conn) + table("quiz_responses").insert({ + "attempt_id": attempt_id, "question_index": 0, + "selected_index": 0, "is_correct": False, + }) + db_conn.execute("DELETE FROM quiz_attempts WHERE id = %s", (attempt_id,)) + left = db_conn.execute( + "SELECT count(*) AS n FROM quiz_responses WHERE attempt_id = %s", + (attempt_id,), + ).fetchone() + assert left["n"] == 0 diff --git a/backend/tests/test_quiz_answers_c.py b/backend/tests/test_quiz_answers_c.py new file mode 100644 index 00000000..d2f01469 --- /dev/null +++ b/backend/tests/test_quiz_answers_c.py @@ -0,0 +1,470 @@ +""" +Workstream C of the pre-revamp quiz repair batch (#541, epic #537): +server-authoritative grading. + +- C1: POST /api/quiz/attempts/{attempt_id}/answer grades one question + server-side; owner-checked; 409 after completion; idempotent on + (attempt_id, question_index) — re-answering returns the FIRST recorded + response (no revision; the revamp decides if that changes). +- C2: responses persist individually in quiz_responses (plaintext + analytics scalars; nothing free-text). +- C3: include_answer_key on generate (default true for the current + client, logged, deprecated by #546) — when false, the response strips + per-option `correct` booleans while the stored questions_json keeps + them for grading. +- C4: submit prefers recorded quiz_responses and falls back to the + payload per question; the atomic completed_at claim (PR #464) stays. +""" +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +from fastapi.testclient import TestClient + +from main import app +from agents.quiz import Quiz, QuizQuestion + +client = TestClient(app) + + +QUESTIONS = [ + { + "id": 1, + "question": "Q1?", + "options": [ + {"label": "A", "text": "a1", "correct": False}, + {"label": "B", "text": "b1", "correct": True}, + ], + "explanation": "B is right.", + "concept_tested": "Loops", + "difficulty": "medium", + }, + { + "id": 2, + "question": "Q2?", + "options": [ + {"label": "A", "text": "a2", "correct": True}, + {"label": "B", "text": "b2", "correct": False}, + ], + "explanation": "A is right.", + "concept_tested": "Loops", + "difficulty": "medium", + }, +] + + +class _ResponsesTable: + """In-memory quiz_responses double enforcing the UNIQUE.""" + + def __init__(self): + self.rows: list[dict] = [] + + def select(self, columns="*", filters=None, order=None, **_): + filters = filters or {} + out = self.rows + for key, expr in filters.items(): + val = expr.split(".", 1)[1] if isinstance(expr, str) else expr + out = [r for r in out if str(r.get(key)) == str(val)] + if order and "question_index" in order: + out = sorted(out, key=lambda r: r["question_index"]) + return list(out) + + def insert(self, payload): + for r in self.rows: + if ( + r["attempt_id"] == payload["attempt_id"] + and r["question_index"] == payload["question_index"] + ): + raise RuntimeError('duplicate key value violates unique constraint "quiz_responses_attempt_question_key" (23505)') + self.rows.append(dict(payload)) + return [dict(payload)] + + +def _tables(*, completed_at=None, responses: "_ResponsesTable | None" = None): + responses = responses or _ResponsesTable() + + def factory(name): + if name == "quiz_responses": + return responses + mock = MagicMock() + if name == "quiz_attempts": + mock.select.return_value = [{ + "id": "quiz1", + "user_id": "user_andres", + "concept_node_id": "node1", + "difficulty": "medium", + "questions_json": QUESTIONS, + "completed_at": completed_at, + }] + mock.update.return_value = [{"id": "quiz1"}] + elif name == "graph_nodes": + mock.select.return_value = [{ + "mastery_score": 0.5, + "concept_name": "Loops", + "course_id": "course1", + }] + else: + mock.select.return_value = [] + mock.update.return_value = [{"id": "updated"}] + return mock + + return factory, responses + + +def _answer(body_extra=None, attempt_id="quiz1"): + return client.post(f"/api/quiz/attempts/{attempt_id}/answer", json={ + "question_index": 0, + "selected_index": 1, + **(body_extra or {}), + }) + + +# ── C1: per-question answer endpoint ──────────────────────────────────────── + + +class TestAnswerEndpoint: + def test_grades_and_persists_a_response(self): + factory, responses = _tables() + with patch("routes.quiz.table", side_effect=factory): + r = _answer({"time_ms": 4200, "confidence": 0.8}) + assert r.status_code == 200 + data = r.json() + assert data["is_correct"] is True # option index 1 is "B", correct + assert data["correct_index"] == 1 + assert data["explanation"] == "B is right." + # next_question is question 2, without the answer key. + nq = data["next_question"] + assert nq["id"] == 2 + assert all("correct" not in o for o in nq["options"]) + # Persisted plaintext scalars (C2). + assert len(responses.rows) == 1 + row = responses.rows[0] + assert row["attempt_id"] == "quiz1" + assert row["question_index"] == 0 + assert row["selected_index"] == 1 + assert row["is_correct"] is True + assert row["time_ms"] == 4200 + assert row["confidence"] == 0.8 + + def test_last_question_has_no_next(self): + factory, _ = _tables() + with patch("routes.quiz.table", side_effect=factory): + r = _answer({"question_index": 1, "selected_index": 1}) + assert r.status_code == 200 + data = r.json() + assert data["is_correct"] is False + assert data["correct_index"] == 0 + assert data["next_question"] is None + + def test_reanswer_returns_first_recorded_response(self): + factory, responses = _tables() + with patch("routes.quiz.table", side_effect=factory): + first = _answer({"selected_index": 1}) + second = _answer({"selected_index": 0}) # tries to revise → no + assert first.status_code == 200 + assert second.status_code == 200 + assert second.json()["is_correct"] is True # the FIRST answer's grade + assert second.json()["recorded"] is False # nothing new was written + assert len(responses.rows) == 1 + assert responses.rows[0]["selected_index"] == 1 + + def test_completed_attempt_409s(self): + factory, _ = _tables(completed_at="2026-08-12T00:00:00Z") + with patch("routes.quiz.table", side_effect=factory): + r = _answer() + assert r.status_code == 409 + assert r.json()["error"]["code"] == "QUIZ_ATTEMPT_ALREADY_COMPLETED" + + def test_unknown_attempt_404s(self): + def factory(name): + mock = MagicMock() + mock.select.return_value = [] + return mock + + with patch("routes.quiz.table", side_effect=factory): + r = _answer(attempt_id="nope") + assert r.status_code == 404 + assert r.json()["error"]["code"] == "QUIZ_ATTEMPT_NOT_FOUND" + + def test_bad_indexes_400(self): + factory, _ = _tables() + with patch("routes.quiz.table", side_effect=factory): + r_q = _answer({"question_index": 9}) + r_o = _answer({"selected_index": 9}) + for r in (r_q, r_o): + assert r.status_code == 400 + assert r.json()["error"]["code"] == "QUIZ_QUESTION_INVALID" + + def test_response_echoes_the_wire_question_id(self): + """Wire question ids are 1-based; question_index is 0-based. Echoing + the id the client displayed makes an off-by-one visible instead of + silently grading the wrong question (which idempotency then locks in).""" + factory, _ = _tables() + with patch("routes.quiz.table", side_effect=factory): + r = _answer({"question_index": 0, "selected_index": 1}) + assert r.status_code == 200 + data = r.json() + assert data["question_index"] == 0 + assert data["question_id"] == 1 # QUESTIONS[0]["id"] + + def test_question_id_mismatch_is_rejected(self): + """A client may send the wire id it displayed alongside the index; + when both are present they must agree, so passing the 1-based id as + the 0-based index fails loudly instead of grading question 2.""" + factory, responses = _tables() + with patch("routes.quiz.table", side_effect=factory): + r = _answer({"question_index": 1, "selected_index": 0, "question_id": 1}) + assert r.status_code == 400 + assert r.json()["error"]["code"] == "QUIZ_QUESTION_INVALID" + assert responses.rows == [] + + +# ── C3: include_answer_key ────────────────────────────────────────────────── + + +def _generate_factory(): + captured = {} + + def factory(name): + mock = MagicMock() + if name == "graph_nodes": + mock.select.return_value = [{ + "id": "node1", + "course_id": "course1", + "concept_name": "Loops", + "mastery_score": 0.5, + }] + elif name == "quiz_attempts": + def _capture(payload): + captured["payload"] = payload + return [{"id": payload["id"]}] + mock.insert.side_effect = _capture + else: + mock.select.return_value = [] + return mock + + return factory, captured + + +def _fake_quiz(): + return Quiz(questions=[ + QuizQuestion( + question="Q?", type="multiple_choice", difficulty="easy", + options=["w", "x", "y", "z"], correct_answer="x", + explanation="ok", concept="Loops", + ), + ]) + + +class TestIncludeAnswerKey: + def _post(self, extra): + return client.post("/api/quiz/generate", json={ + "user_id": "user_andres", + "concept_node_id": "node1", + "num_questions": 1, + "difficulty": "easy", + "use_shared_context": False, + **extra, + }) + + def test_default_true_keeps_the_key_and_logs(self, caplog): + factory, _ = _generate_factory() + with ( + patch("routes.quiz.table", side_effect=factory), + patch("routes.quiz.quiz_agent.run", + new=AsyncMock(return_value=SimpleNamespace(output=_fake_quiz()))), + ): + with caplog.at_level("INFO", logger="routes.quiz"): + r = self._post({}) + assert r.status_code == 200 + q = r.json()["questions"][0] + assert any(o.get("correct") for o in q["options"]) + assert any("answer key" in rec.message for rec in caplog.records), ( + "every keyed response must leave a deprecation breadcrumb (#546)" + ) + + def test_false_strips_the_key_from_response_but_not_storage(self): + from services.encryption import decrypt_json_column + + factory, captured = _generate_factory() + with ( + patch("routes.quiz.table", side_effect=factory), + patch("routes.quiz.quiz_agent.run", + new=AsyncMock(return_value=SimpleNamespace(output=_fake_quiz()))), + ): + r = self._post({"include_answer_key": False}) + assert r.status_code == 200 + q = r.json()["questions"][0] + assert all("correct" not in o for o in q["options"]) + # Storage keeps the key — grading stays server-side. + stored = decrypt_json_column(captured["payload"]["questions_json"]) + assert any(o.get("correct") for o in stored[0]["options"]) + + +# ── C4: submit reconciles recorded responses with the payload ─────────────── + + +class TestSubmitReconciliation: + def _submit_mocks(self, responses): + factory, _ = _tables(responses=responses) + return ( + patch("routes.quiz.table", side_effect=factory), + patch("routes.quiz.apply_graph_update"), + patch("routes.quiz.get_quiz_context", return_value={}), + patch( + "routes.quiz.quiz_context_agent.run", + new=AsyncMock(return_value=SimpleNamespace( + output=SimpleNamespace(model_dump=lambda: {}) + )), + ), + patch("routes.quiz.save_quiz_context"), + ) + + def test_persisted_answers_match_what_was_graded(self): + """answers_json is the attempt's record of what the student answered + — it must be the RECONCILED set, not the raw payload. A recorded-only + submit (answers: []) previously stored an empty answer set beside a + full score; a contradicting payload answer was stored despite losing + to the recorded response.""" + from services.encryption import decrypt_json_column + + responses = _ResponsesTable() + responses.insert({ + "attempt_id": "quiz1", "question_index": 0, + "selected_index": 1, "is_correct": True, + }) + update_calls: list = [] + factory, _ = _tables(responses=responses) + + def capturing(name): + t = factory(name) + if name == "quiz_attempts": + def _update(data, filters=None): + update_calls.append(data) + return [{"id": "quiz1"}] + t.update.side_effect = _update + return t + + with ( + patch("routes.quiz.table", side_effect=capturing), + patch("routes.quiz.apply_graph_update"), + patch("routes.quiz.get_quiz_context", return_value={}), + patch( + "routes.quiz.quiz_context_agent.run", + new=AsyncMock(return_value=SimpleNamespace( + output=SimpleNamespace(model_dump=lambda: {}) + )), + ), + patch("routes.quiz.save_quiz_context"), + ): + r = client.post("/api/quiz/submit", json={ + "quiz_id": "quiz1", + # Question 1 answered via C1 (B, correct) — the payload's + # contradicting "A" must not reach storage. Question 2 has + # no recorded response, so its payload answer is authoritative. + "answers": [ + {"question_id": 1, "selected_label": "A"}, + {"question_id": 2, "selected_label": "A"}, + ], + }) + + assert r.status_code == 200 + stored = [d for d in update_calls if "answers_json" in d] + assert len(stored) == 1 + answers = decrypt_json_column(stored[0]["answers_json"]) + by_qid = {str(a["question_id"]): a["selected_label"] for a in answers} + assert by_qid["1"] == "B", "stored answer must be the graded (recorded) one" + assert by_qid["2"] == "A" + + def test_recorded_only_submit_persists_the_recorded_answers(self): + from services.encryption import decrypt_json_column + + responses = _ResponsesTable() + responses.insert({ + "attempt_id": "quiz1", "question_index": 0, + "selected_index": 1, "is_correct": True, + }) + responses.insert({ + "attempt_id": "quiz1", "question_index": 1, + "selected_index": 0, "is_correct": True, + }) + update_calls: list = [] + factory, _ = _tables(responses=responses) + + def capturing(name): + t = factory(name) + if name == "quiz_attempts": + def _update(data, filters=None): + update_calls.append(data) + return [{"id": "quiz1"}] + t.update.side_effect = _update + return t + + with ( + patch("routes.quiz.table", side_effect=capturing), + patch("routes.quiz.apply_graph_update"), + patch("routes.quiz.get_quiz_context", return_value={}), + patch( + "routes.quiz.quiz_context_agent.run", + new=AsyncMock(return_value=SimpleNamespace( + output=SimpleNamespace(model_dump=lambda: {}) + )), + ), + patch("routes.quiz.save_quiz_context"), + ): + r = client.post("/api/quiz/submit", json={ + "quiz_id": "quiz1", "answers": [], + }) + + assert r.status_code == 200 + assert r.json()["score"] == 2 + stored = [d for d in update_calls if "answers_json" in d][0] + answers = decrypt_json_column(stored["answers_json"]) + assert len(answers) == 2, ( + "a perfect score with an empty stored answer set is a contradictory " + "attempt record" + ) + assert {a["selected_label"] for a in answers} == {"B", "A"} + + def test_mixed_case_prefers_recorded_and_falls_back_to_payload(self): + responses = _ResponsesTable() + # Question 1 (index 0) was answered through C1: correct (B). + responses.insert({ + "attempt_id": "quiz1", "question_index": 0, + "selected_index": 1, "is_correct": True, + }) + mocks = self._submit_mocks(responses) + with mocks[0], mocks[1], mocks[2], mocks[3], mocks[4]: + r = client.post("/api/quiz/submit", json={ + "quiz_id": "quiz1", + "answers": [ + # Payload CONTRADICTS the recorded response for q1 — + # the recorded one wins. + {"question_id": 1, "selected_label": "A"}, + # q2 exists only in the payload: falls back (A = correct). + {"question_id": 2, "selected_label": "A"}, + ], + }) + assert r.status_code == 200 + data = r.json() + assert data["score"] == 2 + flags = {res["question_id"]: res["correct"] for res in data["results"]} + assert flags["1"] is True # recorded response, not the payload's A + assert flags["2"] is True # payload fallback + + def test_recorded_only_submit_needs_no_payload_answers(self): + responses = _ResponsesTable() + responses.insert({ + "attempt_id": "quiz1", "question_index": 0, + "selected_index": 1, "is_correct": True, + }) + responses.insert({ + "attempt_id": "quiz1", "question_index": 1, + "selected_index": 1, "is_correct": False, + }) + mocks = self._submit_mocks(responses) + with mocks[0], mocks[1], mocks[2], mocks[3], mocks[4]: + r = client.post("/api/quiz/submit", json={ + "quiz_id": "quiz1", "answers": [], + }) + assert r.status_code == 200 + assert r.json()["score"] == 1