Skip to content

feat(quiz): adaptive difficulty, /api/quiz/config, stable error envelope (#540)#547

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
AndresL230 merged 3 commits into from
Aug 12, 2026
Merged

feat(quiz): adaptive difficulty, /api/quiz/config, stable error envelope (#540) #547

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
afc97df
feat(quiz): adaptive difficulty, /api/quiz/config, stable error envel…
AndresL230 Aug 12, 2026
1ea4c72
fix(quiz): address #547 review — envelope precision, adaptive history…
AndresL230 Aug 12, 2026
1bd3512
test(e2e): tolerate the CustomSelect ✓ marker in the #540 selector jo…
AndresL230 Aug 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions backend/agents/quiz.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -113,6 +113,18 @@ class Quiz(BaseModel):
"- Don't drop high-mastery, recently-reviewed concepts entirely; "
" include 1 question on a strong-and-fresh concept to keep the "
" quiz from feeling punishing.\n\n"
"ADAPTIVE MODE (#540): when the user message says the quiz is in "
"adaptive mode, there is no user-requested difficulty — you choose "
"each question's difficulty yourself. Base the mix on mastery and "
"`recent_attempts.accuracy`: struggling/low-accuracy concepts get "
"easy-leaning questions, strong/high-accuracy ones get hard-leaning "
"questions; with no history at all, center the mix on medium. The "
"±1-step limits below do not apply in adaptive mode, but every "
"question still carries a concrete easy|medium|hard difficulty — "
"'adaptive' is never a per-question value. Note that PAST attempts "
"in `recent_attempts` may carry difficulty 'adaptive' (that "
"attempt's mix was agent-chosen); treat such an attempt's "
"difficulty as unspecified and judge it by its accuracy alone.\n\n"
"Adaptive-difficulty rules (use `recent_attempts.accuracy`):\n"
"- If the most recent 2-3 attempts on this concept averaged < "
" 0.5 accuracy, drop the difficulty mix one step from what the "
Expand Down
2 changes: 1 addition & 1 deletion backend/db/e2e_checks/quiz.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -32,7 +32,7 @@ def run() -> None:

# ── 1. Generate a quiz to obtain a real quiz_id ───────────────────────────
# GenerateQuizBody: {user_id, concept_node_id, num_questions, difficulty, use_shared_context}
# difficulty must be in {"easy","medium","hard"} (CHECK constraint from 0025).
# difficulty ∈ {"easy","medium","hard","adaptive"} (0025 CHECK + the #540 adaptive migration).
gen_r = client.post(
"/api/quiz/generate",
json={
Expand Down
32 changes: 32 additions & 0 deletions backend/db/migrations/20260812204809_quiz_attempts_adaptive_difficulty.sql
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
-- #540 A1: 'adaptive' becomes a real request-side difficulty. The attempt row
-- records what the student asked for (the response's resolved_difficulty
-- reports what generation actually produced), so the 0025 CHECK
-- (easy|medium|hard) must admit 'adaptive'.
--
-- DEPLOY ORDER: widening the CHECK is backward-compatible — apply this
-- migration BEFORE deploying the code that accepts 'adaptive'. In the gap the
-- other way around, an adaptive generate runs the full LLM call and then 500s
-- on the INSERT. (The promote runner migrates before merging; for staging run
-- `python -m db.migrate` before the deploy picks up the code.)
--
-- The DROP is by introspection, not by an assumed default name: environments
-- with out-of-band table history (staging has had some) can carry the same
-- CHECK under a different name, and a name-keyed `DROP IF EXISTS` would
-- silently no-op and leave the narrow constraint alive beside the new one.
DO $$
DECLARE c record;
BEGIN
FOR c IN
SELECT conname
FROM pg_constraint
WHERE conrelid = 'quiz_attempts'::regclass
AND contype = 'c'
AND pg_get_constraintdef(oid) ILIKE '%difficulty%'
LOOP
EXECUTE format('ALTER TABLE quiz_attempts DROP CONSTRAINT %I', c.conname);
END LOOP;
END $$;

ALTER TABLE quiz_attempts
ADD CONSTRAINT quiz_attempts_difficulty_check
CHECK (difficulty IN ('easy', 'medium', 'hard', 'adaptive'));
34 changes: 31 additions & 3 deletions backend/main.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@
from routes.admin import router as admin_router
from routes.admin_analytics import router as admin_analytics_router
from routes.newsletter import router as newsletter_router
from services import quiz_config, quiz_errors
from services.logfire_scrubber import EXTRA_PATTERNS, scrub_value
from services.request_context import RequestIDMiddleware, current_request_id
from services.storage_service import (
Expand DownExpand Up@@ -183,22 +184,46 @@ def _drop_request_arguments(_request, _attributes):
app.add_middleware(RequestIDMiddleware)


# #540 A3: on /api/quiz/* paths, quiz_errors.error_content wraps errors in
# the coded envelope (QuizAPIError raise sites carry precise codes; plain
# HTTPExceptions fall back to a status-derived one); everywhere else it
# returns the legacy {detail, request_id} shape unchanged.


@app.exception_handler(StarletteHTTPException)
async def http_exception_handler(request: Request, exc: StarletteHTTPException):
rid = getattr(request.state, "request_id", None) or current_request_id()
content = quiz_errors.error_content(
request.url.path,
exc.status_code,
exc.detail,
rid,
code=getattr(exc, "code", None),
machine_detail=getattr(exc, "machine_detail", None),
)
return JSONResponse(
status_code=exc.status_code,
content={"detail": exc.detail, "request_id": rid},
content=content,
headers={"X-Request-ID": rid} if rid else {},
)


@app.exception_handler(RequestValidationError)
async def validation_exception_handler(request: Request, exc: RequestValidationError):
rid = getattr(request.state, "request_id", None) or current_request_id()
code, message = quiz_errors.validation_error_code(exc.errors())
if code is quiz_errors.QuizErrorCode.QUIZ_COUNT_OUT_OF_RANGE:
message = (
f"Quizzes can have between {quiz_config.QUIZ_MIN_QUESTIONS} "
f"and {quiz_config.QUIZ_MAX_QUESTIONS} questions."
)
content = quiz_errors.error_content(
request.url.path, 422, exc.errors(), rid,
code=code, message=message, machine_detail=exc.errors(),
)
return JSONResponse(
status_code=422,
content={"detail": exc.errors(), "request_id": rid},
content=content,
headers={"X-Request-ID": rid} if rid else {},
)

Expand All@@ -207,9 +232,12 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
async def unhandled_exception_handler(request: Request, exc: Exception):
logging.getLogger("main").exception("Unhandled exception")
rid = getattr(request.state, "request_id", None) or current_request_id()
content = quiz_errors.error_content(
request.url.path, 500, "Internal server error.", rid,
)
return JSONResponse(
status_code=500,
content={"detail": "Internal server error.", "request_id": rid},
content=content,
headers={"X-Request-ID": rid} if rid else {},
)

Expand Down
7 changes: 6 additions & 1 deletion backend/models/__init__.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
from typing import Optional, Union, List, Literal
from pydantic import BaseModel, Field

from services.quiz_config import QUIZ_MIN_QUESTIONS, QUIZ_MAX_QUESTIONS


# ── Learn ─────────────────────────────────────────────────────────────────────

Expand DownExpand Up@@ -46,7 +48,10 @@ class ActionBody(BaseModel):
class GenerateQuizBody(BaseModel):
user_id: str = "user_andres"
concept_node_id: str
num_questions: int = Field(default=5, ge=1, le=10)
# Bounds come from services/quiz_config.py — the same constants
# GET /api/quiz/config serves, so the client can't offer a value
# this model rejects (#540 A2).
num_questions: int = Field(default=5, ge=QUIZ_MIN_QUESTIONS, le=QUIZ_MAX_QUESTIONS)
difficulty: str = "medium"
use_shared_context: bool = True
# Mirrors the Learn-route fast/smart toggle so quiz generation has
Expand Down
131 changes: 110 additions & 21 deletions backend/routes/quiz.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,12 @@
from routes.learn import _get_catalog_chunk
from services import events_service
from services.auth_guard import require_self
from services.quiz_config import (
CONCRETE_DIFFICULTIES,
REQUESTED_DIFFICULTIES,
quiz_config_payload,
)
from services.quiz_errors import QuizAPIError, QuizErrorCode
from services.profiles import get_display_name
from services.encryption import encrypt_json, decrypt_json_column
from services.graph_service import apply_graph_update
Expand All@@ -35,8 +41,10 @@

PROMPTS_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "prompts")

# quiz_attempts.difficulty CHECK enum (0025).
VALID_DIFFICULTIES = {"easy", "medium", "hard"}
# Request-side difficulties live in services/quiz_config.py (#540 A2):
# the concrete trio matches the quiz_attempts.difficulty CHECK (0025,
# extended with 'adaptive' by the #540 migration); 'adaptive' hands the
# per-question mix decision to the agent (A1).


def _load_prompt(name: str) -> str:
Expand All@@ -62,6 +70,30 @@ def _load_prompt(name: str) -> str:

_OPTION_LABELS = ["A", "B", "C", "D", "E", "F"]

# Rank order for tie-breaking the overall difficulty report — derived from
# the config tuple so a difficulty added there can't be silently dropped by
# _resolved_difficulty's counting.
_DIFFICULTY_RANK = {d: i for i, d in enumerate(CONCRETE_DIFFICULTIES)}


def _resolved_difficulty(wire_questions: list[dict]) -> str:
"""The overall difficulty generation actually produced (#540 A1).

Mode of the per-question difficulties; ties break to the harder value
so the report never understates what the student is about to face.
Defaults to 'medium' when nothing usable is present (can't happen for
agent output — QuizQuestion.difficulty is a concrete Literal — but
this also runs on stored legacy rows).
"""
counts: dict[str, int] = {}
for q in wire_questions:
d = q.get("difficulty")
if d in _DIFFICULTY_RANK:
counts[d] = counts.get(d, 0) + 1
if not counts:
return "medium"
return max(counts, key=lambda d: (counts[d], _DIFFICULTY_RANK[d]))


def _agent_question_to_wire(q: QuizQuestion, qid: int) -> dict | None:
"""Map an agent QuizQuestion to the legacy wire-format dict, or
Expand DownExpand Up@@ -237,8 +269,24 @@ async def _quiz_via_agent(
# Keep this message routing-only; the workflow + adaptive rules
# live in the system prompt. We just hand the agent the inputs it
# needs and trust the prompt to drive tool calls.
if difficulty == "adaptive":
# #540 A1: no target difficulty — the agent picks the whole mix
# from mastery + recent accuracy (ADAPTIVE MODE in the system
# prompt). Every emitted question still carries a concrete
# easy|medium|hard; the route reports the overall pick back to
# the client as `resolved_difficulty`.
difficulty_clause = (
f"Generate {num_questions} questions in ADAPTIVE MODE: you "
f"choose each question's difficulty (easy, medium, or hard) "
f"from the student's mastery and recent accuracy, per the "
f"adaptive-mode rules in your system prompt."
)
else:
difficulty_clause = (
f"Generate {num_questions} {difficulty} questions for the student."
)
routing_msg = (
f"Generate {num_questions} {difficulty} questions for the student. "
f"{difficulty_clause} "
f"The target concept is '{concept_name}' "
f"(concept_node_id={concept_node_id}). Follow the workflow in your "
f"system prompt; pass concept_node_id='{concept_node_id}' to "
Expand DownExpand Up@@ -290,23 +338,40 @@ async def _quiz_via_agent(
)
return wire_questions

@router.get("/config")
def quiz_config():
"""Selector options for the quiz UI (#540 A2). Single source of truth:
the same constants bound the Pydantic request model, so a client that
builds its selects from this payload can never send a value the route
rejects. No user data, no auth needed."""
return quiz_config_payload()


@router.post("/generate")
async def generate_quiz(body: GenerateQuizBody, request: Request):
require_self(body.user_id, request)
# quiz_attempts.difficulty is CHECK-constrained (0025); reject drift before
# we run the agent or write an attempt row.
if body.difficulty not in VALID_DIFFICULTIES:
raise HTTPException(
# The concrete trio is CHECK-constrained on quiz_attempts (0025 +
# the #540 'adaptive' extension); reject drift before we run the
# agent or write an attempt row.
if body.difficulty not in REQUESTED_DIFFICULTIES:
raise QuizAPIError(
status_code=400,
detail=f"Invalid difficulty '{body.difficulty}'. "
f"Must be one of {sorted(VALID_DIFFICULTIES)}.",
code=QuizErrorCode.QUIZ_DIFFICULTY_INVALID,
message=(
"That difficulty isn't available. Choose easy, medium, "
"hard, or adaptive."
),
)
node_rows = table("graph_nodes").select(
"*",
filters={"id": f"eq.{body.concept_node_id}", "user_id": f"eq.{body.user_id}"},
)
if not node_rows:
raise HTTPException(status_code=404, detail="Concept node not found")
raise QuizAPIError(
status_code=404,
code=QuizErrorCode.QUIZ_CONCEPT_NOT_FOUND,
message="We couldn't find that concept in your knowledge graph.",
)
node = node_rows[0]
course_id = node.get("course_id") or None
concept_name = node.get("concept_name") or ""
Expand DownExpand Up@@ -339,15 +404,17 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
# The raw-Gemini legacy fallback was retired in #145; degrade to 502
# rather than serving a quiz from a second LLM path.
logger.warning("Quiz agent guardrails tripped; returning 502", exc_info=e)
raise HTTPException(
raise QuizAPIError(
status_code=502,
detail="Quiz generation is temporarily unavailable. Please try again.",
code=QuizErrorCode.QUIZ_GENERATION_FAILED,
message="Quiz generation is temporarily unavailable. Please try again.",
) from e
except Exception as e:
logger.exception("Unexpected quiz-agent failure; returning 502")
raise HTTPException(
raise QuizAPIError(
status_code=502,
detail="Quiz generation is temporarily unavailable. Please try again.",
code=QuizErrorCode.QUIZ_GENERATION_FAILED,
message="Quiz generation is temporarily unavailable. Please try again.",
) from e

quiz_id = str(uuid.uuid4())
Expand All@@ -372,14 +439,28 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
"difficulty": body.difficulty,
},
)
return {"quiz_id": quiz_id, "questions": questions}
# #540 A1: echo what generation actually chose. requested_difficulty
# is what the student asked for (may be 'adaptive');
# resolved_difficulty is the overall mix the agent produced (always
# concrete) — so the client can say "we picked hard for you" instead
# of repeating the request back.
return {
"quiz_id": quiz_id,
"questions": questions,
"requested_difficulty": body.difficulty,
"resolved_difficulty": _resolved_difficulty(questions),
}


@router.post("/submit")
def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request: Request):
attempt_rows = table("quiz_attempts").select("*", filters={"id": f"eq.{body.quiz_id}"})
if not attempt_rows:
raise HTTPException(status_code=404, detail="Quiz not found")
raise QuizAPIError(
status_code=404,
code=QuizErrorCode.QUIZ_ATTEMPT_NOT_FOUND,
message="We couldn't find that quiz.",
)
attempt = attempt_rows[0]

user_id = attempt["user_id"]
Expand All@@ -395,8 +476,10 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request
# 200: quiz_attempts stores no mastery_before/after, so faithfully
# reconstructing the first response would need a migration.
if attempt.get("completed_at"):
raise HTTPException(
status_code=409, detail="Quiz attempt has already been submitted"
raise QuizAPIError(
status_code=409,
code=QuizErrorCode.QUIZ_ATTEMPT_ALREADY_COMPLETED,
message="This quiz has already been submitted.",
)
# The read above is only the fast path — two CONCURRENT submits (a
# double-click on the final submit) would both pass it. The atomic claim
Expand All@@ -410,8 +493,10 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request
filters={"id": f"eq.{body.quiz_id}", "completed_at": "is.null"},
)
if not claimed:
raise HTTPException(
status_code=409, detail="Quiz attempt has already been submitted"
raise QuizAPIError(
status_code=409,
code=QuizErrorCode.QUIZ_ATTEMPT_ALREADY_COMPLETED,
message="This quiz has already been submitted.",
)

concept_node_id = attempt["concept_node_id"]
Expand DownExpand Up@@ -450,7 +535,11 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request
filters={"id": f"eq.{concept_node_id}", "user_id": f"eq.{user_id}"},
)
if not node_rows:
raise HTTPException(status_code=404, detail="Concept node not found")
raise QuizAPIError(
status_code=404,
code=QuizErrorCode.QUIZ_CONCEPT_NOT_FOUND,
message="We couldn't find that concept in your knowledge graph.",
)
node = node_rows[0]
mastery_before = node["mastery_score"]
mastery_after = max(0.0, min(1.0, mastery_before + (score * 0.03) - ((total - score) * 0.02)))
Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
feat(quiz): adaptive difficulty, /api/quiz/config, stable error envelope (#540) by AndresL230 · Pull Request #547 · SaplingLearn/Sapling · GitHub
Skip to content
feat(quiz): adaptive difficulty, /api/quiz/config, stable error envelope (#540)#547

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

AndresL230
merged 3 commits into

from

Aug 12, 2026
 Merged


feat(quiz): adaptive difficulty, /api/quiz/config, stable error envelope (#540)
#547
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
afc97df

feat(quiz): adaptive difficulty, /api/quiz/config, stable error envel…


AndresL230 Aug 12, 2026
1ea4c72

fix(quiz): address #547 review — envelope precision, adaptive history…


AndresL230 Aug 12, 2026
1bd3512

test(e2e): tolerate the CustomSelect ✓ marker in the #540 selector jo…


AndresL230 Aug 12, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 12 additions & 0 deletions
backend/agents/quiz.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -113,6 +113,18 @@ class Quiz(BaseModel):
 "- Don't drop high-mastery, recently-reviewed concepts entirely; "
 " include 1 question on a strong-and-fresh concept to keep the "
 " quiz from feeling punishing.\n\n"
 "ADAPTIVE MODE (#540): when the user message says the quiz is in "
 "adaptive mode, there is no user-requested difficulty — you choose "
 "each question's difficulty yourself. Base the mix on mastery and "
 "`recent_attempts.accuracy`: struggling/low-accuracy concepts get "
 "easy-leaning questions, strong/high-accuracy ones get hard-leaning "
 "questions; with no history at all, center the mix on medium. The "
 "±1-step limits below do not apply in adaptive mode, but every "
 "question still carries a concrete easy|medium|hard difficulty — "
 "'adaptive' is never a per-question value. Note that PAST attempts "
 "in `recent_attempts` may carry difficulty 'adaptive' (that "
 "attempt's mix was agent-chosen); treat such an attempt's "
 "difficulty as unspecified and judge it by its accuracy alone.\n\n"
 "Adaptive-difficulty rules (use `recent_attempts.accuracy`):\n"
 "- If the most recent 2-3 attempts on this concept averaged < "
 " 0.5 accuracy, drop the difficulty mix one step from what the "
Expand Down

2 changes: 1 addition & 1 deletion
backend/db/e2e_checks/quiz.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -32,7 +32,7 @@ def run() -> None:

 # ── 1. Generate a quiz to obtain a real quiz_id ───────────────────────────
 # GenerateQuizBody: {user_id, concept_node_id, num_questions, difficulty, use_shared_context}
 # difficulty must be in {"easy","medium","hard"} (CHECK constraint from 0025).
 # difficulty ∈ {"easy","medium","hard","adaptive"} (0025 CHECK + the #540 adaptive migration).
 gen_r = client.post(
 "/api/quiz/generate",
 json={
Expand Down

32 changes: 32 additions & 0 deletions
backend/db/migrations/20260812204809_quiz_attempts_adaptive_difficulty.sql

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
-- #540 A1: 'adaptive' becomes a real request-side difficulty. The attempt row
-- records what the student asked for (the response's resolved_difficulty
-- reports what generation actually produced), so the 0025 CHECK
-- (easy|medium|hard) must admit 'adaptive'.
--
-- DEPLOY ORDER: widening the CHECK is backward-compatible — apply this
-- migration BEFORE deploying the code that accepts 'adaptive'. In the gap the
-- other way around, an adaptive generate runs the full LLM call and then 500s
-- on the INSERT. (The promote runner migrates before merging; for staging run
-- `python -m db.migrate` before the deploy picks up the code.)
--
-- The DROP is by introspection, not by an assumed default name: environments
-- with out-of-band table history (staging has had some) can carry the same
-- CHECK under a different name, and a name-keyed `DROP IF EXISTS` would
-- silently no-op and leave the narrow constraint alive beside the new one.
DO $$
DECLARE c record;
BEGIN
 FOR c IN
 SELECT conname
 FROM pg_constraint
 WHERE conrelid = 'quiz_attempts'::regclass
 AND contype = 'c'
 AND pg_get_constraintdef(oid) ILIKE '%difficulty%'
 LOOP
 EXECUTE format('ALTER TABLE quiz_attempts DROP CONSTRAINT %I', c.conname);
 END LOOP;
END $$;

ALTER TABLE quiz_attempts
 ADD CONSTRAINT quiz_attempts_difficulty_check
 CHECK (difficulty IN ('easy', 'medium', 'hard', 'adaptive'));

34 changes: 31 additions & 3 deletions
backend/main.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@
from routes.admin import router as admin_router
from routes.admin_analytics import router as admin_analytics_router
from routes.newsletter import router as newsletter_router
from services import quiz_config, quiz_errors
from services.logfire_scrubber import EXTRA_PATTERNS, scrub_value
from services.request_context import RequestIDMiddleware, current_request_id
from services.storage_service import (
Expand DownExpand Up@@ -183,22 +184,46 @@ def _drop_request_arguments(_request, _attributes):
app.add_middleware(RequestIDMiddleware)


# #540 A3: on /api/quiz/* paths, quiz_errors.error_content wraps errors in
# the coded envelope (QuizAPIError raise sites carry precise codes; plain
# HTTPExceptions fall back to a status-derived one); everywhere else it
# returns the legacy {detail, request_id} shape unchanged.


@app.exception_handler(StarletteHTTPException)
async def http_exception_handler(request: Request, exc: StarletteHTTPException):
 rid = getattr(request.state, "request_id", None) or current_request_id()
 content = quiz_errors.error_content(
 request.url.path,
 exc.status_code,
 exc.detail,
 rid,
 code=getattr(exc, "code", None),
 machine_detail=getattr(exc, "machine_detail", None),
 )
 return JSONResponse(
 status_code=exc.status_code,
 content={"detail": exc.detail, "request_id": rid},
 content=content,
 headers={"X-Request-ID": rid} if rid else {},
 )


@app.exception_handler(RequestValidationError)
async def validation_exception_handler(request: Request, exc: RequestValidationError):
 rid = getattr(request.state, "request_id", None) or current_request_id()
 code, message = quiz_errors.validation_error_code(exc.errors())
 if code is quiz_errors.QuizErrorCode.QUIZ_COUNT_OUT_OF_RANGE:
 message = (
 f"Quizzes can have between {quiz_config.QUIZ_MIN_QUESTIONS} "
 f"and {quiz_config.QUIZ_MAX_QUESTIONS} questions."
 )
 content = quiz_errors.error_content(
 request.url.path, 422, exc.errors(), rid,
 code=code, message=message, machine_detail=exc.errors(),
 )
 return JSONResponse(
 status_code=422,
 content={"detail": exc.errors(), "request_id": rid},
 content=content,
 headers={"X-Request-ID": rid} if rid else {},
 )

Expand All@@ -207,9 +232,12 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
async def unhandled_exception_handler(request: Request, exc: Exception):
 logging.getLogger("main").exception("Unhandled exception")
 rid = getattr(request.state, "request_id", None) or current_request_id()
 content = quiz_errors.error_content(
 request.url.path, 500, "Internal server error.", rid,
 )
 return JSONResponse(
 status_code=500,
 content={"detail": "Internal server error.", "request_id": rid},
 content=content,
 headers={"X-Request-ID": rid} if rid else {},
 )

Expand Down

7 changes: 6 additions & 1 deletion
backend/models/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
from typing import Optional, Union, List, Literal
from pydantic import BaseModel, Field

from services.quiz_config import QUIZ_MIN_QUESTIONS, QUIZ_MAX_QUESTIONS


# ── Learn ─────────────────────────────────────────────────────────────────────

Expand DownExpand Up@@ -46,7 +48,10 @@ class ActionBody(BaseModel):
class GenerateQuizBody(BaseModel):
 user_id: str = "user_andres"
 concept_node_id: str
 num_questions: int = Field(default=5, ge=1, le=10)
 # Bounds come from services/quiz_config.py — the same constants
 # GET /api/quiz/config serves, so the client can't offer a value
 # this model rejects (#540 A2).
 num_questions: int = Field(default=5, ge=QUIZ_MIN_QUESTIONS, le=QUIZ_MAX_QUESTIONS)
 difficulty: str = "medium"
 use_shared_context: bool = True
 # Mirrors the Learn-route fast/smart toggle so quiz generation has
Expand Down

131 changes: 110 additions & 21 deletions
backend/routes/quiz.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,12 @@
from routes.learn import _get_catalog_chunk
from services import events_service
from services.auth_guard import require_self
from services.quiz_config import (
 CONCRETE_DIFFICULTIES,
 REQUESTED_DIFFICULTIES,
 quiz_config_payload,
)
from services.quiz_errors import QuizAPIError, QuizErrorCode
from services.profiles import get_display_name
from services.encryption import encrypt_json, decrypt_json_column
from services.graph_service import apply_graph_update
Expand All@@ -35,8 +41,10 @@

PROMPTS_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "prompts")

# quiz_attempts.difficulty CHECK enum (0025).
VALID_DIFFICULTIES = {"easy", "medium", "hard"}
# Request-side difficulties live in services/quiz_config.py (#540 A2):
# the concrete trio matches the quiz_attempts.difficulty CHECK (0025,
# extended with 'adaptive' by the #540 migration); 'adaptive' hands the
# per-question mix decision to the agent (A1).


def _load_prompt(name: str) -> str:
Expand All@@ -62,6 +70,30 @@ def _load_prompt(name: str) -> str:

_OPTION_LABELS = ["A", "B", "C", "D", "E", "F"]

# Rank order for tie-breaking the overall difficulty report — derived from
# the config tuple so a difficulty added there can't be silently dropped by
# _resolved_difficulty's counting.
_DIFFICULTY_RANK = {d: i for i, d in enumerate(CONCRETE_DIFFICULTIES)}


def _resolved_difficulty(wire_questions: list[dict]) -> str:
 """The overall difficulty generation actually produced (#540 A1).

 Mode of the per-question difficulties; ties break to the harder value
 so the report never understates what the student is about to face.
 Defaults to 'medium' when nothing usable is present (can't happen for
 agent output — QuizQuestion.difficulty is a concrete Literal — but
 this also runs on stored legacy rows).
 """
 counts: dict[str, int] = {}
 for q in wire_questions:
 d = q.get("difficulty")
 if d in _DIFFICULTY_RANK:
 counts[d] = counts.get(d, 0) + 1
 if not counts:
 return "medium"
 return max(counts, key=lambda d: (counts[d], _DIFFICULTY_RANK[d]))


def _agent_question_to_wire(q: QuizQuestion, qid: int) -> dict | None:
 """Map an agent QuizQuestion to the legacy wire-format dict, or
Expand DownExpand Up@@ -237,8 +269,24 @@ async def _quiz_via_agent(
 # Keep this message routing-only; the workflow + adaptive rules
 # live in the system prompt. We just hand the agent the inputs it
 # needs and trust the prompt to drive tool calls.
 if difficulty == "adaptive":
 # #540 A1: no target difficulty — the agent picks the whole mix
 # from mastery + recent accuracy (ADAPTIVE MODE in the system
 # prompt). Every emitted question still carries a concrete
 # easy|medium|hard; the route reports the overall pick back to
 # the client as `resolved_difficulty`.
 difficulty_clause = (
 f"Generate {num_questions} questions in ADAPTIVE MODE: you "
 f"choose each question's difficulty (easy, medium, or hard) "
 f"from the student's mastery and recent accuracy, per the "
 f"adaptive-mode rules in your system prompt."
 )
 else:
 difficulty_clause = (
 f"Generate {num_questions} {difficulty} questions for the student."
 )
 routing_msg = (
 f"Generate {num_questions} {difficulty} questions for the student. "
 f"{difficulty_clause} "
 f"The target concept is '{concept_name}' "
 f"(concept_node_id={concept_node_id}). Follow the workflow in your "
 f"system prompt; pass concept_node_id='{concept_node_id}' to "
Expand DownExpand Up@@ -290,23 +338,40 @@ async def _quiz_via_agent(
 )
 return wire_questions

@router.get("/config")
def quiz_config():
 """Selector options for the quiz UI (#540 A2). Single source of truth:
 the same constants bound the Pydantic request model, so a client that
 builds its selects from this payload can never send a value the route
 rejects. No user data, no auth needed."""
 return quiz_config_payload()


@router.post("/generate")
async def generate_quiz(body: GenerateQuizBody, request: Request):
 require_self(body.user_id, request)
 # quiz_attempts.difficulty is CHECK-constrained (0025); reject drift before
 # we run the agent or write an attempt row.
 if body.difficulty not in VALID_DIFFICULTIES:
 raise HTTPException(
 # The concrete trio is CHECK-constrained on quiz_attempts (0025 +
 # the #540 'adaptive' extension); reject drift before we run the
 # agent or write an attempt row.
 if body.difficulty not in REQUESTED_DIFFICULTIES:
 raise QuizAPIError(
 status_code=400,
 detail=f"Invalid difficulty '{body.difficulty}'. "
 f"Must be one of {sorted(VALID_DIFFICULTIES)}.",
 code=QuizErrorCode.QUIZ_DIFFICULTY_INVALID,
 message=(
 "That difficulty isn't available. Choose easy, medium, "
 "hard, or adaptive."
 ),
 )
 node_rows = table("graph_nodes").select(
 "*",
 filters={"id": f"eq.{body.concept_node_id}", "user_id": f"eq.{body.user_id}"},
 )
 if not node_rows:
 raise HTTPException(status_code=404, detail="Concept node not found")
 raise QuizAPIError(
 status_code=404,
 code=QuizErrorCode.QUIZ_CONCEPT_NOT_FOUND,
 message="We couldn't find that concept in your knowledge graph.",
 )
 node = node_rows[0]
 course_id = node.get("course_id") or None
 concept_name = node.get("concept_name") or ""
Expand DownExpand Up@@ -339,15 +404,17 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
 # The raw-Gemini legacy fallback was retired in #145; degrade to 502
 # rather than serving a quiz from a second LLM path.
 logger.warning("Quiz agent guardrails tripped; returning 502", exc_info=e)
 raise HTTPException(
 raise QuizAPIError(
 status_code=502,
 detail="Quiz generation is temporarily unavailable. Please try again.",
 code=QuizErrorCode.QUIZ_GENERATION_FAILED,
 message="Quiz generation is temporarily unavailable. Please try again.",
 ) from e
 except Exception as e:
 logger.exception("Unexpected quiz-agent failure; returning 502")
 raise HTTPException(
 raise QuizAPIError(
 status_code=502,
 detail="Quiz generation is temporarily unavailable. Please try again.",
 code=QuizErrorCode.QUIZ_GENERATION_FAILED,
 message="Quiz generation is temporarily unavailable. Please try again.",
 ) from e

 quiz_id = str(uuid.uuid4())
Expand All@@ -372,14 +439,28 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
 "difficulty": body.difficulty,
 },
 )
 return {"quiz_id": quiz_id, "questions": questions}
 # #540 A1: echo what generation actually chose. requested_difficulty
 # is what the student asked for (may be 'adaptive');
 # resolved_difficulty is the overall mix the agent produced (always
 # concrete) — so the client can say "we picked hard for you" instead
 # of repeating the request back.
 return {
 "quiz_id": quiz_id,
 "questions": questions,
 "requested_difficulty": body.difficulty,
 "resolved_difficulty": _resolved_difficulty(questions),
 }


@router.post("/submit")
def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request: Request):
 attempt_rows = table("quiz_attempts").select("*", filters={"id": f"eq.{body.quiz_id}"})
 if not attempt_rows:
 raise HTTPException(status_code=404, detail="Quiz not found")
 raise QuizAPIError(
 status_code=404,
 code=QuizErrorCode.QUIZ_ATTEMPT_NOT_FOUND,
 message="We couldn't find that quiz.",
 )
 attempt = attempt_rows[0]

 user_id = attempt["user_id"]
Expand All@@ -395,8 +476,10 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request
 # 200: quiz_attempts stores no mastery_before/after, so faithfully
 # reconstructing the first response would need a migration.
 if attempt.get("completed_at"):
 raise HTTPException(
 status_code=409, detail="Quiz attempt has already been submitted"
 raise QuizAPIError(
 status_code=409,
 code=QuizErrorCode.QUIZ_ATTEMPT_ALREADY_COMPLETED,
 message="This quiz has already been submitted.",
 )
 # The read above is only the fast path — two CONCURRENT submits (a
 # double-click on the final submit) would both pass it. The atomic claim
Expand All@@ -410,8 +493,10 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request
 filters={"id": f"eq.{body.quiz_id}", "completed_at": "is.null"},
 )
 if not claimed:
 raise HTTPException(
 status_code=409, detail="Quiz attempt has already been submitted"
 raise QuizAPIError(
 status_code=409,
 code=QuizErrorCode.QUIZ_ATTEMPT_ALREADY_COMPLETED,
 message="This quiz has already been submitted.",
 )

 concept_node_id = attempt["concept_node_id"]
Expand DownExpand Up@@ -450,7 +535,11 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request
 filters={"id": f"eq.{concept_node_id}", "user_id": f"eq.{user_id}"},
 )
 if not node_rows:
 raise HTTPException(status_code=404, detail="Concept node not found")
 raise QuizAPIError(
 status_code=404,
 code=QuizErrorCode.QUIZ_CONCEPT_NOT_FOUND,
 message="We couldn't find that concept in your knowledge graph.",
 )
 node = node_rows[0]
 mastery_before = node["mastery_score"]
 mastery_after = max(0.0, min(1.0, mastery_before + (score * 0.03) - ((total - score) * 0.02)))
Expand Down
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
feat(quiz): adaptive difficulty, /api/quiz/config, stable error envelope (#540) by AndresL230 · Pull Request #547 · SaplingLearn/Sapling · GitHub
Skip to content
feat(quiz): adaptive difficulty, /api/quiz/config, stable error envelope (#540)#547

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

AndresL230
merged 3 commits into

from

Aug 12, 2026
 Merged


feat(quiz): adaptive difficulty, /api/quiz/config, stable error envelope (#540)
#547
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
afc97df

feat(quiz): adaptive difficulty, /api/quiz/config, stable error envel…


AndresL230 Aug 12, 2026
1ea4c72

fix(quiz): address #547 review — envelope precision, adaptive history…


AndresL230 Aug 12, 2026
1bd3512

test(e2e): tolerate the CustomSelect ✓ marker in the #540 selector jo…


AndresL230 Aug 12, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 12 additions & 0 deletions
backend/agents/quiz.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -113,6 +113,18 @@ class Quiz(BaseModel):
 "- Don't drop high-mastery, recently-reviewed concepts entirely; "
 " include 1 question on a strong-and-fresh concept to keep the "
 " quiz from feeling punishing.\n\n"
 "ADAPTIVE MODE (#540): when the user message says the quiz is in "
 "adaptive mode, there is no user-requested difficulty — you choose "
 "each question's difficulty yourself. Base the mix on mastery and "
 "`recent_attempts.accuracy`: struggling/low-accuracy concepts get "
 "easy-leaning questions, strong/high-accuracy ones get hard-leaning "
 "questions; with no history at all, center the mix on medium. The "
 "±1-step limits below do not apply in adaptive mode, but every "
 "question still carries a concrete easy|medium|hard difficulty — "
 "'adaptive' is never a per-question value. Note that PAST attempts "
 "in `recent_attempts` may carry difficulty 'adaptive' (that "
 "attempt's mix was agent-chosen); treat such an attempt's "
 "difficulty as unspecified and judge it by its accuracy alone.\n\n"
 "Adaptive-difficulty rules (use `recent_attempts.accuracy`):\n"
 "- If the most recent 2-3 attempts on this concept averaged < "
 " 0.5 accuracy, drop the difficulty mix one step from what the "
Expand Down

2 changes: 1 addition & 1 deletion
backend/db/e2e_checks/quiz.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -32,7 +32,7 @@ def run() -> None:

 # ── 1. Generate a quiz to obtain a real quiz_id ───────────────────────────
 # GenerateQuizBody: {user_id, concept_node_id, num_questions, difficulty, use_shared_context}
 # difficulty must be in {"easy","medium","hard"} (CHECK constraint from 0025).
 # difficulty ∈ {"easy","medium","hard","adaptive"} (0025 CHECK + the #540 adaptive migration).
 gen_r = client.post(
 "/api/quiz/generate",
 json={
Expand Down

32 changes: 32 additions & 0 deletions
backend/db/migrations/20260812204809_quiz_attempts_adaptive_difficulty.sql

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
-- #540 A1: 'adaptive' becomes a real request-side difficulty. The attempt row
-- records what the student asked for (the response's resolved_difficulty
-- reports what generation actually produced), so the 0025 CHECK
-- (easy|medium|hard) must admit 'adaptive'.
--
-- DEPLOY ORDER: widening the CHECK is backward-compatible — apply this
-- migration BEFORE deploying the code that accepts 'adaptive'. In the gap the
-- other way around, an adaptive generate runs the full LLM call and then 500s
-- on the INSERT. (The promote runner migrates before merging; for staging run
-- `python -m db.migrate` before the deploy picks up the code.)
--
-- The DROP is by introspection, not by an assumed default name: environments
-- with out-of-band table history (staging has had some) can carry the same
-- CHECK under a different name, and a name-keyed `DROP IF EXISTS` would
-- silently no-op and leave the narrow constraint alive beside the new one.
DO $$
DECLARE c record;
BEGIN
 FOR c IN
 SELECT conname
 FROM pg_constraint
 WHERE conrelid = 'quiz_attempts'::regclass
 AND contype = 'c'
 AND pg_get_constraintdef(oid) ILIKE '%difficulty%'
 LOOP
 EXECUTE format('ALTER TABLE quiz_attempts DROP CONSTRAINT %I', c.conname);
 END LOOP;
END $$;

ALTER TABLE quiz_attempts
 ADD CONSTRAINT quiz_attempts_difficulty_check
 CHECK (difficulty IN ('easy', 'medium', 'hard', 'adaptive'));

34 changes: 31 additions & 3 deletions
backend/main.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@
from routes.admin import router as admin_router
from routes.admin_analytics import router as admin_analytics_router
from routes.newsletter import router as newsletter_router
from services import quiz_config, quiz_errors
from services.logfire_scrubber import EXTRA_PATTERNS, scrub_value
from services.request_context import RequestIDMiddleware, current_request_id
from services.storage_service import (
Expand DownExpand Up@@ -183,22 +184,46 @@ def _drop_request_arguments(_request, _attributes):
app.add_middleware(RequestIDMiddleware)


# #540 A3: on /api/quiz/* paths, quiz_errors.error_content wraps errors in
# the coded envelope (QuizAPIError raise sites carry precise codes; plain
# HTTPExceptions fall back to a status-derived one); everywhere else it
# returns the legacy {detail, request_id} shape unchanged.


@app.exception_handler(StarletteHTTPException)
async def http_exception_handler(request: Request, exc: StarletteHTTPException):
 rid = getattr(request.state, "request_id", None) or current_request_id()
 content = quiz_errors.error_content(
 request.url.path,
 exc.status_code,
 exc.detail,
 rid,
 code=getattr(exc, "code", None),
 machine_detail=getattr(exc, "machine_detail", None),
 )
 return JSONResponse(
 status_code=exc.status_code,
 content={"detail": exc.detail, "request_id": rid},
 content=content,
 headers={"X-Request-ID": rid} if rid else {},
 )


@app.exception_handler(RequestValidationError)
async def validation_exception_handler(request: Request, exc: RequestValidationError):
 rid = getattr(request.state, "request_id", None) or current_request_id()
 code, message = quiz_errors.validation_error_code(exc.errors())
 if code is quiz_errors.QuizErrorCode.QUIZ_COUNT_OUT_OF_RANGE:
 message = (
 f"Quizzes can have between {quiz_config.QUIZ_MIN_QUESTIONS} "
 f"and {quiz_config.QUIZ_MAX_QUESTIONS} questions."
 )
 content = quiz_errors.error_content(
 request.url.path, 422, exc.errors(), rid,
 code=code, message=message, machine_detail=exc.errors(),
 )
 return JSONResponse(
 status_code=422,
 content={"detail": exc.errors(), "request_id": rid},
 content=content,
 headers={"X-Request-ID": rid} if rid else {},
 )

Expand All@@ -207,9 +232,12 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
async def unhandled_exception_handler(request: Request, exc: Exception):
 logging.getLogger("main").exception("Unhandled exception")
 rid = getattr(request.state, "request_id", None) or current_request_id()
 content = quiz_errors.error_content(
 request.url.path, 500, "Internal server error.", rid,
 )
 return JSONResponse(
 status_code=500,
 content={"detail": "Internal server error.", "request_id": rid},
 content=content,
 headers={"X-Request-ID": rid} if rid else {},
 )

Expand Down

7 changes: 6 additions & 1 deletion
backend/models/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
from typing import Optional, Union, List, Literal
from pydantic import BaseModel, Field

from services.quiz_config import QUIZ_MIN_QUESTIONS, QUIZ_MAX_QUESTIONS


# ── Learn ─────────────────────────────────────────────────────────────────────

Expand DownExpand Up@@ -46,7 +48,10 @@ class ActionBody(BaseModel):
class GenerateQuizBody(BaseModel):
 user_id: str = "user_andres"
 concept_node_id: str
 num_questions: int = Field(default=5, ge=1, le=10)
 # Bounds come from services/quiz_config.py — the same constants
 # GET /api/quiz/config serves, so the client can't offer a value
 # this model rejects (#540 A2).
 num_questions: int = Field(default=5, ge=QUIZ_MIN_QUESTIONS, le=QUIZ_MAX_QUESTIONS)
 difficulty: str = "medium"
 use_shared_context: bool = True
 # Mirrors the Learn-route fast/smart toggle so quiz generation has
Expand Down

131 changes: 110 additions & 21 deletions
backend/routes/quiz.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,12 @@
from routes.learn import _get_catalog_chunk
from services import events_service
from services.auth_guard import require_self
from services.quiz_config import (
 CONCRETE_DIFFICULTIES,
 REQUESTED_DIFFICULTIES,
 quiz_config_payload,
)
from services.quiz_errors import QuizAPIError, QuizErrorCode
from services.profiles import get_display_name
from services.encryption import encrypt_json, decrypt_json_column
from services.graph_service import apply_graph_update
Expand All@@ -35,8 +41,10 @@

PROMPTS_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "prompts")

# quiz_attempts.difficulty CHECK enum (0025).
VALID_DIFFICULTIES = {"easy", "medium", "hard"}
# Request-side difficulties live in services/quiz_config.py (#540 A2):
# the concrete trio matches the quiz_attempts.difficulty CHECK (0025,
# extended with 'adaptive' by the #540 migration); 'adaptive' hands the
# per-question mix decision to the agent (A1).


def _load_prompt(name: str) -> str:
Expand All@@ -62,6 +70,30 @@ def _load_prompt(name: str) -> str:

_OPTION_LABELS = ["A", "B", "C", "D", "E", "F"]

# Rank order for tie-breaking the overall difficulty report — derived from
# the config tuple so a difficulty added there can't be silently dropped by
# _resolved_difficulty's counting.
_DIFFICULTY_RANK = {d: i for i, d in enumerate(CONCRETE_DIFFICULTIES)}


def _resolved_difficulty(wire_questions: list[dict]) -> str:
 """The overall difficulty generation actually produced (#540 A1).

 Mode of the per-question difficulties; ties break to the harder value
 so the report never understates what the student is about to face.
 Defaults to 'medium' when nothing usable is present (can't happen for
 agent output — QuizQuestion.difficulty is a concrete Literal — but
 this also runs on stored legacy rows).
 """
 counts: dict[str, int] = {}
 for q in wire_questions:
 d = q.get("difficulty")
 if d in _DIFFICULTY_RANK:
 counts[d] = counts.get(d, 0) + 1
 if not counts:
 return "medium"
 return max(counts, key=lambda d: (counts[d], _DIFFICULTY_RANK[d]))


def _agent_question_to_wire(q: QuizQuestion, qid: int) -> dict | None:
 """Map an agent QuizQuestion to the legacy wire-format dict, or
Expand DownExpand Up@@ -237,8 +269,24 @@ async def _quiz_via_agent(
 # Keep this message routing-only; the workflow + adaptive rules
 # live in the system prompt. We just hand the agent the inputs it
 # needs and trust the prompt to drive tool calls.
 if difficulty == "adaptive":
 # #540 A1: no target difficulty — the agent picks the whole mix
 # from mastery + recent accuracy (ADAPTIVE MODE in the system
 # prompt). Every emitted question still carries a concrete
 # easy|medium|hard; the route reports the overall pick back to
 # the client as `resolved_difficulty`.
 difficulty_clause = (
 f"Generate {num_questions} questions in ADAPTIVE MODE: you "
 f"choose each question's difficulty (easy, medium, or hard) "
 f"from the student's mastery and recent accuracy, per the "
 f"adaptive-mode rules in your system prompt."
 )
 else:
 difficulty_clause = (
 f"Generate {num_questions} {difficulty} questions for the student."
 )
 routing_msg = (
 f"Generate {num_questions} {difficulty} questions for the student. "
 f"{difficulty_clause} "
 f"The target concept is '{concept_name}' "
 f"(concept_node_id={concept_node_id}). Follow the workflow in your "
 f"system prompt; pass concept_node_id='{concept_node_id}' to "
Expand DownExpand Up@@ -290,23 +338,40 @@ async def _quiz_via_agent(
 )
 return wire_questions

@router.get("/config")
def quiz_config():
 """Selector options for the quiz UI (#540 A2). Single source of truth:
 the same constants bound the Pydantic request model, so a client that
 builds its selects from this payload can never send a value the route
 rejects. No user data, no auth needed."""
 return quiz_config_payload()


@router.post("/generate")
async def generate_quiz(body: GenerateQuizBody, request: Request):
 require_self(body.user_id, request)
 # quiz_attempts.difficulty is CHECK-constrained (0025); reject drift before
 # we run the agent or write an attempt row.
 if body.difficulty not in VALID_DIFFICULTIES:
 raise HTTPException(
 # The concrete trio is CHECK-constrained on quiz_attempts (0025 +
 # the #540 'adaptive' extension); reject drift before we run the
 # agent or write an attempt row.
 if body.difficulty not in REQUESTED_DIFFICULTIES:
 raise QuizAPIError(
 status_code=400,
 detail=f"Invalid difficulty '{body.difficulty}'. "
 f"Must be one of {sorted(VALID_DIFFICULTIES)}.",
 code=QuizErrorCode.QUIZ_DIFFICULTY_INVALID,
 message=(
 "That difficulty isn't available. Choose easy, medium, "
 "hard, or adaptive."
 ),
 )
 node_rows = table("graph_nodes").select(
 "*",
 filters={"id": f"eq.{body.concept_node_id}", "user_id": f"eq.{body.user_id}"},
 )
 if not node_rows:
 raise HTTPException(status_code=404, detail="Concept node not found")
 raise QuizAPIError(
 status_code=404,
 code=QuizErrorCode.QUIZ_CONCEPT_NOT_FOUND,
 message="We couldn't find that concept in your knowledge graph.",
 )
 node = node_rows[0]
 course_id = node.get("course_id") or None
 concept_name = node.get("concept_name") or ""
Expand DownExpand Up@@ -339,15 +404,17 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
 # The raw-Gemini legacy fallback was retired in #145; degrade to 502
 # rather than serving a quiz from a second LLM path.
 logger.warning("Quiz agent guardrails tripped; returning 502", exc_info=e)
 raise HTTPException(
 raise QuizAPIError(
 status_code=502,
 detail="Quiz generation is temporarily unavailable. Please try again.",
 code=QuizErrorCode.QUIZ_GENERATION_FAILED,
 message="Quiz generation is temporarily unavailable. Please try again.",
 ) from e
 except Exception as e:
 logger.exception("Unexpected quiz-agent failure; returning 502")
 raise HTTPException(
 raise QuizAPIError(
 status_code=502,
 detail="Quiz generation is temporarily unavailable. Please try again.",
 code=QuizErrorCode.QUIZ_GENERATION_FAILED,
 message="Quiz generation is temporarily unavailable. Please try again.",
 ) from e

 quiz_id = str(uuid.uuid4())
Expand All@@ -372,14 +439,28 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
 "difficulty": body.difficulty,
 },
 )
 return {"quiz_id": quiz_id, "questions": questions}
 # #540 A1: echo what generation actually chose. requested_difficulty
 # is what the student asked for (may be 'adaptive');
 # resolved_difficulty is the overall mix the agent produced (always
 # concrete) — so the client can say "we picked hard for you" instead
 # of repeating the request back.
 return {
 "quiz_id": quiz_id,
 "questions": questions,
 "requested_difficulty": body.difficulty,
 "resolved_difficulty": _resolved_difficulty(questions),
 }


@router.post("/submit")
def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request: Request):
 attempt_rows = table("quiz_attempts").select("*", filters={"id": f"eq.{body.quiz_id}"})
 if not attempt_rows:
 raise HTTPException(status_code=404, detail="Quiz not found")
 raise QuizAPIError(
 status_code=404,
 code=QuizErrorCode.QUIZ_ATTEMPT_NOT_FOUND,
 message="We couldn't find that quiz.",
 )
 attempt = attempt_rows[0]

 user_id = attempt["user_id"]
Expand All@@ -395,8 +476,10 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request
 # 200: quiz_attempts stores no mastery_before/after, so faithfully
 # reconstructing the first response would need a migration.
 if attempt.get("completed_at"):
 raise HTTPException(
 status_code=409, detail="Quiz attempt has already been submitted"
 raise QuizAPIError(
 status_code=409,
 code=QuizErrorCode.QUIZ_ATTEMPT_ALREADY_COMPLETED,
 message="This quiz has already been submitted.",
 )
 # The read above is only the fast path — two CONCURRENT submits (a
 # double-click on the final submit) would both pass it. The atomic claim
Expand All@@ -410,8 +493,10 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request
 filters={"id": f"eq.{body.quiz_id}", "completed_at": "is.null"},
 )
 if not claimed:
 raise HTTPException(
 status_code=409, detail="Quiz attempt has already been submitted"
 raise QuizAPIError(
 status_code=409,
 code=QuizErrorCode.QUIZ_ATTEMPT_ALREADY_COMPLETED,
 message="This quiz has already been submitted.",
 )

 concept_node_id = attempt["concept_node_id"]
Expand DownExpand Up@@ -450,7 +535,11 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request
 filters={"id": f"eq.{concept_node_id}", "user_id": f"eq.{user_id}"},
 )
 if not node_rows:
 raise HTTPException(status_code=404, detail="Concept node not found")
 raise QuizAPIError(
 status_code=404,
 code=QuizErrorCode.QUIZ_CONCEPT_NOT_FOUND,
 message="We couldn't find that concept in your knowledge graph.",
 )
 node = node_rows[0]
 mastery_before = node["mastery_score"]
 mastery_after = max(0.0, min(1.0, mastery_before + (score * 0.03) - ((total - score) * 0.02)))
Expand Down
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
feat(quiz): adaptive difficulty, /api/quiz/config, stable error envelope (#540) by AndresL230 · Pull Request #547 · SaplingLearn/Sapling · GitHub
Skip to content
feat(quiz): adaptive difficulty, /api/quiz/config, stable error envelope (#540)#547

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

AndresL230
merged 3 commits into

from

Aug 12, 2026
 Merged


feat(quiz): adaptive difficulty, /api/quiz/config, stable error envelope (#540)
#547
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
afc97df

feat(quiz): adaptive difficulty, /api/quiz/config, stable error envel…


AndresL230 Aug 12, 2026
1ea4c72

fix(quiz): address #547 review — envelope precision, adaptive history…


AndresL230 Aug 12, 2026
1bd3512

test(e2e): tolerate the CustomSelect ✓ marker in the #540 selector jo…


AndresL230 Aug 12, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 12 additions & 0 deletions
backend/agents/quiz.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -113,6 +113,18 @@ class Quiz(BaseModel):
 "- Don't drop high-mastery, recently-reviewed concepts entirely; "
 " include 1 question on a strong-and-fresh concept to keep the "
 " quiz from feeling punishing.\n\n"
 "ADAPTIVE MODE (#540): when the user message says the quiz is in "
 "adaptive mode, there is no user-requested difficulty — you choose "
 "each question's difficulty yourself. Base the mix on mastery and "
 "`recent_attempts.accuracy`: struggling/low-accuracy concepts get "
 "easy-leaning questions, strong/high-accuracy ones get hard-leaning "
 "questions; with no history at all, center the mix on medium. The "
 "±1-step limits below do not apply in adaptive mode, but every "
 "question still carries a concrete easy|medium|hard difficulty — "
 "'adaptive' is never a per-question value. Note that PAST attempts "
 "in `recent_attempts` may carry difficulty 'adaptive' (that "
 "attempt's mix was agent-chosen); treat such an attempt's "
 "difficulty as unspecified and judge it by its accuracy alone.\n\n"
 "Adaptive-difficulty rules (use `recent_attempts.accuracy`):\n"
 "- If the most recent 2-3 attempts on this concept averaged < "
 " 0.5 accuracy, drop the difficulty mix one step from what the "
Expand Down

2 changes: 1 addition & 1 deletion
backend/db/e2e_checks/quiz.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -32,7 +32,7 @@ def run() -> None:

 # ── 1. Generate a quiz to obtain a real quiz_id ───────────────────────────
 # GenerateQuizBody: {user_id, concept_node_id, num_questions, difficulty, use_shared_context}
 # difficulty must be in {"easy","medium","hard"} (CHECK constraint from 0025).
 # difficulty ∈ {"easy","medium","hard","adaptive"} (0025 CHECK + the #540 adaptive migration).
 gen_r = client.post(
 "/api/quiz/generate",
 json={
Expand Down

32 changes: 32 additions & 0 deletions
backend/db/migrations/20260812204809_quiz_attempts_adaptive_difficulty.sql

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
-- #540 A1: 'adaptive' becomes a real request-side difficulty. The attempt row
-- records what the student asked for (the response's resolved_difficulty
-- reports what generation actually produced), so the 0025 CHECK
-- (easy|medium|hard) must admit 'adaptive'.
--
-- DEPLOY ORDER: widening the CHECK is backward-compatible — apply this
-- migration BEFORE deploying the code that accepts 'adaptive'. In the gap the
-- other way around, an adaptive generate runs the full LLM call and then 500s
-- on the INSERT. (The promote runner migrates before merging; for staging run
-- `python -m db.migrate` before the deploy picks up the code.)
--
-- The DROP is by introspection, not by an assumed default name: environments
-- with out-of-band table history (staging has had some) can carry the same
-- CHECK under a different name, and a name-keyed `DROP IF EXISTS` would
-- silently no-op and leave the narrow constraint alive beside the new one.
DO $$
DECLARE c record;
BEGIN
 FOR c IN
 SELECT conname
 FROM pg_constraint
 WHERE conrelid = 'quiz_attempts'::regclass
 AND contype = 'c'
 AND pg_get_constraintdef(oid) ILIKE '%difficulty%'
 LOOP
 EXECUTE format('ALTER TABLE quiz_attempts DROP CONSTRAINT %I', c.conname);
 END LOOP;
END $$;

ALTER TABLE quiz_attempts
 ADD CONSTRAINT quiz_attempts_difficulty_check
 CHECK (difficulty IN ('easy', 'medium', 'hard', 'adaptive'));

34 changes: 31 additions & 3 deletions
backend/main.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@
from routes.admin import router as admin_router
from routes.admin_analytics import router as admin_analytics_router
from routes.newsletter import router as newsletter_router
from services import quiz_config, quiz_errors
from services.logfire_scrubber import EXTRA_PATTERNS, scrub_value
from services.request_context import RequestIDMiddleware, current_request_id
from services.storage_service import (
Expand DownExpand Up@@ -183,22 +184,46 @@ def _drop_request_arguments(_request, _attributes):
app.add_middleware(RequestIDMiddleware)


# #540 A3: on /api/quiz/* paths, quiz_errors.error_content wraps errors in
# the coded envelope (QuizAPIError raise sites carry precise codes; plain
# HTTPExceptions fall back to a status-derived one); everywhere else it
# returns the legacy {detail, request_id} shape unchanged.


@app.exception_handler(StarletteHTTPException)
async def http_exception_handler(request: Request, exc: StarletteHTTPException):
 rid = getattr(request.state, "request_id", None) or current_request_id()
 content = quiz_errors.error_content(
 request.url.path,
 exc.status_code,
 exc.detail,
 rid,
 code=getattr(exc, "code", None),
 machine_detail=getattr(exc, "machine_detail", None),
 )
 return JSONResponse(
 status_code=exc.status_code,
 content={"detail": exc.detail, "request_id": rid},
 content=content,
 headers={"X-Request-ID": rid} if rid else {},
 )


@app.exception_handler(RequestValidationError)
async def validation_exception_handler(request: Request, exc: RequestValidationError):
 rid = getattr(request.state, "request_id", None) or current_request_id()
 code, message = quiz_errors.validation_error_code(exc.errors())
 if code is quiz_errors.QuizErrorCode.QUIZ_COUNT_OUT_OF_RANGE:
 message = (
 f"Quizzes can have between {quiz_config.QUIZ_MIN_QUESTIONS} "
 f"and {quiz_config.QUIZ_MAX_QUESTIONS} questions."
 )
 content = quiz_errors.error_content(
 request.url.path, 422, exc.errors(), rid,
 code=code, message=message, machine_detail=exc.errors(),
 )
 return JSONResponse(
 status_code=422,
 content={"detail": exc.errors(), "request_id": rid},
 content=content,
 headers={"X-Request-ID": rid} if rid else {},
 )

Expand All@@ -207,9 +232,12 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
async def unhandled_exception_handler(request: Request, exc: Exception):
 logging.getLogger("main").exception("Unhandled exception")
 rid = getattr(request.state, "request_id", None) or current_request_id()
 content = quiz_errors.error_content(
 request.url.path, 500, "Internal server error.", rid,
 )
 return JSONResponse(
 status_code=500,
 content={"detail": "Internal server error.", "request_id": rid},
 content=content,
 headers={"X-Request-ID": rid} if rid else {},
 )

Expand Down

7 changes: 6 additions & 1 deletion
backend/models/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
from typing import Optional, Union, List, Literal
from pydantic import BaseModel, Field

from services.quiz_config import QUIZ_MIN_QUESTIONS, QUIZ_MAX_QUESTIONS


# ── Learn ─────────────────────────────────────────────────────────────────────

Expand DownExpand Up@@ -46,7 +48,10 @@ class ActionBody(BaseModel):
class GenerateQuizBody(BaseModel):
 user_id: str = "user_andres"
 concept_node_id: str
 num_questions: int = Field(default=5, ge=1, le=10)
 # Bounds come from services/quiz_config.py — the same constants
 # GET /api/quiz/config serves, so the client can't offer a value
 # this model rejects (#540 A2).
 num_questions: int = Field(default=5, ge=QUIZ_MIN_QUESTIONS, le=QUIZ_MAX_QUESTIONS)
 difficulty: str = "medium"
 use_shared_context: bool = True
 # Mirrors the Learn-route fast/smart toggle so quiz generation has
Expand Down

131 changes: 110 additions & 21 deletions
backend/routes/quiz.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,12 @@
from routes.learn import _get_catalog_chunk
from services import events_service
from services.auth_guard import require_self
from services.quiz_config import (
 CONCRETE_DIFFICULTIES,
 REQUESTED_DIFFICULTIES,
 quiz_config_payload,
)
from services.quiz_errors import QuizAPIError, QuizErrorCode
from services.profiles import get_display_name
from services.encryption import encrypt_json, decrypt_json_column
from services.graph_service import apply_graph_update
Expand All@@ -35,8 +41,10 @@

PROMPTS_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "prompts")

# quiz_attempts.difficulty CHECK enum (0025).
VALID_DIFFICULTIES = {"easy", "medium", "hard"}
# Request-side difficulties live in services/quiz_config.py (#540 A2):
# the concrete trio matches the quiz_attempts.difficulty CHECK (0025,
# extended with 'adaptive' by the #540 migration); 'adaptive' hands the
# per-question mix decision to the agent (A1).


def _load_prompt(name: str) -> str:
Expand All@@ -62,6 +70,30 @@ def _load_prompt(name: str) -> str:

_OPTION_LABELS = ["A", "B", "C", "D", "E", "F"]

# Rank order for tie-breaking the overall difficulty report — derived from
# the config tuple so a difficulty added there can't be silently dropped by
# _resolved_difficulty's counting.
_DIFFICULTY_RANK = {d: i for i, d in enumerate(CONCRETE_DIFFICULTIES)}


def _resolved_difficulty(wire_questions: list[dict]) -> str:
 """The overall difficulty generation actually produced (#540 A1).

 Mode of the per-question difficulties; ties break to the harder value
 so the report never understates what the student is about to face.
 Defaults to 'medium' when nothing usable is present (can't happen for
 agent output — QuizQuestion.difficulty is a concrete Literal — but
 this also runs on stored legacy rows).
 """
 counts: dict[str, int] = {}
 for q in wire_questions:
 d = q.get("difficulty")
 if d in _DIFFICULTY_RANK:
 counts[d] = counts.get(d, 0) + 1
 if not counts:
 return "medium"
 return max(counts, key=lambda d: (counts[d], _DIFFICULTY_RANK[d]))


def _agent_question_to_wire(q: QuizQuestion, qid: int) -> dict | None:
 """Map an agent QuizQuestion to the legacy wire-format dict, or
Expand DownExpand Up@@ -237,8 +269,24 @@ async def _quiz_via_agent(
 # Keep this message routing-only; the workflow + adaptive rules
 # live in the system prompt. We just hand the agent the inputs it
 # needs and trust the prompt to drive tool calls.
 if difficulty == "adaptive":
 # #540 A1: no target difficulty — the agent picks the whole mix
 # from mastery + recent accuracy (ADAPTIVE MODE in the system
 # prompt). Every emitted question still carries a concrete
 # easy|medium|hard; the route reports the overall pick back to
 # the client as `resolved_difficulty`.
 difficulty_clause = (
 f"Generate {num_questions} questions in ADAPTIVE MODE: you "
 f"choose each question's difficulty (easy, medium, or hard) "
 f"from the student's mastery and recent accuracy, per the "
 f"adaptive-mode rules in your system prompt."
 )
 else:
 difficulty_clause = (
 f"Generate {num_questions} {difficulty} questions for the student."
 )
 routing_msg = (
 f"Generate {num_questions} {difficulty} questions for the student. "
 f"{difficulty_clause} "
 f"The target concept is '{concept_name}' "
 f"(concept_node_id={concept_node_id}). Follow the workflow in your "
 f"system prompt; pass concept_node_id='{concept_node_id}' to "
Expand DownExpand Up@@ -290,23 +338,40 @@ async def _quiz_via_agent(
 )
 return wire_questions

@router.get("/config")
def quiz_config():
 """Selector options for the quiz UI (#540 A2). Single source of truth:
 the same constants bound the Pydantic request model, so a client that
 builds its selects from this payload can never send a value the route
 rejects. No user data, no auth needed."""
 return quiz_config_payload()


@router.post("/generate")
async def generate_quiz(body: GenerateQuizBody, request: Request):
 require_self(body.user_id, request)
 # quiz_attempts.difficulty is CHECK-constrained (0025); reject drift before
 # we run the agent or write an attempt row.
 if body.difficulty not in VALID_DIFFICULTIES:
 raise HTTPException(
 # The concrete trio is CHECK-constrained on quiz_attempts (0025 +
 # the #540 'adaptive' extension); reject drift before we run the
 # agent or write an attempt row.
 if body.difficulty not in REQUESTED_DIFFICULTIES:
 raise QuizAPIError(
 status_code=400,
 detail=f"Invalid difficulty '{body.difficulty}'. "
 f"Must be one of {sorted(VALID_DIFFICULTIES)}.",
 code=QuizErrorCode.QUIZ_DIFFICULTY_INVALID,
 message=(
 "That difficulty isn't available. Choose easy, medium, "
 "hard, or adaptive."
 ),
 )
 node_rows = table("graph_nodes").select(
 "*",
 filters={"id": f"eq.{body.concept_node_id}", "user_id": f"eq.{body.user_id}"},
 )
 if not node_rows:
 raise HTTPException(status_code=404, detail="Concept node not found")
 raise QuizAPIError(
 status_code=404,
 code=QuizErrorCode.QUIZ_CONCEPT_NOT_FOUND,
 message="We couldn't find that concept in your knowledge graph.",
 )
 node = node_rows[0]
 course_id = node.get("course_id") or None
 concept_name = node.get("concept_name") or ""
Expand DownExpand Up@@ -339,15 +404,17 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
 # The raw-Gemini legacy fallback was retired in #145; degrade to 502
 # rather than serving a quiz from a second LLM path.
 logger.warning("Quiz agent guardrails tripped; returning 502", exc_info=e)
 raise HTTPException(
 raise QuizAPIError(
 status_code=502,
 detail="Quiz generation is temporarily unavailable. Please try again.",
 code=QuizErrorCode.QUIZ_GENERATION_FAILED,
 message="Quiz generation is temporarily unavailable. Please try again.",
 ) from e
 except Exception as e:
 logger.exception("Unexpected quiz-agent failure; returning 502")
 raise HTTPException(
 raise QuizAPIError(
 status_code=502,
 detail="Quiz generation is temporarily unavailable. Please try again.",
 code=QuizErrorCode.QUIZ_GENERATION_FAILED,
 message="Quiz generation is temporarily unavailable. Please try again.",
 ) from e

 quiz_id = str(uuid.uuid4())
Expand All@@ -372,14 +439,28 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
 "difficulty": body.difficulty,
 },
 )
 return {"quiz_id": quiz_id, "questions": questions}
 # #540 A1: echo what generation actually chose. requested_difficulty
 # is what the student asked for (may be 'adaptive');
 # resolved_difficulty is the overall mix the agent produced (always
 # concrete) — so the client can say "we picked hard for you" instead
 # of repeating the request back.
 return {
 "quiz_id": quiz_id,
 "questions": questions,
 "requested_difficulty": body.difficulty,
 "resolved_difficulty": _resolved_difficulty(questions),
 }


@router.post("/submit")
def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request: Request):
 attempt_rows = table("quiz_attempts").select("*", filters={"id": f"eq.{body.quiz_id}"})
 if not attempt_rows:
 raise HTTPException(status_code=404, detail="Quiz not found")
 raise QuizAPIError(
 status_code=404,
 code=QuizErrorCode.QUIZ_ATTEMPT_NOT_FOUND,
 message="We couldn't find that quiz.",
 )
 attempt = attempt_rows[0]

 user_id = attempt["user_id"]
Expand All@@ -395,8 +476,10 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request
 # 200: quiz_attempts stores no mastery_before/after, so faithfully
 # reconstructing the first response would need a migration.
 if attempt.get("completed_at"):
 raise HTTPException(
 status_code=409, detail="Quiz attempt has already been submitted"
 raise QuizAPIError(
 status_code=409,
 code=QuizErrorCode.QUIZ_ATTEMPT_ALREADY_COMPLETED,
 message="This quiz has already been submitted.",
 )
 # The read above is only the fast path — two CONCURRENT submits (a
 # double-click on the final submit) would both pass it. The atomic claim
Expand All@@ -410,8 +493,10 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request
 filters={"id": f"eq.{body.quiz_id}", "completed_at": "is.null"},
 )
 if not claimed:
 raise HTTPException(
 status_code=409, detail="Quiz attempt has already been submitted"
 raise QuizAPIError(
 status_code=409,
 code=QuizErrorCode.QUIZ_ATTEMPT_ALREADY_COMPLETED,
 message="This quiz has already been submitted.",
 )

 concept_node_id = attempt["concept_node_id"]
Expand DownExpand Up@@ -450,7 +535,11 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request
 filters={"id": f"eq.{concept_node_id}", "user_id": f"eq.{user_id}"},
 )
 if not node_rows:
 raise HTTPException(status_code=404, detail="Concept node not found")
 raise QuizAPIError(
 status_code=404,
 code=QuizErrorCode.QUIZ_CONCEPT_NOT_FOUND,
 message="We couldn't find that concept in your knowledge graph.",
 )
 node = node_rows[0]
 mastery_before = node["mastery_score"]
 mastery_after = max(0.0, min(1.0, mastery_before + (score * 0.03) - ((total - score) * 0.02)))
Expand Down
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '
feat(quiz): adaptive difficulty, /api/quiz/config, stable error envelope (#540) by AndresL230 · Pull Request #547 · SaplingLearn/Sapling · GitHub
Skip to content
feat(quiz): adaptive difficulty, /api/quiz/config, stable error envelope (#540)#547

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

AndresL230
merged 3 commits into

from

Aug 12, 2026
 Merged


feat(quiz): adaptive difficulty, /api/quiz/config, stable error envelope (#540)
#547
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
afc97df

feat(quiz): adaptive difficulty, /api/quiz/config, stable error envel…


AndresL230 Aug 12, 2026
1ea4c72

fix(quiz): address #547 review — envelope precision, adaptive history…


AndresL230 Aug 12, 2026
1bd3512

test(e2e): tolerate the CustomSelect ✓ marker in the #540 selector jo…


AndresL230 Aug 12, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 12 additions & 0 deletions
backend/agents/quiz.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -113,6 +113,18 @@ class Quiz(BaseModel):
 "- Don't drop high-mastery, recently-reviewed concepts entirely; "
 " include 1 question on a strong-and-fresh concept to keep the "
 " quiz from feeling punishing.\n\n"
 "ADAPTIVE MODE (#540): when the user message says the quiz is in "
 "adaptive mode, there is no user-requested difficulty — you choose "
 "each question's difficulty yourself. Base the mix on mastery and "
 "`recent_attempts.accuracy`: struggling/low-accuracy concepts get "
 "easy-leaning questions, strong/high-accuracy ones get hard-leaning "
 "questions; with no history at all, center the mix on medium. The "
 "±1-step limits below do not apply in adaptive mode, but every "
 "question still carries a concrete easy|medium|hard difficulty — "
 "'adaptive' is never a per-question value. Note that PAST attempts "
 "in `recent_attempts` may carry difficulty 'adaptive' (that "
 "attempt's mix was agent-chosen); treat such an attempt's "
 "difficulty as unspecified and judge it by its accuracy alone.\n\n"
 "Adaptive-difficulty rules (use `recent_attempts.accuracy`):\n"
 "- If the most recent 2-3 attempts on this concept averaged < "
 " 0.5 accuracy, drop the difficulty mix one step from what the "
Expand Down

2 changes: 1 addition & 1 deletion
backend/db/e2e_checks/quiz.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -32,7 +32,7 @@ def run() -> None:

 # ── 1. Generate a quiz to obtain a real quiz_id ───────────────────────────
 # GenerateQuizBody: {user_id, concept_node_id, num_questions, difficulty, use_shared_context}
 # difficulty must be in {"easy","medium","hard"} (CHECK constraint from 0025).
 # difficulty ∈ {"easy","medium","hard","adaptive"} (0025 CHECK + the #540 adaptive migration).
 gen_r = client.post(
 "/api/quiz/generate",
 json={
Expand Down

32 changes: 32 additions & 0 deletions
backend/db/migrations/20260812204809_quiz_attempts_adaptive_difficulty.sql

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
-- #540 A1: 'adaptive' becomes a real request-side difficulty. The attempt row
-- records what the student asked for (the response's resolved_difficulty
-- reports what generation actually produced), so the 0025 CHECK
-- (easy|medium|hard) must admit 'adaptive'.
--
-- DEPLOY ORDER: widening the CHECK is backward-compatible — apply this
-- migration BEFORE deploying the code that accepts 'adaptive'. In the gap the
-- other way around, an adaptive generate runs the full LLM call and then 500s
-- on the INSERT. (The promote runner migrates before merging; for staging run
-- `python -m db.migrate` before the deploy picks up the code.)
--
-- The DROP is by introspection, not by an assumed default name: environments
-- with out-of-band table history (staging has had some) can carry the same
-- CHECK under a different name, and a name-keyed `DROP IF EXISTS` would
-- silently no-op and leave the narrow constraint alive beside the new one.
DO $$
DECLARE c record;
BEGIN
 FOR c IN
 SELECT conname
 FROM pg_constraint
 WHERE conrelid = 'quiz_attempts'::regclass
 AND contype = 'c'
 AND pg_get_constraintdef(oid) ILIKE '%difficulty%'
 LOOP
 EXECUTE format('ALTER TABLE quiz_attempts DROP CONSTRAINT %I', c.conname);
 END LOOP;
END $$;

ALTER TABLE quiz_attempts
 ADD CONSTRAINT quiz_attempts_difficulty_check
 CHECK (difficulty IN ('easy', 'medium', 'hard', 'adaptive'));

34 changes: 31 additions & 3 deletions
backend/main.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@
from routes.admin import router as admin_router
from routes.admin_analytics import router as admin_analytics_router
from routes.newsletter import router as newsletter_router
from services import quiz_config, quiz_errors
from services.logfire_scrubber import EXTRA_PATTERNS, scrub_value
from services.request_context import RequestIDMiddleware, current_request_id
from services.storage_service import (
Expand DownExpand Up@@ -183,22 +184,46 @@ def _drop_request_arguments(_request, _attributes):
app.add_middleware(RequestIDMiddleware)


# #540 A3: on /api/quiz/* paths, quiz_errors.error_content wraps errors in
# the coded envelope (QuizAPIError raise sites carry precise codes; plain
# HTTPExceptions fall back to a status-derived one); everywhere else it
# returns the legacy {detail, request_id} shape unchanged.


@app.exception_handler(StarletteHTTPException)
async def http_exception_handler(request: Request, exc: StarletteHTTPException):
 rid = getattr(request.state, "request_id", None) or current_request_id()
 content = quiz_errors.error_content(
 request.url.path,
 exc.status_code,
 exc.detail,
 rid,
 code=getattr(exc, "code", None),
 machine_detail=getattr(exc, "machine_detail", None),
 )
 return JSONResponse(
 status_code=exc.status_code,
 content={"detail": exc.detail, "request_id": rid},
 content=content,
 headers={"X-Request-ID": rid} if rid else {},
 )


@app.exception_handler(RequestValidationError)
async def validation_exception_handler(request: Request, exc: RequestValidationError):
 rid = getattr(request.state, "request_id", None) or current_request_id()
 code, message = quiz_errors.validation_error_code(exc.errors())
 if code is quiz_errors.QuizErrorCode.QUIZ_COUNT_OUT_OF_RANGE:
 message = (
 f"Quizzes can have between {quiz_config.QUIZ_MIN_QUESTIONS} "
 f"and {quiz_config.QUIZ_MAX_QUESTIONS} questions."
 )
 content = quiz_errors.error_content(
 request.url.path, 422, exc.errors(), rid,
 code=code, message=message, machine_detail=exc.errors(),
 )
 return JSONResponse(
 status_code=422,
 content={"detail": exc.errors(), "request_id": rid},
 content=content,
 headers={"X-Request-ID": rid} if rid else {},
 )

Expand All@@ -207,9 +232,12 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
async def unhandled_exception_handler(request: Request, exc: Exception):
 logging.getLogger("main").exception("Unhandled exception")
 rid = getattr(request.state, "request_id", None) or current_request_id()
 content = quiz_errors.error_content(
 request.url.path, 500, "Internal server error.", rid,
 )
 return JSONResponse(
 status_code=500,
 content={"detail": "Internal server error.", "request_id": rid},
 content=content,
 headers={"X-Request-ID": rid} if rid else {},
 )

Expand Down

7 changes: 6 additions & 1 deletion
backend/models/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
from typing import Optional, Union, List, Literal
from pydantic import BaseModel, Field

from services.quiz_config import QUIZ_MIN_QUESTIONS, QUIZ_MAX_QUESTIONS


# ── Learn ─────────────────────────────────────────────────────────────────────

Expand DownExpand Up@@ -46,7 +48,10 @@ class ActionBody(BaseModel):
class GenerateQuizBody(BaseModel):
 user_id: str = "user_andres"
 concept_node_id: str
 num_questions: int = Field(default=5, ge=1, le=10)
 # Bounds come from services/quiz_config.py — the same constants
 # GET /api/quiz/config serves, so the client can't offer a value
 # this model rejects (#540 A2).
 num_questions: int = Field(default=5, ge=QUIZ_MIN_QUESTIONS, le=QUIZ_MAX_QUESTIONS)
 difficulty: str = "medium"
 use_shared_context: bool = True
 # Mirrors the Learn-route fast/smart toggle so quiz generation has
Expand Down

131 changes: 110 additions & 21 deletions
backend/routes/quiz.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,12 @@
from routes.learn import _get_catalog_chunk
from services import events_service
from services.auth_guard import require_self
from services.quiz_config import (
 CONCRETE_DIFFICULTIES,
 REQUESTED_DIFFICULTIES,
 quiz_config_payload,
)
from services.quiz_errors import QuizAPIError, QuizErrorCode
from services.profiles import get_display_name
from services.encryption import encrypt_json, decrypt_json_column
from services.graph_service import apply_graph_update
Expand All@@ -35,8 +41,10 @@

PROMPTS_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "prompts")

# quiz_attempts.difficulty CHECK enum (0025).
VALID_DIFFICULTIES = {"easy", "medium", "hard"}
# Request-side difficulties live in services/quiz_config.py (#540 A2):
# the concrete trio matches the quiz_attempts.difficulty CHECK (0025,
# extended with 'adaptive' by the #540 migration); 'adaptive' hands the
# per-question mix decision to the agent (A1).


def _load_prompt(name: str) -> str:
Expand All@@ -62,6 +70,30 @@ def _load_prompt(name: str) -> str:

_OPTION_LABELS = ["A", "B", "C", "D", "E", "F"]

# Rank order for tie-breaking the overall difficulty report — derived from
# the config tuple so a difficulty added there can't be silently dropped by
# _resolved_difficulty's counting.
_DIFFICULTY_RANK = {d: i for i, d in enumerate(CONCRETE_DIFFICULTIES)}


def _resolved_difficulty(wire_questions: list[dict]) -> str:
 """The overall difficulty generation actually produced (#540 A1).

 Mode of the per-question difficulties; ties break to the harder value
 so the report never understates what the student is about to face.
 Defaults to 'medium' when nothing usable is present (can't happen for
 agent output — QuizQuestion.difficulty is a concrete Literal — but
 this also runs on stored legacy rows).
 """
 counts: dict[str, int] = {}
 for q in wire_questions:
 d = q.get("difficulty")
 if d in _DIFFICULTY_RANK:
 counts[d] = counts.get(d, 0) + 1
 if not counts:
 return "medium"
 return max(counts, key=lambda d: (counts[d], _DIFFICULTY_RANK[d]))


def _agent_question_to_wire(q: QuizQuestion, qid: int) -> dict | None:
 """Map an agent QuizQuestion to the legacy wire-format dict, or
Expand DownExpand Up@@ -237,8 +269,24 @@ async def _quiz_via_agent(
 # Keep this message routing-only; the workflow + adaptive rules
 # live in the system prompt. We just hand the agent the inputs it
 # needs and trust the prompt to drive tool calls.
 if difficulty == "adaptive":
 # #540 A1: no target difficulty — the agent picks the whole mix
 # from mastery + recent accuracy (ADAPTIVE MODE in the system
 # prompt). Every emitted question still carries a concrete
 # easy|medium|hard; the route reports the overall pick back to
 # the client as `resolved_difficulty`.
 difficulty_clause = (
 f"Generate {num_questions} questions in ADAPTIVE MODE: you "
 f"choose each question's difficulty (easy, medium, or hard) "
 f"from the student's mastery and recent accuracy, per the "
 f"adaptive-mode rules in your system prompt."
 )
 else:
 difficulty_clause = (
 f"Generate {num_questions} {difficulty} questions for the student."
 )
 routing_msg = (
 f"Generate {num_questions} {difficulty} questions for the student. "
 f"{difficulty_clause} "
 f"The target concept is '{concept_name}' "
 f"(concept_node_id={concept_node_id}). Follow the workflow in your "
 f"system prompt; pass concept_node_id='{concept_node_id}' to "
Expand DownExpand Up@@ -290,23 +338,40 @@ async def _quiz_via_agent(
 )
 return wire_questions

@router.get("/config")
def quiz_config():
 """Selector options for the quiz UI (#540 A2). Single source of truth:
 the same constants bound the Pydantic request model, so a client that
 builds its selects from this payload can never send a value the route
 rejects. No user data, no auth needed."""
 return quiz_config_payload()


@router.post("/generate")
async def generate_quiz(body: GenerateQuizBody, request: Request):
 require_self(body.user_id, request)
 # quiz_attempts.difficulty is CHECK-constrained (0025); reject drift before
 # we run the agent or write an attempt row.
 if body.difficulty not in VALID_DIFFICULTIES:
 raise HTTPException(
 # The concrete trio is CHECK-constrained on quiz_attempts (0025 +
 # the #540 'adaptive' extension); reject drift before we run the
 # agent or write an attempt row.
 if body.difficulty not in REQUESTED_DIFFICULTIES:
 raise QuizAPIError(
 status_code=400,
 detail=f"Invalid difficulty '{body.difficulty}'. "
 f"Must be one of {sorted(VALID_DIFFICULTIES)}.",
 code=QuizErrorCode.QUIZ_DIFFICULTY_INVALID,
 message=(
 "That difficulty isn't available. Choose easy, medium, "
 "hard, or adaptive."
 ),
 )
 node_rows = table("graph_nodes").select(
 "*",
 filters={"id": f"eq.{body.concept_node_id}", "user_id": f"eq.{body.user_id}"},
 )
 if not node_rows:
 raise HTTPException(status_code=404, detail="Concept node not found")
 raise QuizAPIError(
 status_code=404,
 code=QuizErrorCode.QUIZ_CONCEPT_NOT_FOUND,
 message="We couldn't find that concept in your knowledge graph.",
 )
 node = node_rows[0]
 course_id = node.get("course_id") or None
 concept_name = node.get("concept_name") or ""
Expand DownExpand Up@@ -339,15 +404,17 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
 # The raw-Gemini legacy fallback was retired in #145; degrade to 502
 # rather than serving a quiz from a second LLM path.
 logger.warning("Quiz agent guardrails tripped; returning 502", exc_info=e)
 raise HTTPException(
 raise QuizAPIError(
 status_code=502,
 detail="Quiz generation is temporarily unavailable. Please try again.",
 code=QuizErrorCode.QUIZ_GENERATION_FAILED,
 message="Quiz generation is temporarily unavailable. Please try again.",
 ) from e
 except Exception as e:
 logger.exception("Unexpected quiz-agent failure; returning 502")
 raise HTTPException(
 raise QuizAPIError(
 status_code=502,
 detail="Quiz generation is temporarily unavailable. Please try again.",
 code=QuizErrorCode.QUIZ_GENERATION_FAILED,
 message="Quiz generation is temporarily unavailable. Please try again.",
 ) from e

 quiz_id = str(uuid.uuid4())
Expand All@@ -372,14 +439,28 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
 "difficulty": body.difficulty,
 },
 )
 return {"quiz_id": quiz_id, "questions": questions}
 # #540 A1: echo what generation actually chose. requested_difficulty
 # is what the student asked for (may be 'adaptive');
 # resolved_difficulty is the overall mix the agent produced (always
 # concrete) — so the client can say "we picked hard for you" instead
 # of repeating the request back.
 return {
 "quiz_id": quiz_id,
 "questions": questions,
 "requested_difficulty": body.difficulty,
 "resolved_difficulty": _resolved_difficulty(questions),
 }


@router.post("/submit")
def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request: Request):
 attempt_rows = table("quiz_attempts").select("*", filters={"id": f"eq.{body.quiz_id}"})
 if not attempt_rows:
 raise HTTPException(status_code=404, detail="Quiz not found")
 raise QuizAPIError(
 status_code=404,
 code=QuizErrorCode.QUIZ_ATTEMPT_NOT_FOUND,
 message="We couldn't find that quiz.",
 )
 attempt = attempt_rows[0]

 user_id = attempt["user_id"]
Expand All@@ -395,8 +476,10 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request
 # 200: quiz_attempts stores no mastery_before/after, so faithfully
 # reconstructing the first response would need a migration.
 if attempt.get("completed_at"):
 raise HTTPException(
 status_code=409, detail="Quiz attempt has already been submitted"
 raise QuizAPIError(
 status_code=409,
 code=QuizErrorCode.QUIZ_ATTEMPT_ALREADY_COMPLETED,
 message="This quiz has already been submitted.",
 )
 # The read above is only the fast path — two CONCURRENT submits (a
 # double-click on the final submit) would both pass it. The atomic claim
Expand All@@ -410,8 +493,10 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request
 filters={"id": f"eq.{body.quiz_id}", "completed_at": "is.null"},
 )
 if not claimed:
 raise HTTPException(
 status_code=409, detail="Quiz attempt has already been submitted"
 raise QuizAPIError(
 status_code=409,
 code=QuizErrorCode.QUIZ_ATTEMPT_ALREADY_COMPLETED,
 message="This quiz has already been submitted.",
 )

 concept_node_id = attempt["concept_node_id"]
Expand DownExpand Up@@ -450,7 +535,11 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request
 filters={"id": f"eq.{concept_node_id}", "user_id": f"eq.{user_id}"},
 )
 if not node_rows:
 raise HTTPException(status_code=404, detail="Concept node not found")
 raise QuizAPIError(
 status_code=404,
 code=QuizErrorCode.QUIZ_CONCEPT_NOT_FOUND,
 message="We couldn't find that concept in your knowledge graph.",
 )
 node = node_rows[0]
 mastery_before = node["mastery_score"]
 mastery_after = max(0.0, min(1.0, mastery_before + (score * 0.03) - ((total - score) * 0.02)))
Expand Down
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
feat(quiz): adaptive difficulty, /api/quiz/config, stable error envelope (#540) by AndresL230 · Pull Request #547 · SaplingLearn/Sapling · GitHub
Skip to content
feat(quiz): adaptive difficulty, /api/quiz/config, stable error envelope (#540)#547

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

AndresL230
merged 3 commits into

from

Aug 12, 2026
 Merged


feat(quiz): adaptive difficulty, /api/quiz/config, stable error envelope (#540)
#547
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
afc97df

feat(quiz): adaptive difficulty, /api/quiz/config, stable error envel…


AndresL230 Aug 12, 2026
1ea4c72

fix(quiz): address #547 review — envelope precision, adaptive history…


AndresL230 Aug 12, 2026
1bd3512

test(e2e): tolerate the CustomSelect ✓ marker in the #540 selector jo…


AndresL230 Aug 12, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 12 additions & 0 deletions
backend/agents/quiz.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -113,6 +113,18 @@ class Quiz(BaseModel):
 "- Don't drop high-mastery, recently-reviewed concepts entirely; "
 " include 1 question on a strong-and-fresh concept to keep the "
 " quiz from feeling punishing.\n\n"
 "ADAPTIVE MODE (#540): when the user message says the quiz is in "
 "adaptive mode, there is no user-requested difficulty — you choose "
 "each question's difficulty yourself. Base the mix on mastery and "
 "`recent_attempts.accuracy`: struggling/low-accuracy concepts get "
 "easy-leaning questions, strong/high-accuracy ones get hard-leaning "
 "questions; with no history at all, center the mix on medium. The "
 "±1-step limits below do not apply in adaptive mode, but every "
 "question still carries a concrete easy|medium|hard difficulty — "
 "'adaptive' is never a per-question value. Note that PAST attempts "
 "in `recent_attempts` may carry difficulty 'adaptive' (that "
 "attempt's mix was agent-chosen); treat such an attempt's "
 "difficulty as unspecified and judge it by its accuracy alone.\n\n"
 "Adaptive-difficulty rules (use `recent_attempts.accuracy`):\n"
 "- If the most recent 2-3 attempts on this concept averaged < "
 " 0.5 accuracy, drop the difficulty mix one step from what the "
Expand Down

2 changes: 1 addition & 1 deletion
backend/db/e2e_checks/quiz.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -32,7 +32,7 @@ def run() -> None:

 # ── 1. Generate a quiz to obtain a real quiz_id ───────────────────────────
 # GenerateQuizBody: {user_id, concept_node_id, num_questions, difficulty, use_shared_context}
 # difficulty must be in {"easy","medium","hard"} (CHECK constraint from 0025).
 # difficulty ∈ {"easy","medium","hard","adaptive"} (0025 CHECK + the #540 adaptive migration).
 gen_r = client.post(
 "/api/quiz/generate",
 json={
Expand Down

32 changes: 32 additions & 0 deletions
backend/db/migrations/20260812204809_quiz_attempts_adaptive_difficulty.sql

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
-- #540 A1: 'adaptive' becomes a real request-side difficulty. The attempt row
-- records what the student asked for (the response's resolved_difficulty
-- reports what generation actually produced), so the 0025 CHECK
-- (easy|medium|hard) must admit 'adaptive'.
--
-- DEPLOY ORDER: widening the CHECK is backward-compatible — apply this
-- migration BEFORE deploying the code that accepts 'adaptive'. In the gap the
-- other way around, an adaptive generate runs the full LLM call and then 500s
-- on the INSERT. (The promote runner migrates before merging; for staging run
-- `python -m db.migrate` before the deploy picks up the code.)
--
-- The DROP is by introspection, not by an assumed default name: environments
-- with out-of-band table history (staging has had some) can carry the same
-- CHECK under a different name, and a name-keyed `DROP IF EXISTS` would
-- silently no-op and leave the narrow constraint alive beside the new one.
DO $$
DECLARE c record;
BEGIN
 FOR c IN
 SELECT conname
 FROM pg_constraint
 WHERE conrelid = 'quiz_attempts'::regclass
 AND contype = 'c'
 AND pg_get_constraintdef(oid) ILIKE '%difficulty%'
 LOOP
 EXECUTE format('ALTER TABLE quiz_attempts DROP CONSTRAINT %I', c.conname);
 END LOOP;
END $$;

ALTER TABLE quiz_attempts
 ADD CONSTRAINT quiz_attempts_difficulty_check
 CHECK (difficulty IN ('easy', 'medium', 'hard', 'adaptive'));

34 changes: 31 additions & 3 deletions
backend/main.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@
from routes.admin import router as admin_router
from routes.admin_analytics import router as admin_analytics_router
from routes.newsletter import router as newsletter_router
from services import quiz_config, quiz_errors
from services.logfire_scrubber import EXTRA_PATTERNS, scrub_value
from services.request_context import RequestIDMiddleware, current_request_id
from services.storage_service import (
Expand DownExpand Up@@ -183,22 +184,46 @@ def _drop_request_arguments(_request, _attributes):
app.add_middleware(RequestIDMiddleware)


# #540 A3: on /api/quiz/* paths, quiz_errors.error_content wraps errors in
# the coded envelope (QuizAPIError raise sites carry precise codes; plain
# HTTPExceptions fall back to a status-derived one); everywhere else it
# returns the legacy {detail, request_id} shape unchanged.


@app.exception_handler(StarletteHTTPException)
async def http_exception_handler(request: Request, exc: StarletteHTTPException):
 rid = getattr(request.state, "request_id", None) or current_request_id()
 content = quiz_errors.error_content(
 request.url.path,
 exc.status_code,
 exc.detail,
 rid,
 code=getattr(exc, "code", None),
 machine_detail=getattr(exc, "machine_detail", None),
 )
 return JSONResponse(
 status_code=exc.status_code,
 content={"detail": exc.detail, "request_id": rid},
 content=content,
 headers={"X-Request-ID": rid} if rid else {},
 )


@app.exception_handler(RequestValidationError)
async def validation_exception_handler(request: Request, exc: RequestValidationError):
 rid = getattr(request.state, "request_id", None) or current_request_id()
 code, message = quiz_errors.validation_error_code(exc.errors())
 if code is quiz_errors.QuizErrorCode.QUIZ_COUNT_OUT_OF_RANGE:
 message = (
 f"Quizzes can have between {quiz_config.QUIZ_MIN_QUESTIONS} "
 f"and {quiz_config.QUIZ_MAX_QUESTIONS} questions."
 )
 content = quiz_errors.error_content(
 request.url.path, 422, exc.errors(), rid,
 code=code, message=message, machine_detail=exc.errors(),
 )
 return JSONResponse(
 status_code=422,
 content={"detail": exc.errors(), "request_id": rid},
 content=content,
 headers={"X-Request-ID": rid} if rid else {},
 )

Expand All@@ -207,9 +232,12 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
async def unhandled_exception_handler(request: Request, exc: Exception):
 logging.getLogger("main").exception("Unhandled exception")
 rid = getattr(request.state, "request_id", None) or current_request_id()
 content = quiz_errors.error_content(
 request.url.path, 500, "Internal server error.", rid,
 )
 return JSONResponse(
 status_code=500,
 content={"detail": "Internal server error.", "request_id": rid},
 content=content,
 headers={"X-Request-ID": rid} if rid else {},
 )

Expand Down

7 changes: 6 additions & 1 deletion
backend/models/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
from typing import Optional, Union, List, Literal
from pydantic import BaseModel, Field

from services.quiz_config import QUIZ_MIN_QUESTIONS, QUIZ_MAX_QUESTIONS


# ── Learn ─────────────────────────────────────────────────────────────────────

Expand DownExpand Up@@ -46,7 +48,10 @@ class ActionBody(BaseModel):
class GenerateQuizBody(BaseModel):
 user_id: str = "user_andres"
 concept_node_id: str
 num_questions: int = Field(default=5, ge=1, le=10)
 # Bounds come from services/quiz_config.py — the same constants
 # GET /api/quiz/config serves, so the client can't offer a value
 # this model rejects (#540 A2).
 num_questions: int = Field(default=5, ge=QUIZ_MIN_QUESTIONS, le=QUIZ_MAX_QUESTIONS)
 difficulty: str = "medium"
 use_shared_context: bool = True
 # Mirrors the Learn-route fast/smart toggle so quiz generation has
Expand Down

131 changes: 110 additions & 21 deletions
backend/routes/quiz.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,12 @@
from routes.learn import _get_catalog_chunk
from services import events_service
from services.auth_guard import require_self
from services.quiz_config import (
 CONCRETE_DIFFICULTIES,
 REQUESTED_DIFFICULTIES,
 quiz_config_payload,
)
from services.quiz_errors import QuizAPIError, QuizErrorCode
from services.profiles import get_display_name
from services.encryption import encrypt_json, decrypt_json_column
from services.graph_service import apply_graph_update
Expand All@@ -35,8 +41,10 @@

PROMPTS_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "prompts")

# quiz_attempts.difficulty CHECK enum (0025).
VALID_DIFFICULTIES = {"easy", "medium", "hard"}
# Request-side difficulties live in services/quiz_config.py (#540 A2):
# the concrete trio matches the quiz_attempts.difficulty CHECK (0025,
# extended with 'adaptive' by the #540 migration); 'adaptive' hands the
# per-question mix decision to the agent (A1).


def _load_prompt(name: str) -> str:
Expand All@@ -62,6 +70,30 @@ def _load_prompt(name: str) -> str:

_OPTION_LABELS = ["A", "B", "C", "D", "E", "F"]

# Rank order for tie-breaking the overall difficulty report — derived from
# the config tuple so a difficulty added there can't be silently dropped by
# _resolved_difficulty's counting.
_DIFFICULTY_RANK = {d: i for i, d in enumerate(CONCRETE_DIFFICULTIES)}


def _resolved_difficulty(wire_questions: list[dict]) -> str:
 """The overall difficulty generation actually produced (#540 A1).

 Mode of the per-question difficulties; ties break to the harder value
 so the report never understates what the student is about to face.
 Defaults to 'medium' when nothing usable is present (can't happen for
 agent output — QuizQuestion.difficulty is a concrete Literal — but
 this also runs on stored legacy rows).
 """
 counts: dict[str, int] = {}
 for q in wire_questions:
 d = q.get("difficulty")
 if d in _DIFFICULTY_RANK:
 counts[d] = counts.get(d, 0) + 1
 if not counts:
 return "medium"
 return max(counts, key=lambda d: (counts[d], _DIFFICULTY_RANK[d]))


def _agent_question_to_wire(q: QuizQuestion, qid: int) -> dict | None:
 """Map an agent QuizQuestion to the legacy wire-format dict, or
Expand DownExpand Up@@ -237,8 +269,24 @@ async def _quiz_via_agent(
 # Keep this message routing-only; the workflow + adaptive rules
 # live in the system prompt. We just hand the agent the inputs it
 # needs and trust the prompt to drive tool calls.
 if difficulty == "adaptive":
 # #540 A1: no target difficulty — the agent picks the whole mix
 # from mastery + recent accuracy (ADAPTIVE MODE in the system
 # prompt). Every emitted question still carries a concrete
 # easy|medium|hard; the route reports the overall pick back to
 # the client as `resolved_difficulty`.
 difficulty_clause = (
 f"Generate {num_questions} questions in ADAPTIVE MODE: you "
 f"choose each question's difficulty (easy, medium, or hard) "
 f"from the student's mastery and recent accuracy, per the "
 f"adaptive-mode rules in your system prompt."
 )
 else:
 difficulty_clause = (
 f"Generate {num_questions} {difficulty} questions for the student."
 )
 routing_msg = (
 f"Generate {num_questions} {difficulty} questions for the student. "
 f"{difficulty_clause} "
 f"The target concept is '{concept_name}' "
 f"(concept_node_id={concept_node_id}). Follow the workflow in your "
 f"system prompt; pass concept_node_id='{concept_node_id}' to "
Expand DownExpand Up@@ -290,23 +338,40 @@ async def _quiz_via_agent(
 )
 return wire_questions

@router.get("/config")
def quiz_config():
 """Selector options for the quiz UI (#540 A2). Single source of truth:
 the same constants bound the Pydantic request model, so a client that
 builds its selects from this payload can never send a value the route
 rejects. No user data, no auth needed."""
 return quiz_config_payload()


@router.post("/generate")
async def generate_quiz(body: GenerateQuizBody, request: Request):
 require_self(body.user_id, request)
 # quiz_attempts.difficulty is CHECK-constrained (0025); reject drift before
 # we run the agent or write an attempt row.
 if body.difficulty not in VALID_DIFFICULTIES:
 raise HTTPException(
 # The concrete trio is CHECK-constrained on quiz_attempts (0025 +
 # the #540 'adaptive' extension); reject drift before we run the
 # agent or write an attempt row.
 if body.difficulty not in REQUESTED_DIFFICULTIES:
 raise QuizAPIError(
 status_code=400,
 detail=f"Invalid difficulty '{body.difficulty}'. "
 f"Must be one of {sorted(VALID_DIFFICULTIES)}.",
 code=QuizErrorCode.QUIZ_DIFFICULTY_INVALID,
 message=(
 "That difficulty isn't available. Choose easy, medium, "
 "hard, or adaptive."
 ),
 )
 node_rows = table("graph_nodes").select(
 "*",
 filters={"id": f"eq.{body.concept_node_id}", "user_id": f"eq.{body.user_id}"},
 )
 if not node_rows:
 raise HTTPException(status_code=404, detail="Concept node not found")
 raise QuizAPIError(
 status_code=404,
 code=QuizErrorCode.QUIZ_CONCEPT_NOT_FOUND,
 message="We couldn't find that concept in your knowledge graph.",
 )
 node = node_rows[0]
 course_id = node.get("course_id") or None
 concept_name = node.get("concept_name") or ""
Expand DownExpand Up@@ -339,15 +404,17 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
 # The raw-Gemini legacy fallback was retired in #145; degrade to 502
 # rather than serving a quiz from a second LLM path.
 logger.warning("Quiz agent guardrails tripped; returning 502", exc_info=e)
 raise HTTPException(
 raise QuizAPIError(
 status_code=502,
 detail="Quiz generation is temporarily unavailable. Please try again.",
 code=QuizErrorCode.QUIZ_GENERATION_FAILED,
 message="Quiz generation is temporarily unavailable. Please try again.",
 ) from e
 except Exception as e:
 logger.exception("Unexpected quiz-agent failure; returning 502")
 raise HTTPException(
 raise QuizAPIError(
 status_code=502,
 detail="Quiz generation is temporarily unavailable. Please try again.",
 code=QuizErrorCode.QUIZ_GENERATION_FAILED,
 message="Quiz generation is temporarily unavailable. Please try again.",
 ) from e

 quiz_id = str(uuid.uuid4())
Expand All@@ -372,14 +439,28 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
 "difficulty": body.difficulty,
 },
 )
 return {"quiz_id": quiz_id, "questions": questions}
 # #540 A1: echo what generation actually chose. requested_difficulty
 # is what the student asked for (may be 'adaptive');
 # resolved_difficulty is the overall mix the agent produced (always
 # concrete) — so the client can say "we picked hard for you" instead
 # of repeating the request back.
 return {
 "quiz_id": quiz_id,
 "questions": questions,
 "requested_difficulty": body.difficulty,
 "resolved_difficulty": _resolved_difficulty(questions),
 }


@router.post("/submit")
def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request: Request):
 attempt_rows = table("quiz_attempts").select("*", filters={"id": f"eq.{body.quiz_id}"})
 if not attempt_rows:
 raise HTTPException(status_code=404, detail="Quiz not found")
 raise QuizAPIError(
 status_code=404,
 code=QuizErrorCode.QUIZ_ATTEMPT_NOT_FOUND,
 message="We couldn't find that quiz.",
 )
 attempt = attempt_rows[0]

 user_id = attempt["user_id"]
Expand All@@ -395,8 +476,10 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request
 # 200: quiz_attempts stores no mastery_before/after, so faithfully
 # reconstructing the first response would need a migration.
 if attempt.get("completed_at"):
 raise HTTPException(
 status_code=409, detail="Quiz attempt has already been submitted"
 raise QuizAPIError(
 status_code=409,
 code=QuizErrorCode.QUIZ_ATTEMPT_ALREADY_COMPLETED,
 message="This quiz has already been submitted.",
 )
 # The read above is only the fast path — two CONCURRENT submits (a
 # double-click on the final submit) would both pass it. The atomic claim
Expand All@@ -410,8 +493,10 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request
 filters={"id": f"eq.{body.quiz_id}", "completed_at": "is.null"},
 )
 if not claimed:
 raise HTTPException(
 status_code=409, detail="Quiz attempt has already been submitted"
 raise QuizAPIError(
 status_code=409,
 code=QuizErrorCode.QUIZ_ATTEMPT_ALREADY_COMPLETED,
 message="This quiz has already been submitted.",
 )

 concept_node_id = attempt["concept_node_id"]
Expand DownExpand Up@@ -450,7 +535,11 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request
 filters={"id": f"eq.{concept_node_id}", "user_id": f"eq.{user_id}"},
 )
 if not node_rows:
 raise HTTPException(status_code=404, detail="Concept node not found")
 raise QuizAPIError(
 status_code=404,
 code=QuizErrorCode.QUIZ_CONCEPT_NOT_FOUND,
 message="We couldn't find that concept in your knowledge graph.",
 )
 node = node_rows[0]
 mastery_before = node["mastery_score"]
 mastery_after = max(0.0, min(1.0, mastery_before + (score * 0.03) - ((total - score) * 0.02)))
Expand Down
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
feat(quiz): adaptive difficulty, /api/quiz/config, stable error envelope (#540) by AndresL230 · Pull Request #547 · SaplingLearn/Sapling · GitHub
Skip to content
feat(quiz): adaptive difficulty, /api/quiz/config, stable error envelope (#540)#547

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

AndresL230
merged 3 commits into

from

Aug 12, 2026
 Merged


feat(quiz): adaptive difficulty, /api/quiz/config, stable error envelope (#540)
#547
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
afc97df

feat(quiz): adaptive difficulty, /api/quiz/config, stable error envel…


AndresL230 Aug 12, 2026
1ea4c72

fix(quiz): address #547 review — envelope precision, adaptive history…


AndresL230 Aug 12, 2026
1bd3512

test(e2e): tolerate the CustomSelect ✓ marker in the #540 selector jo…


AndresL230 Aug 12, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 12 additions & 0 deletions
backend/agents/quiz.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -113,6 +113,18 @@ class Quiz(BaseModel):
 "- Don't drop high-mastery, recently-reviewed concepts entirely; "
 " include 1 question on a strong-and-fresh concept to keep the "
 " quiz from feeling punishing.\n\n"
 "ADAPTIVE MODE (#540): when the user message says the quiz is in "
 "adaptive mode, there is no user-requested difficulty — you choose "
 "each question's difficulty yourself. Base the mix on mastery and "
 "`recent_attempts.accuracy`: struggling/low-accuracy concepts get "
 "easy-leaning questions, strong/high-accuracy ones get hard-leaning "
 "questions; with no history at all, center the mix on medium. The "
 "±1-step limits below do not apply in adaptive mode, but every "
 "question still carries a concrete easy|medium|hard difficulty — "
 "'adaptive' is never a per-question value. Note that PAST attempts "
 "in `recent_attempts` may carry difficulty 'adaptive' (that "
 "attempt's mix was agent-chosen); treat such an attempt's "
 "difficulty as unspecified and judge it by its accuracy alone.\n\n"
 "Adaptive-difficulty rules (use `recent_attempts.accuracy`):\n"
 "- If the most recent 2-3 attempts on this concept averaged < "
 " 0.5 accuracy, drop the difficulty mix one step from what the "
Expand Down

2 changes: 1 addition & 1 deletion
backend/db/e2e_checks/quiz.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -32,7 +32,7 @@ def run() -> None:

 # ── 1. Generate a quiz to obtain a real quiz_id ───────────────────────────
 # GenerateQuizBody: {user_id, concept_node_id, num_questions, difficulty, use_shared_context}
 # difficulty must be in {"easy","medium","hard"} (CHECK constraint from 0025).
 # difficulty ∈ {"easy","medium","hard","adaptive"} (0025 CHECK + the #540 adaptive migration).
 gen_r = client.post(
 "/api/quiz/generate",
 json={
Expand Down

32 changes: 32 additions & 0 deletions
backend/db/migrations/20260812204809_quiz_attempts_adaptive_difficulty.sql

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
-- #540 A1: 'adaptive' becomes a real request-side difficulty. The attempt row
-- records what the student asked for (the response's resolved_difficulty
-- reports what generation actually produced), so the 0025 CHECK
-- (easy|medium|hard) must admit 'adaptive'.
--
-- DEPLOY ORDER: widening the CHECK is backward-compatible — apply this
-- migration BEFORE deploying the code that accepts 'adaptive'. In the gap the
-- other way around, an adaptive generate runs the full LLM call and then 500s
-- on the INSERT. (The promote runner migrates before merging; for staging run
-- `python -m db.migrate` before the deploy picks up the code.)
--
-- The DROP is by introspection, not by an assumed default name: environments
-- with out-of-band table history (staging has had some) can carry the same
-- CHECK under a different name, and a name-keyed `DROP IF EXISTS` would
-- silently no-op and leave the narrow constraint alive beside the new one.
DO $$
DECLARE c record;
BEGIN
 FOR c IN
 SELECT conname
 FROM pg_constraint
 WHERE conrelid = 'quiz_attempts'::regclass
 AND contype = 'c'
 AND pg_get_constraintdef(oid) ILIKE '%difficulty%'
 LOOP
 EXECUTE format('ALTER TABLE quiz_attempts DROP CONSTRAINT %I', c.conname);
 END LOOP;
END $$;

ALTER TABLE quiz_attempts
 ADD CONSTRAINT quiz_attempts_difficulty_check
 CHECK (difficulty IN ('easy', 'medium', 'hard', 'adaptive'));

34 changes: 31 additions & 3 deletions
backend/main.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@
from routes.admin import router as admin_router
from routes.admin_analytics import router as admin_analytics_router
from routes.newsletter import router as newsletter_router
from services import quiz_config, quiz_errors
from services.logfire_scrubber import EXTRA_PATTERNS, scrub_value
from services.request_context import RequestIDMiddleware, current_request_id
from services.storage_service import (
Expand DownExpand Up@@ -183,22 +184,46 @@ def _drop_request_arguments(_request, _attributes):
app.add_middleware(RequestIDMiddleware)


# #540 A3: on /api/quiz/* paths, quiz_errors.error_content wraps errors in
# the coded envelope (QuizAPIError raise sites carry precise codes; plain
# HTTPExceptions fall back to a status-derived one); everywhere else it
# returns the legacy {detail, request_id} shape unchanged.


@app.exception_handler(StarletteHTTPException)
async def http_exception_handler(request: Request, exc: StarletteHTTPException):
 rid = getattr(request.state, "request_id", None) or current_request_id()
 content = quiz_errors.error_content(
 request.url.path,
 exc.status_code,
 exc.detail,
 rid,
 code=getattr(exc, "code", None),
 machine_detail=getattr(exc, "machine_detail", None),
 )
 return JSONResponse(
 status_code=exc.status_code,
 content={"detail": exc.detail, "request_id": rid},
 content=content,
 headers={"X-Request-ID": rid} if rid else {},
 )


@app.exception_handler(RequestValidationError)
async def validation_exception_handler(request: Request, exc: RequestValidationError):
 rid = getattr(request.state, "request_id", None) or current_request_id()
 code, message = quiz_errors.validation_error_code(exc.errors())
 if code is quiz_errors.QuizErrorCode.QUIZ_COUNT_OUT_OF_RANGE:
 message = (
 f"Quizzes can have between {quiz_config.QUIZ_MIN_QUESTIONS} "
 f"and {quiz_config.QUIZ_MAX_QUESTIONS} questions."
 )
 content = quiz_errors.error_content(
 request.url.path, 422, exc.errors(), rid,
 code=code, message=message, machine_detail=exc.errors(),
 )
 return JSONResponse(
 status_code=422,
 content={"detail": exc.errors(), "request_id": rid},
 content=content,
 headers={"X-Request-ID": rid} if rid else {},
 )

Expand All@@ -207,9 +232,12 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
async def unhandled_exception_handler(request: Request, exc: Exception):
 logging.getLogger("main").exception("Unhandled exception")
 rid = getattr(request.state, "request_id", None) or current_request_id()
 content = quiz_errors.error_content(
 request.url.path, 500, "Internal server error.", rid,
 )
 return JSONResponse(
 status_code=500,
 content={"detail": "Internal server error.", "request_id": rid},
 content=content,
 headers={"X-Request-ID": rid} if rid else {},
 )

Expand Down

7 changes: 6 additions & 1 deletion
backend/models/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
from typing import Optional, Union, List, Literal
from pydantic import BaseModel, Field

from services.quiz_config import QUIZ_MIN_QUESTIONS, QUIZ_MAX_QUESTIONS


# ── Learn ─────────────────────────────────────────────────────────────────────

Expand DownExpand Up@@ -46,7 +48,10 @@ class ActionBody(BaseModel):
class GenerateQuizBody(BaseModel):
 user_id: str = "user_andres"
 concept_node_id: str
 num_questions: int = Field(default=5, ge=1, le=10)
 # Bounds come from services/quiz_config.py — the same constants
 # GET /api/quiz/config serves, so the client can't offer a value
 # this model rejects (#540 A2).
 num_questions: int = Field(default=5, ge=QUIZ_MIN_QUESTIONS, le=QUIZ_MAX_QUESTIONS)
 difficulty: str = "medium"
 use_shared_context: bool = True
 # Mirrors the Learn-route fast/smart toggle so quiz generation has
Expand Down

131 changes: 110 additions & 21 deletions
backend/routes/quiz.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,12 @@
from routes.learn import _get_catalog_chunk
from services import events_service
from services.auth_guard import require_self
from services.quiz_config import (
 CONCRETE_DIFFICULTIES,
 REQUESTED_DIFFICULTIES,
 quiz_config_payload,
)
from services.quiz_errors import QuizAPIError, QuizErrorCode
from services.profiles import get_display_name
from services.encryption import encrypt_json, decrypt_json_column
from services.graph_service import apply_graph_update
Expand All@@ -35,8 +41,10 @@

PROMPTS_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "prompts")

# quiz_attempts.difficulty CHECK enum (0025).
VALID_DIFFICULTIES = {"easy", "medium", "hard"}
# Request-side difficulties live in services/quiz_config.py (#540 A2):
# the concrete trio matches the quiz_attempts.difficulty CHECK (0025,
# extended with 'adaptive' by the #540 migration); 'adaptive' hands the
# per-question mix decision to the agent (A1).


def _load_prompt(name: str) -> str:
Expand All@@ -62,6 +70,30 @@ def _load_prompt(name: str) -> str:

_OPTION_LABELS = ["A", "B", "C", "D", "E", "F"]

# Rank order for tie-breaking the overall difficulty report — derived from
# the config tuple so a difficulty added there can't be silently dropped by
# _resolved_difficulty's counting.
_DIFFICULTY_RANK = {d: i for i, d in enumerate(CONCRETE_DIFFICULTIES)}


def _resolved_difficulty(wire_questions: list[dict]) -> str:
 """The overall difficulty generation actually produced (#540 A1).

 Mode of the per-question difficulties; ties break to the harder value
 so the report never understates what the student is about to face.
 Defaults to 'medium' when nothing usable is present (can't happen for
 agent output — QuizQuestion.difficulty is a concrete Literal — but
 this also runs on stored legacy rows).
 """
 counts: dict[str, int] = {}
 for q in wire_questions:
 d = q.get("difficulty")
 if d in _DIFFICULTY_RANK:
 counts[d] = counts.get(d, 0) + 1
 if not counts:
 return "medium"
 return max(counts, key=lambda d: (counts[d], _DIFFICULTY_RANK[d]))


def _agent_question_to_wire(q: QuizQuestion, qid: int) -> dict | None:
 """Map an agent QuizQuestion to the legacy wire-format dict, or
Expand DownExpand Up@@ -237,8 +269,24 @@ async def _quiz_via_agent(
 # Keep this message routing-only; the workflow + adaptive rules
 # live in the system prompt. We just hand the agent the inputs it
 # needs and trust the prompt to drive tool calls.
 if difficulty == "adaptive":
 # #540 A1: no target difficulty — the agent picks the whole mix
 # from mastery + recent accuracy (ADAPTIVE MODE in the system
 # prompt). Every emitted question still carries a concrete
 # easy|medium|hard; the route reports the overall pick back to
 # the client as `resolved_difficulty`.
 difficulty_clause = (
 f"Generate {num_questions} questions in ADAPTIVE MODE: you "
 f"choose each question's difficulty (easy, medium, or hard) "
 f"from the student's mastery and recent accuracy, per the "
 f"adaptive-mode rules in your system prompt."
 )
 else:
 difficulty_clause = (
 f"Generate {num_questions} {difficulty} questions for the student."
 )
 routing_msg = (
 f"Generate {num_questions} {difficulty} questions for the student. "
 f"{difficulty_clause} "
 f"The target concept is '{concept_name}' "
 f"(concept_node_id={concept_node_id}). Follow the workflow in your "
 f"system prompt; pass concept_node_id='{concept_node_id}' to "
Expand DownExpand Up@@ -290,23 +338,40 @@ async def _quiz_via_agent(
 )
 return wire_questions

@router.get("/config")
def quiz_config():
 """Selector options for the quiz UI (#540 A2). Single source of truth:
 the same constants bound the Pydantic request model, so a client that
 builds its selects from this payload can never send a value the route
 rejects. No user data, no auth needed."""
 return quiz_config_payload()


@router.post("/generate")
async def generate_quiz(body: GenerateQuizBody, request: Request):
 require_self(body.user_id, request)
 # quiz_attempts.difficulty is CHECK-constrained (0025); reject drift before
 # we run the agent or write an attempt row.
 if body.difficulty not in VALID_DIFFICULTIES:
 raise HTTPException(
 # The concrete trio is CHECK-constrained on quiz_attempts (0025 +
 # the #540 'adaptive' extension); reject drift before we run the
 # agent or write an attempt row.
 if body.difficulty not in REQUESTED_DIFFICULTIES:
 raise QuizAPIError(
 status_code=400,
 detail=f"Invalid difficulty '{body.difficulty}'. "
 f"Must be one of {sorted(VALID_DIFFICULTIES)}.",
 code=QuizErrorCode.QUIZ_DIFFICULTY_INVALID,
 message=(
 "That difficulty isn't available. Choose easy, medium, "
 "hard, or adaptive."
 ),
 )
 node_rows = table("graph_nodes").select(
 "*",
 filters={"id": f"eq.{body.concept_node_id}", "user_id": f"eq.{body.user_id}"},
 )
 if not node_rows:
 raise HTTPException(status_code=404, detail="Concept node not found")
 raise QuizAPIError(
 status_code=404,
 code=QuizErrorCode.QUIZ_CONCEPT_NOT_FOUND,
 message="We couldn't find that concept in your knowledge graph.",
 )
 node = node_rows[0]
 course_id = node.get("course_id") or None
 concept_name = node.get("concept_name") or ""
Expand DownExpand Up@@ -339,15 +404,17 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
 # The raw-Gemini legacy fallback was retired in #145; degrade to 502
 # rather than serving a quiz from a second LLM path.
 logger.warning("Quiz agent guardrails tripped; returning 502", exc_info=e)
 raise HTTPException(
 raise QuizAPIError(
 status_code=502,
 detail="Quiz generation is temporarily unavailable. Please try again.",
 code=QuizErrorCode.QUIZ_GENERATION_FAILED,
 message="Quiz generation is temporarily unavailable. Please try again.",
 ) from e
 except Exception as e:
 logger.exception("Unexpected quiz-agent failure; returning 502")
 raise HTTPException(
 raise QuizAPIError(
 status_code=502,
 detail="Quiz generation is temporarily unavailable. Please try again.",
 code=QuizErrorCode.QUIZ_GENERATION_FAILED,
 message="Quiz generation is temporarily unavailable. Please try again.",
 ) from e

 quiz_id = str(uuid.uuid4())
Expand All@@ -372,14 +439,28 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
 "difficulty": body.difficulty,
 },
 )
 return {"quiz_id": quiz_id, "questions": questions}
 # #540 A1: echo what generation actually chose. requested_difficulty
 # is what the student asked for (may be 'adaptive');
 # resolved_difficulty is the overall mix the agent produced (always
 # concrete) — so the client can say "we picked hard for you" instead
 # of repeating the request back.
 return {
 "quiz_id": quiz_id,
 "questions": questions,
 "requested_difficulty": body.difficulty,
 "resolved_difficulty": _resolved_difficulty(questions),
 }


@router.post("/submit")
def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request: Request):
 attempt_rows = table("quiz_attempts").select("*", filters={"id": f"eq.{body.quiz_id}"})
 if not attempt_rows:
 raise HTTPException(status_code=404, detail="Quiz not found")
 raise QuizAPIError(
 status_code=404,
 code=QuizErrorCode.QUIZ_ATTEMPT_NOT_FOUND,
 message="We couldn't find that quiz.",
 )
 attempt = attempt_rows[0]

 user_id = attempt["user_id"]
Expand All@@ -395,8 +476,10 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request
 # 200: quiz_attempts stores no mastery_before/after, so faithfully
 # reconstructing the first response would need a migration.
 if attempt.get("completed_at"):
 raise HTTPException(
 status_code=409, detail="Quiz attempt has already been submitted"
 raise QuizAPIError(
 status_code=409,
 code=QuizErrorCode.QUIZ_ATTEMPT_ALREADY_COMPLETED,
 message="This quiz has already been submitted.",
 )
 # The read above is only the fast path — two CONCURRENT submits (a
 # double-click on the final submit) would both pass it. The atomic claim
Expand All@@ -410,8 +493,10 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request
 filters={"id": f"eq.{body.quiz_id}", "completed_at": "is.null"},
 )
 if not claimed:
 raise HTTPException(
 status_code=409, detail="Quiz attempt has already been submitted"
 raise QuizAPIError(
 status_code=409,
 code=QuizErrorCode.QUIZ_ATTEMPT_ALREADY_COMPLETED,
 message="This quiz has already been submitted.",
 )

 concept_node_id = attempt["concept_node_id"]
Expand DownExpand Up@@ -450,7 +535,11 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request
 filters={"id": f"eq.{concept_node_id}", "user_id": f"eq.{user_id}"},
 )
 if not node_rows:
 raise HTTPException(status_code=404, detail="Concept node not found")
 raise QuizAPIError(
 status_code=404,
 code=QuizErrorCode.QUIZ_CONCEPT_NOT_FOUND,
 message="We couldn't find that concept in your knowledge graph.",
 )
 node = node_rows[0]
 mastery_before = node["mastery_score"]
 mastery_after = max(0.0, min(1.0, mastery_before + (score * 0.03) - ((total - score) * 0.02)))
Expand Down
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();
feat(quiz): adaptive difficulty, /api/quiz/config, stable error envelope (#540) by AndresL230 · Pull Request #547 · SaplingLearn/Sapling · GitHub
Skip to content
feat(quiz): adaptive difficulty, /api/quiz/config, stable error envelope (#540)#547

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

AndresL230
merged 3 commits into

from

Aug 12, 2026
 Merged


feat(quiz): adaptive difficulty, /api/quiz/config, stable error envelope (#540)
#547
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
afc97df

feat(quiz): adaptive difficulty, /api/quiz/config, stable error envel…


AndresL230 Aug 12, 2026
1ea4c72

fix(quiz): address #547 review — envelope precision, adaptive history…


AndresL230 Aug 12, 2026
1bd3512

test(e2e): tolerate the CustomSelect ✓ marker in the #540 selector jo…


AndresL230 Aug 12, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


12 changes: 12 additions & 0 deletions
backend/agents/quiz.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -113,6 +113,18 @@ class Quiz(BaseModel):
 "- Don't drop high-mastery, recently-reviewed concepts entirely; "
 " include 1 question on a strong-and-fresh concept to keep the "
 " quiz from feeling punishing.\n\n"
 "ADAPTIVE MODE (#540): when the user message says the quiz is in "
 "adaptive mode, there is no user-requested difficulty — you choose "
 "each question's difficulty yourself. Base the mix on mastery and "
 "`recent_attempts.accuracy`: struggling/low-accuracy concepts get "
 "easy-leaning questions, strong/high-accuracy ones get hard-leaning "
 "questions; with no history at all, center the mix on medium. The "
 "±1-step limits below do not apply in adaptive mode, but every "
 "question still carries a concrete easy|medium|hard difficulty — "
 "'adaptive' is never a per-question value. Note that PAST attempts "
 "in `recent_attempts` may carry difficulty 'adaptive' (that "
 "attempt's mix was agent-chosen); treat such an attempt's "
 "difficulty as unspecified and judge it by its accuracy alone.\n\n"
 "Adaptive-difficulty rules (use `recent_attempts.accuracy`):\n"
 "- If the most recent 2-3 attempts on this concept averaged < "
 " 0.5 accuracy, drop the difficulty mix one step from what the "
Expand Down

2 changes: 1 addition & 1 deletion
backend/db/e2e_checks/quiz.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -32,7 +32,7 @@ def run() -> None:

 # ── 1. Generate a quiz to obtain a real quiz_id ───────────────────────────
 # GenerateQuizBody: {user_id, concept_node_id, num_questions, difficulty, use_shared_context}
 # difficulty must be in {"easy","medium","hard"} (CHECK constraint from 0025).
 # difficulty ∈ {"easy","medium","hard","adaptive"} (0025 CHECK + the #540 adaptive migration).
 gen_r = client.post(
 "/api/quiz/generate",
 json={
Expand Down

32 changes: 32 additions & 0 deletions
backend/db/migrations/20260812204809_quiz_attempts_adaptive_difficulty.sql

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
-- #540 A1: 'adaptive' becomes a real request-side difficulty. The attempt row
-- records what the student asked for (the response's resolved_difficulty
-- reports what generation actually produced), so the 0025 CHECK
-- (easy|medium|hard) must admit 'adaptive'.
--
-- DEPLOY ORDER: widening the CHECK is backward-compatible — apply this
-- migration BEFORE deploying the code that accepts 'adaptive'. In the gap the
-- other way around, an adaptive generate runs the full LLM call and then 500s
-- on the INSERT. (The promote runner migrates before merging; for staging run
-- `python -m db.migrate` before the deploy picks up the code.)
--
-- The DROP is by introspection, not by an assumed default name: environments
-- with out-of-band table history (staging has had some) can carry the same
-- CHECK under a different name, and a name-keyed `DROP IF EXISTS` would
-- silently no-op and leave the narrow constraint alive beside the new one.
DO $$
DECLARE c record;
BEGIN
 FOR c IN
 SELECT conname
 FROM pg_constraint
 WHERE conrelid = 'quiz_attempts'::regclass
 AND contype = 'c'
 AND pg_get_constraintdef(oid) ILIKE '%difficulty%'
 LOOP
 EXECUTE format('ALTER TABLE quiz_attempts DROP CONSTRAINT %I', c.conname);
 END LOOP;
END $$;

ALTER TABLE quiz_attempts
 ADD CONSTRAINT quiz_attempts_difficulty_check
 CHECK (difficulty IN ('easy', 'medium', 'hard', 'adaptive'));

34 changes: 31 additions & 3 deletions
backend/main.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@
from routes.admin import router as admin_router
from routes.admin_analytics import router as admin_analytics_router
from routes.newsletter import router as newsletter_router
from services import quiz_config, quiz_errors
from services.logfire_scrubber import EXTRA_PATTERNS, scrub_value
from services.request_context import RequestIDMiddleware, current_request_id
from services.storage_service import (
Expand DownExpand Up@@ -183,22 +184,46 @@ def _drop_request_arguments(_request, _attributes):
app.add_middleware(RequestIDMiddleware)


# #540 A3: on /api/quiz/* paths, quiz_errors.error_content wraps errors in
# the coded envelope (QuizAPIError raise sites carry precise codes; plain
# HTTPExceptions fall back to a status-derived one); everywhere else it
# returns the legacy {detail, request_id} shape unchanged.


@app.exception_handler(StarletteHTTPException)
async def http_exception_handler(request: Request, exc: StarletteHTTPException):
 rid = getattr(request.state, "request_id", None) or current_request_id()
 content = quiz_errors.error_content(
 request.url.path,
 exc.status_code,
 exc.detail,
 rid,
 code=getattr(exc, "code", None),
 machine_detail=getattr(exc, "machine_detail", None),
 )
 return JSONResponse(
 status_code=exc.status_code,
 content={"detail": exc.detail, "request_id": rid},
 content=content,
 headers={"X-Request-ID": rid} if rid else {},
 )


@app.exception_handler(RequestValidationError)
async def validation_exception_handler(request: Request, exc: RequestValidationError):
 rid = getattr(request.state, "request_id", None) or current_request_id()
 code, message = quiz_errors.validation_error_code(exc.errors())
 if code is quiz_errors.QuizErrorCode.QUIZ_COUNT_OUT_OF_RANGE:
 message = (
 f"Quizzes can have between {quiz_config.QUIZ_MIN_QUESTIONS} "
 f"and {quiz_config.QUIZ_MAX_QUESTIONS} questions."
 )
 content = quiz_errors.error_content(
 request.url.path, 422, exc.errors(), rid,
 code=code, message=message, machine_detail=exc.errors(),
 )
 return JSONResponse(
 status_code=422,
 content={"detail": exc.errors(), "request_id": rid},
 content=content,
 headers={"X-Request-ID": rid} if rid else {},
 )

Expand All@@ -207,9 +232,12 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
async def unhandled_exception_handler(request: Request, exc: Exception):
 logging.getLogger("main").exception("Unhandled exception")
 rid = getattr(request.state, "request_id", None) or current_request_id()
 content = quiz_errors.error_content(
 request.url.path, 500, "Internal server error.", rid,
 )
 return JSONResponse(
 status_code=500,
 content={"detail": "Internal server error.", "request_id": rid},
 content=content,
 headers={"X-Request-ID": rid} if rid else {},
 )

Expand Down

7 changes: 6 additions & 1 deletion
backend/models/__init__.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
from typing import Optional, Union, List, Literal
from pydantic import BaseModel, Field

from services.quiz_config import QUIZ_MIN_QUESTIONS, QUIZ_MAX_QUESTIONS


# ── Learn ─────────────────────────────────────────────────────────────────────

Expand DownExpand Up@@ -46,7 +48,10 @@ class ActionBody(BaseModel):
class GenerateQuizBody(BaseModel):
 user_id: str = "user_andres"
 concept_node_id: str
 num_questions: int = Field(default=5, ge=1, le=10)
 # Bounds come from services/quiz_config.py — the same constants
 # GET /api/quiz/config serves, so the client can't offer a value
 # this model rejects (#540 A2).
 num_questions: int = Field(default=5, ge=QUIZ_MIN_QUESTIONS, le=QUIZ_MAX_QUESTIONS)
 difficulty: str = "medium"
 use_shared_context: bool = True
 # Mirrors the Learn-route fast/smart toggle so quiz generation has
Expand Down

131 changes: 110 additions & 21 deletions
backend/routes/quiz.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,6 +20,12 @@
from routes.learn import _get_catalog_chunk
from services import events_service
from services.auth_guard import require_self
from services.quiz_config import (
 CONCRETE_DIFFICULTIES,
 REQUESTED_DIFFICULTIES,
 quiz_config_payload,
)
from services.quiz_errors import QuizAPIError, QuizErrorCode
from services.profiles import get_display_name
from services.encryption import encrypt_json, decrypt_json_column
from services.graph_service import apply_graph_update
Expand All@@ -35,8 +41,10 @@

PROMPTS_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "prompts")

# quiz_attempts.difficulty CHECK enum (0025).
VALID_DIFFICULTIES = {"easy", "medium", "hard"}
# Request-side difficulties live in services/quiz_config.py (#540 A2):
# the concrete trio matches the quiz_attempts.difficulty CHECK (0025,
# extended with 'adaptive' by the #540 migration); 'adaptive' hands the
# per-question mix decision to the agent (A1).


def _load_prompt(name: str) -> str:
Expand All@@ -62,6 +70,30 @@ def _load_prompt(name: str) -> str:

_OPTION_LABELS = ["A", "B", "C", "D", "E", "F"]

# Rank order for tie-breaking the overall difficulty report — derived from
# the config tuple so a difficulty added there can't be silently dropped by
# _resolved_difficulty's counting.
_DIFFICULTY_RANK = {d: i for i, d in enumerate(CONCRETE_DIFFICULTIES)}


def _resolved_difficulty(wire_questions: list[dict]) -> str:
 """The overall difficulty generation actually produced (#540 A1).

 Mode of the per-question difficulties; ties break to the harder value
 so the report never understates what the student is about to face.
 Defaults to 'medium' when nothing usable is present (can't happen for
 agent output — QuizQuestion.difficulty is a concrete Literal — but
 this also runs on stored legacy rows).
 """
 counts: dict[str, int] = {}
 for q in wire_questions:
 d = q.get("difficulty")
 if d in _DIFFICULTY_RANK:
 counts[d] = counts.get(d, 0) + 1
 if not counts:
 return "medium"
 return max(counts, key=lambda d: (counts[d], _DIFFICULTY_RANK[d]))


def _agent_question_to_wire(q: QuizQuestion, qid: int) -> dict | None:
 """Map an agent QuizQuestion to the legacy wire-format dict, or
Expand DownExpand Up@@ -237,8 +269,24 @@ async def _quiz_via_agent(
 # Keep this message routing-only; the workflow + adaptive rules
 # live in the system prompt. We just hand the agent the inputs it
 # needs and trust the prompt to drive tool calls.
 if difficulty == "adaptive":
 # #540 A1: no target difficulty — the agent picks the whole mix
 # from mastery + recent accuracy (ADAPTIVE MODE in the system
 # prompt). Every emitted question still carries a concrete
 # easy|medium|hard; the route reports the overall pick back to
 # the client as `resolved_difficulty`.
 difficulty_clause = (
 f"Generate {num_questions} questions in ADAPTIVE MODE: you "
 f"choose each question's difficulty (easy, medium, or hard) "
 f"from the student's mastery and recent accuracy, per the "
 f"adaptive-mode rules in your system prompt."
 )
 else:
 difficulty_clause = (
 f"Generate {num_questions} {difficulty} questions for the student."
 )
 routing_msg = (
 f"Generate {num_questions} {difficulty} questions for the student. "
 f"{difficulty_clause} "
 f"The target concept is '{concept_name}' "
 f"(concept_node_id={concept_node_id}). Follow the workflow in your "
 f"system prompt; pass concept_node_id='{concept_node_id}' to "
Expand DownExpand Up@@ -290,23 +338,40 @@ async def _quiz_via_agent(
 )
 return wire_questions

@router.get("/config")
def quiz_config():
 """Selector options for the quiz UI (#540 A2). Single source of truth:
 the same constants bound the Pydantic request model, so a client that
 builds its selects from this payload can never send a value the route
 rejects. No user data, no auth needed."""
 return quiz_config_payload()


@router.post("/generate")
async def generate_quiz(body: GenerateQuizBody, request: Request):
 require_self(body.user_id, request)
 # quiz_attempts.difficulty is CHECK-constrained (0025); reject drift before
 # we run the agent or write an attempt row.
 if body.difficulty not in VALID_DIFFICULTIES:
 raise HTTPException(
 # The concrete trio is CHECK-constrained on quiz_attempts (0025 +
 # the #540 'adaptive' extension); reject drift before we run the
 # agent or write an attempt row.
 if body.difficulty not in REQUESTED_DIFFICULTIES:
 raise QuizAPIError(
 status_code=400,
 detail=f"Invalid difficulty '{body.difficulty}'. "
 f"Must be one of {sorted(VALID_DIFFICULTIES)}.",
 code=QuizErrorCode.QUIZ_DIFFICULTY_INVALID,
 message=(
 "That difficulty isn't available. Choose easy, medium, "
 "hard, or adaptive."
 ),
 )
 node_rows = table("graph_nodes").select(
 "*",
 filters={"id": f"eq.{body.concept_node_id}", "user_id": f"eq.{body.user_id}"},
 )
 if not node_rows:
 raise HTTPException(status_code=404, detail="Concept node not found")
 raise QuizAPIError(
 status_code=404,
 code=QuizErrorCode.QUIZ_CONCEPT_NOT_FOUND,
 message="We couldn't find that concept in your knowledge graph.",
 )
 node = node_rows[0]
 course_id = node.get("course_id") or None
 concept_name = node.get("concept_name") or ""
Expand DownExpand Up@@ -339,15 +404,17 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
 # The raw-Gemini legacy fallback was retired in #145; degrade to 502
 # rather than serving a quiz from a second LLM path.
 logger.warning("Quiz agent guardrails tripped; returning 502", exc_info=e)
 raise HTTPException(
 raise QuizAPIError(
 status_code=502,
 detail="Quiz generation is temporarily unavailable. Please try again.",
 code=QuizErrorCode.QUIZ_GENERATION_FAILED,
 message="Quiz generation is temporarily unavailable. Please try again.",
 ) from e
 except Exception as e:
 logger.exception("Unexpected quiz-agent failure; returning 502")
 raise HTTPException(
 raise QuizAPIError(
 status_code=502,
 detail="Quiz generation is temporarily unavailable. Please try again.",
 code=QuizErrorCode.QUIZ_GENERATION_FAILED,
 message="Quiz generation is temporarily unavailable. Please try again.",
 ) from e

 quiz_id = str(uuid.uuid4())
Expand All@@ -372,14 +439,28 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
 "difficulty": body.difficulty,
 },
 )
 return {"quiz_id": quiz_id, "questions": questions}
 # #540 A1: echo what generation actually chose. requested_difficulty
 # is what the student asked for (may be 'adaptive');
 # resolved_difficulty is the overall mix the agent produced (always
 # concrete) — so the client can say "we picked hard for you" instead
 # of repeating the request back.
 return {
 "quiz_id": quiz_id,
 "questions": questions,
 "requested_difficulty": body.difficulty,
 "resolved_difficulty": _resolved_difficulty(questions),
 }


@router.post("/submit")
def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request: Request):
 attempt_rows = table("quiz_attempts").select("*", filters={"id": f"eq.{body.quiz_id}"})
 if not attempt_rows:
 raise HTTPException(status_code=404, detail="Quiz not found")
 raise QuizAPIError(
 status_code=404,
 code=QuizErrorCode.QUIZ_ATTEMPT_NOT_FOUND,
 message="We couldn't find that quiz.",
 )
 attempt = attempt_rows[0]

 user_id = attempt["user_id"]
Expand All@@ -395,8 +476,10 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request
 # 200: quiz_attempts stores no mastery_before/after, so faithfully
 # reconstructing the first response would need a migration.
 if attempt.get("completed_at"):
 raise HTTPException(
 status_code=409, detail="Quiz attempt has already been submitted"
 raise QuizAPIError(
 status_code=409,
 code=QuizErrorCode.QUIZ_ATTEMPT_ALREADY_COMPLETED,
 message="This quiz has already been submitted.",
 )
 # The read above is only the fast path — two CONCURRENT submits (a
 # double-click on the final submit) would both pass it. The atomic claim
Expand All@@ -410,8 +493,10 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request
 filters={"id": f"eq.{body.quiz_id}", "completed_at": "is.null"},
 )
 if not claimed:
 raise HTTPException(
 status_code=409, detail="Quiz attempt has already been submitted"
 raise QuizAPIError(
 status_code=409,
 code=QuizErrorCode.QUIZ_ATTEMPT_ALREADY_COMPLETED,
 message="This quiz has already been submitted.",
 )

 concept_node_id = attempt["concept_node_id"]
Expand DownExpand Up@@ -450,7 +535,11 @@ def submit_quiz(body: SubmitQuizBody, background_tasks: BackgroundTasks, request
 filters={"id": f"eq.{concept_node_id}", "user_id": f"eq.{user_id}"},
 )
 if not node_rows:
 raise HTTPException(status_code=404, detail="Concept node not found")
 raise QuizAPIError(
 status_code=404,
 code=QuizErrorCode.QUIZ_CONCEPT_NOT_FOUND,
 message="We couldn't find that concept in your knowledge graph.",
 )
 node = node_rows[0]
 mastery_before = node["mastery_score"]
 mastery_after = max(0.0, min(1.0, mastery_before + (score * 0.03) - ((total - score) * 0.02)))
Expand Down
Loading
Loading