Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
-- H3 (#555): how many days away the student's next exam was when this quiz
-- was generated.
--
-- Stored on the attempt, not merely used in the prompt, because the point of
-- the issue is to be able to ASK LATER whether deadline-aware quizzes perform
-- differently. A value that only ever reached a prompt leaves no way to
-- compare a quiz taken three days before a midterm against one taken in week
-- two — the measurement is the deliverable, not the prompt line.
--
-- Nullable with no default, and the distinction is load-bearing:
-- NULL = we did not know (no course, no enrollment, no dated exams, or the
-- lookup failed — `days_until_next_exam` degrades to None rather
-- than failing a generation)
-- 0 = the exam is TODAY, which is the most actionable value the feature
-- produces. A DEFAULT 0 would make every legacy row and every
-- unknown look like exam day.
--
-- Dates only. This column is a day count derived from `assignments.due_date`,
-- which is plaintext and indexed; no grade VALUE is read, stored or prompted
-- anywhere on this path (points columns are encrypted per #521, and the audit
-- flags that the current ToS does not clearly cover feeding grades to a
-- model).
--
-- Additive and idempotent. Ordering still matters: PostgREST 400s on a column
-- its schema cache doesn't have, and `generate_quiz` includes this key in the
-- attempt INSERT — so this must be applied strictly before the code ships, as
-- with 20260814051517.
ALTER TABLE quiz_attempts
ADD COLUMN IF NOT EXISTS exam_days_away INTEGER;
137 changes: 113 additions & 24 deletions backend/routes/quiz.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -42,6 +42,7 @@
from services.encryption import encrypt_json, decrypt_json_column
from services.graph_service import apply_graph_update
from services.quiz_context_service import get_quiz_context, save_quiz_context
from services.exam_proximity import PROMPT_HORIZON_DAYS, days_until_next_exam
from services.quiz_distractors import build_distractor_profile
from services.fingerprint import fingerprint
from services.quiz_identity import question_hash, normalize_text
Expand DownExpand Up@@ -795,6 +796,50 @@ def _do_not_repeat_block(recent: list[RecentQuestion]) -> str:
)


def _insert_attempt(attempt_row: dict) -> None:
"""Write the attempt, surviving a schema that predates `exam_days_away`.

The omit-when-None rule alone does NOT make a pre-migration environment
safe, and the failure is nastier than it looks: it strikes exactly the
students the feature is FOR (the ones with a dated upcoming exam), so it
presents as a random partial outage rather than an obvious missing
migration. And it strikes late — the agent has already run and been
billed — so an unhandled 400 here loses the generated quiz, writes no
attempt row, emits no `quiz.generation_failed`, and never refunds the
rate-limit slot.
#
Ordering is still the rule (migration before code, as with
20260814051517). This is the seatbelt for the window where PostgREST has
not yet reloaded its schema cache, not a licence to deploy first.
"""
try:
table("quiz_attempts").insert(attempt_row)
return
except Exception:
if "exam_days_away" not in attempt_row:
raise
retry = {k: v for k, v in attempt_row.items() if k != "exam_days_away"}
logger.warning(
"quiz: attempt insert failed with exam_days_away present; retrying "
"without it (is 20260822090747 applied?) quiz_id=%s",
attempt_row.get("id"),
)
table("quiz_attempts").insert(retry)


class GeneratedQuiz(NamedTuple):
"""What one generation produced.

`exam_days_away` rides back with the questions rather than being resolved
again by the caller: it is used for the prompt here and stored on the
attempt there, and two lookups could disagree if an exam were entered
between them.
"""

questions: list[dict]
exam_days_away: int | None = None


async def _quiz_via_agent(
*,
user_id: str,
Expand All@@ -806,7 +851,7 @@ async def _quiz_via_agent(
use_shared_context: bool,
request_id: str,
model_pref: str | None = None,
) -> list[dict]:
) -> GeneratedQuiz:
"""Run quiz_agent and return questions in the legacy wire shape.

The agent's tools (read_concepts_for_user, read_misconceptions_for_course)
Expand DownExpand Up@@ -849,42 +894,39 @@ async def _quiz_via_agent(
difficulty_clause = (
f"Generate {num_questions} {difficulty} questions for the student."
)
routing_msg = (
f"{difficulty_clause} "
f"The target concept is '{concept_name}' "
f"(concept_node_id={concept_node_id}). Follow the workflow in your "
f"system prompt; pass concept_node_id='{concept_node_id}' to "
f"read_recent_quiz_attempts."
)
if use_shared_context:
routing_msg += (
" Also call read_misconceptions_for_course and use those misconceptions "
"as distractors and probes."
)

# Course-material grounding does blocking network I/O (a Gemini
# embedding call, bounded at 60s) plus sync Supabase reads. Run it in a
# worker thread so a slow/stalled retrieval can't freeze this worker's
# event loop for every other in-flight request. Matches the
# asyncio.to_thread pattern used by the agent read tools.
#
# E6's recently-asked read is an independent Supabase read + decrypt, so
# it runs CONCURRENTLY with grounding rather than after it — the two have
# nothing to say to each other and serializing them would add the slower
# one's latency to every generation.
# E6's recently-asked read and H3's exam-proximity lookup are independent
# Supabase reads, so all three run CONCURRENTLY rather than in sequence —
# they have nothing to say to each other, and serializing them would add
# every one of their latencies to every generation. Proximity costs
# several round-trips on its own, so running it before this block made it
# fully additive.
#
# return_exceptions=True because BOTH are best-effort context, and a bare
# gather propagates the first failure straight out of generation: an
# return_exceptions=True because ALL THREE are best-effort context, and a
# bare gather propagates the first failure straight out of generation: an
# unreadable past attempt would 502 a quiz that needed no history at all.
# Each helper already degrades internally; this is the backstop for the
# failure they cannot catch (an unexpected raise on the way in or out).
material, recent = await asyncio.gather(
material, recent, exam_days_away = await asyncio.gather(
asyncio.to_thread(_course_material, course_id, concept_name),
asyncio.to_thread(
recent_question_identities, user_id, concept_node_id
),
asyncio.to_thread(days_until_next_exam, user_id, course_id),
return_exceptions=True,
)
if isinstance(exam_days_away, BaseException):
logger.warning(
"quiz: exam-proximity lookup failed (%s); generating without it",
type(exam_days_away).__name__, exc_info=exam_days_away,
)
exam_days_away = None
if isinstance(material, BaseException):
logger.warning(
"quiz: course-material assembly failed (%s); generating ungrounded",
Expand All@@ -900,6 +942,40 @@ async def _quiz_via_agent(
"do-not-repeat list", type(recent).__name__, exc_info=recent,
)
recent = []

routing_msg = (
f"{difficulty_clause} "
f"The target concept is '{concept_name}' "
f"(concept_node_id={concept_node_id}). Follow the workflow in your "
f"system prompt; pass concept_node_id='{concept_node_id}' to "
f"read_recent_quiz_attempts."
)
if use_shared_context:
routing_msg += (
" Also call read_misconceptions_for_course and use those misconceptions "
"as distractors and probes."
)
# H3/#555: one line, dates only. Says how near the deadline is and lets
# the model decide what that implies — not "make it harder", which would
# contradict the adaptive difficulty the student actually chose. Omitted
# entirely when unknown: "next exam: unknown" is prompt tokens spent to
# say nothing.
# Bounded by a horizon: a final dated 87 days out would otherwise put
# "there is an exam coming, weight toward what an exam tests" on EVERY
# quiz for the whole semester, which carries no proximity signal and
# steers week-two practice toward exam-style questions — the opposite of
# what this line is for. The stored `exam_days_away` is NOT clamped: the
# analytics want the real distance, including the far ones.
if exam_days_away is not None and exam_days_away <= PROMPT_HORIZON_DAYS:
when = (
"TODAY" if exam_days_away == 0
else "tomorrow" if exam_days_away == 1
else f"in {exam_days_away} days"
)
routing_msg += (
f" The student's next exam in this course is {when}. Weight the"
" questions toward what an exam would actually test."
)
_log_rag_uncovered(
material,
user_id=user_id,
Expand DownExpand Up@@ -1078,7 +1154,10 @@ def _absorb(quiz: Quiz, model: str) -> None:
"quiz_agent produced no valid questions after wire-format validation"
)
# Never serve more than asked for (a generous top-up run can overshoot).
return wire_questions[:num_questions]
return GeneratedQuiz(
questions=wire_questions[:num_questions],
exam_days_away=exam_days_away,
)

@router.get("/config")
def quiz_config():
Expand DownExpand Up@@ -1169,7 +1248,7 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
# QUIZ_GENERATION_TIMEOUT_SEC (see _run) — cancelling the whole
# coroutine here would discard a partial quiz the top-up handler
# is designed to serve.
questions = await _quiz_via_agent(
generated = await _quiz_via_agent(
user_id=body.user_id,
course_id=course_id,
concept_node_id=body.concept_node_id,
Expand All@@ -1180,6 +1259,8 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
request_id=request_id,
model_pref=body.model_pref,
)
questions = generated.questions
exam_days_away = generated.exam_days_away
except HTTPException:
# The 404 for an unknown concept node is raised before the agent call;
# never swallow a known HTTP state.
Expand DownExpand Up@@ -1224,13 +1305,21 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
) from e

quiz_id = str(uuid.uuid4())
table("quiz_attempts").insert({
attempt_row = {
"id": quiz_id,
"user_id": body.user_id,
"concept_node_id": body.concept_node_id,
"difficulty": body.difficulty,
"questions_json": encrypt_json(questions),
})
}
# H3/#555: recorded so the question "do deadline-aware quizzes perform
# differently?" is answerable later. Omitted when unknown rather than
# written as an explicit null, mirroring apply_graph_update's rule: an
# environment that took this code before the migration keeps generating
# instead of 400ing on a column PostgREST's schema cache doesn't have.
if exam_days_away is not None:
attempt_row["exam_days_away"] = exam_days_away
_insert_attempt(attempt_row)
# #117: quiz.started once the attempt row exists. num_questions is the
# actual generated count (the agent may return fewer than requested).
events_service.log_event(
Expand Down
15 changes: 6 additions & 9 deletions backend/routes/study_guide.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@
)
from services.graph_service import get_courses as graph_get_courses
from services.auth_guard import require_self
from services.exam_proximity import is_exam
from services.encryption import (
decrypt_if_present,
decrypt_json,
Expand DownExpand Up@@ -303,15 +304,11 @@ def get_exams(
order="due_date.asc",
) or []

exam_keywords = ["exam", "midterm", "final", "quiz"]
exams = []
for a in all_assignments:
atype = (a.get("assignment_type") or "").lower()
title = (a.get("title") or "").lower()
if atype == "exam" or any(kw in title for kw in exam_keywords):
exams.append(a)

return {"exams": exams}
# The heuristic moved to services/exam_proximity.py (#555), which needed
# the same question answered on the quiz path. One definition, two
# callers — a second copy would drift, which is the failure #557 spent a
# workstream undoing.
return {"exams": [a for a in all_assignments if is_exam(a)]}


@router.get("/{user_id}/guide")
Expand Down
6 changes: 5 additions & 1 deletion backend/scripts/benchmark_quiz.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -147,7 +147,10 @@ async def _generate_quiz_for_async(concept_name: str) -> list[dict]:
so this exercises the real production grounding path rather than
generating an ungrounded quiz.
"""
return await _quiz_via_agent(
# `.questions`: _quiz_via_agent returns a GeneratedQuiz (#555) so the
# exam-proximity value it resolved can reach the attempt row. This bench
# only wants the questions.
generated = await _quiz_via_agent(
user_id="quizfix-user-0001",
course_id=FIXTURE_COURSE_ID,
concept_node_id="quizfix-node-0001",
Expand All@@ -157,6 +160,7 @@ async def _generate_quiz_for_async(concept_name: str) -> list[dict]:
use_shared_context=False,
request_id="quizfix-bench",
)
return generated.questions


def generate_quiz_for(concept_name: str) -> list[dict]:
Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
feat(quiz): weight generation toward an approaching exam (#555) by AndresL230 · Pull Request #573 · SaplingLearn/Sapling · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
-- H3 (#555): how many days away the student's next exam was when this quiz
-- was generated.
--
-- Stored on the attempt, not merely used in the prompt, because the point of
-- the issue is to be able to ASK LATER whether deadline-aware quizzes perform
-- differently. A value that only ever reached a prompt leaves no way to
-- compare a quiz taken three days before a midterm against one taken in week
-- two — the measurement is the deliverable, not the prompt line.
--
-- Nullable with no default, and the distinction is load-bearing:
-- NULL = we did not know (no course, no enrollment, no dated exams, or the
-- lookup failed — `days_until_next_exam` degrades to None rather
-- than failing a generation)
-- 0 = the exam is TODAY, which is the most actionable value the feature
-- produces. A DEFAULT 0 would make every legacy row and every
-- unknown look like exam day.
--
-- Dates only. This column is a day count derived from `assignments.due_date`,
-- which is plaintext and indexed; no grade VALUE is read, stored or prompted
-- anywhere on this path (points columns are encrypted per #521, and the audit
-- flags that the current ToS does not clearly cover feeding grades to a
-- model).
--
-- Additive and idempotent. Ordering still matters: PostgREST 400s on a column
-- its schema cache doesn't have, and `generate_quiz` includes this key in the
-- attempt INSERT — so this must be applied strictly before the code ships, as
-- with 20260814051517.
ALTER TABLE quiz_attempts
ADD COLUMN IF NOT EXISTS exam_days_away INTEGER;
137 changes: 113 additions & 24 deletions backend/routes/quiz.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -42,6 +42,7 @@
from services.encryption import encrypt_json, decrypt_json_column
from services.graph_service import apply_graph_update
from services.quiz_context_service import get_quiz_context, save_quiz_context
from services.exam_proximity import PROMPT_HORIZON_DAYS, days_until_next_exam
from services.quiz_distractors import build_distractor_profile
from services.fingerprint import fingerprint
from services.quiz_identity import question_hash, normalize_text
Expand DownExpand Up@@ -795,6 +796,50 @@ def _do_not_repeat_block(recent: list[RecentQuestion]) -> str:
)


def _insert_attempt(attempt_row: dict) -> None:
"""Write the attempt, surviving a schema that predates `exam_days_away`.

The omit-when-None rule alone does NOT make a pre-migration environment
safe, and the failure is nastier than it looks: it strikes exactly the
students the feature is FOR (the ones with a dated upcoming exam), so it
presents as a random partial outage rather than an obvious missing
migration. And it strikes late — the agent has already run and been
billed — so an unhandled 400 here loses the generated quiz, writes no
attempt row, emits no `quiz.generation_failed`, and never refunds the
rate-limit slot.
#
Ordering is still the rule (migration before code, as with
20260814051517). This is the seatbelt for the window where PostgREST has
not yet reloaded its schema cache, not a licence to deploy first.
"""
try:
table("quiz_attempts").insert(attempt_row)
return
except Exception:
if "exam_days_away" not in attempt_row:
raise
retry = {k: v for k, v in attempt_row.items() if k != "exam_days_away"}
logger.warning(
"quiz: attempt insert failed with exam_days_away present; retrying "
"without it (is 20260822090747 applied?) quiz_id=%s",
attempt_row.get("id"),
)
table("quiz_attempts").insert(retry)


class GeneratedQuiz(NamedTuple):
"""What one generation produced.

`exam_days_away` rides back with the questions rather than being resolved
again by the caller: it is used for the prompt here and stored on the
attempt there, and two lookups could disagree if an exam were entered
between them.
"""

questions: list[dict]
exam_days_away: int | None = None


async def _quiz_via_agent(
*,
user_id: str,
Expand All@@ -806,7 +851,7 @@ async def _quiz_via_agent(
use_shared_context: bool,
request_id: str,
model_pref: str | None = None,
) -> list[dict]:
) -> GeneratedQuiz:
"""Run quiz_agent and return questions in the legacy wire shape.

The agent's tools (read_concepts_for_user, read_misconceptions_for_course)
Expand DownExpand Up@@ -849,42 +894,39 @@ async def _quiz_via_agent(
difficulty_clause = (
f"Generate {num_questions} {difficulty} questions for the student."
)
routing_msg = (
f"{difficulty_clause} "
f"The target concept is '{concept_name}' "
f"(concept_node_id={concept_node_id}). Follow the workflow in your "
f"system prompt; pass concept_node_id='{concept_node_id}' to "
f"read_recent_quiz_attempts."
)
if use_shared_context:
routing_msg += (
" Also call read_misconceptions_for_course and use those misconceptions "
"as distractors and probes."
)

# Course-material grounding does blocking network I/O (a Gemini
# embedding call, bounded at 60s) plus sync Supabase reads. Run it in a
# worker thread so a slow/stalled retrieval can't freeze this worker's
# event loop for every other in-flight request. Matches the
# asyncio.to_thread pattern used by the agent read tools.
#
# E6's recently-asked read is an independent Supabase read + decrypt, so
# it runs CONCURRENTLY with grounding rather than after it — the two have
# nothing to say to each other and serializing them would add the slower
# one's latency to every generation.
# E6's recently-asked read and H3's exam-proximity lookup are independent
# Supabase reads, so all three run CONCURRENTLY rather than in sequence —
# they have nothing to say to each other, and serializing them would add
# every one of their latencies to every generation. Proximity costs
# several round-trips on its own, so running it before this block made it
# fully additive.
#
# return_exceptions=True because BOTH are best-effort context, and a bare
# gather propagates the first failure straight out of generation: an
# return_exceptions=True because ALL THREE are best-effort context, and a
# bare gather propagates the first failure straight out of generation: an
# unreadable past attempt would 502 a quiz that needed no history at all.
# Each helper already degrades internally; this is the backstop for the
# failure they cannot catch (an unexpected raise on the way in or out).
material, recent = await asyncio.gather(
material, recent, exam_days_away = await asyncio.gather(
asyncio.to_thread(_course_material, course_id, concept_name),
asyncio.to_thread(
recent_question_identities, user_id, concept_node_id
),
asyncio.to_thread(days_until_next_exam, user_id, course_id),
return_exceptions=True,
)
if isinstance(exam_days_away, BaseException):
logger.warning(
"quiz: exam-proximity lookup failed (%s); generating without it",
type(exam_days_away).__name__, exc_info=exam_days_away,
)
exam_days_away = None
if isinstance(material, BaseException):
logger.warning(
"quiz: course-material assembly failed (%s); generating ungrounded",
Expand All@@ -900,6 +942,40 @@ async def _quiz_via_agent(
"do-not-repeat list", type(recent).__name__, exc_info=recent,
)
recent = []

routing_msg = (
f"{difficulty_clause} "
f"The target concept is '{concept_name}' "
f"(concept_node_id={concept_node_id}). Follow the workflow in your "
f"system prompt; pass concept_node_id='{concept_node_id}' to "
f"read_recent_quiz_attempts."
)
if use_shared_context:
routing_msg += (
" Also call read_misconceptions_for_course and use those misconceptions "
"as distractors and probes."
)
# H3/#555: one line, dates only. Says how near the deadline is and lets
# the model decide what that implies — not "make it harder", which would
# contradict the adaptive difficulty the student actually chose. Omitted
# entirely when unknown: "next exam: unknown" is prompt tokens spent to
# say nothing.
# Bounded by a horizon: a final dated 87 days out would otherwise put
# "there is an exam coming, weight toward what an exam tests" on EVERY
# quiz for the whole semester, which carries no proximity signal and
# steers week-two practice toward exam-style questions — the opposite of
# what this line is for. The stored `exam_days_away` is NOT clamped: the
# analytics want the real distance, including the far ones.
if exam_days_away is not None and exam_days_away <= PROMPT_HORIZON_DAYS:
when = (
"TODAY" if exam_days_away == 0
else "tomorrow" if exam_days_away == 1
else f"in {exam_days_away} days"
)
routing_msg += (
f" The student's next exam in this course is {when}. Weight the"
" questions toward what an exam would actually test."
)
_log_rag_uncovered(
material,
user_id=user_id,
Expand DownExpand Up@@ -1078,7 +1154,10 @@ def _absorb(quiz: Quiz, model: str) -> None:
"quiz_agent produced no valid questions after wire-format validation"
)
# Never serve more than asked for (a generous top-up run can overshoot).
return wire_questions[:num_questions]
return GeneratedQuiz(
questions=wire_questions[:num_questions],
exam_days_away=exam_days_away,
)

@router.get("/config")
def quiz_config():
Expand DownExpand Up@@ -1169,7 +1248,7 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
# QUIZ_GENERATION_TIMEOUT_SEC (see _run) — cancelling the whole
# coroutine here would discard a partial quiz the top-up handler
# is designed to serve.
questions = await _quiz_via_agent(
generated = await _quiz_via_agent(
user_id=body.user_id,
course_id=course_id,
concept_node_id=body.concept_node_id,
Expand All@@ -1180,6 +1259,8 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
request_id=request_id,
model_pref=body.model_pref,
)
questions = generated.questions
exam_days_away = generated.exam_days_away
except HTTPException:
# The 404 for an unknown concept node is raised before the agent call;
# never swallow a known HTTP state.
Expand DownExpand Up@@ -1224,13 +1305,21 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
) from e

quiz_id = str(uuid.uuid4())
table("quiz_attempts").insert({
attempt_row = {
"id": quiz_id,
"user_id": body.user_id,
"concept_node_id": body.concept_node_id,
"difficulty": body.difficulty,
"questions_json": encrypt_json(questions),
})
}
# H3/#555: recorded so the question "do deadline-aware quizzes perform
# differently?" is answerable later. Omitted when unknown rather than
# written as an explicit null, mirroring apply_graph_update's rule: an
# environment that took this code before the migration keeps generating
# instead of 400ing on a column PostgREST's schema cache doesn't have.
if exam_days_away is not None:
attempt_row["exam_days_away"] = exam_days_away
_insert_attempt(attempt_row)
# #117: quiz.started once the attempt row exists. num_questions is the
# actual generated count (the agent may return fewer than requested).
events_service.log_event(
Expand Down
15 changes: 6 additions & 9 deletions backend/routes/study_guide.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@
)
from services.graph_service import get_courses as graph_get_courses
from services.auth_guard import require_self
from services.exam_proximity import is_exam
from services.encryption import (
decrypt_if_present,
decrypt_json,
Expand DownExpand Up@@ -303,15 +304,11 @@ def get_exams(
order="due_date.asc",
) or []

exam_keywords = ["exam", "midterm", "final", "quiz"]
exams = []
for a in all_assignments:
atype = (a.get("assignment_type") or "").lower()
title = (a.get("title") or "").lower()
if atype == "exam" or any(kw in title for kw in exam_keywords):
exams.append(a)

return {"exams": exams}
# The heuristic moved to services/exam_proximity.py (#555), which needed
# the same question answered on the quiz path. One definition, two
# callers — a second copy would drift, which is the failure #557 spent a
# workstream undoing.
return {"exams": [a for a in all_assignments if is_exam(a)]}


@router.get("/{user_id}/guide")
Expand Down
6 changes: 5 additions & 1 deletion backend/scripts/benchmark_quiz.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -147,7 +147,10 @@ async def _generate_quiz_for_async(concept_name: str) -> list[dict]:
so this exercises the real production grounding path rather than
generating an ungrounded quiz.
"""
return await _quiz_via_agent(
# `.questions`: _quiz_via_agent returns a GeneratedQuiz (#555) so the
# exam-proximity value it resolved can reach the attempt row. This bench
# only wants the questions.
generated = await _quiz_via_agent(
user_id="quizfix-user-0001",
course_id=FIXTURE_COURSE_ID,
concept_node_id="quizfix-node-0001",
Expand All@@ -157,6 +160,7 @@ async def _generate_quiz_for_async(concept_name: str) -> list[dict]:
use_shared_context=False,
request_id="quizfix-bench",
)
return generated.questions


def generate_quiz_for(concept_name: str) -> list[dict]:
Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Force GitHub README to respect dark mode (function() { var style = document.createElement('style'); style.textContent = ' .markdown-body { color-scheme: dark light; } .markdown-body pre { background: #161b22 !important; } .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; } .markdown-body table th, .markdown-body table td { border-color: #30363d !important; } .markdown-body img { background: #0d1117; } .markdown-body blockquote { border-left-color: #8b949e; } .markdown-body hr { border-color: #30363d; } '; document.head.appendChild(style); })(); } } catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' feat(quiz): weight generation toward an approaching exam (#555) by AndresL230 · Pull Request #573 · SaplingLearn/Sapling · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
-- H3 (#555): how many days away the student's next exam was when this quiz
-- was generated.
--
-- Stored on the attempt, not merely used in the prompt, because the point of
-- the issue is to be able to ASK LATER whether deadline-aware quizzes perform
-- differently. A value that only ever reached a prompt leaves no way to
-- compare a quiz taken three days before a midterm against one taken in week
-- two — the measurement is the deliverable, not the prompt line.
--
-- Nullable with no default, and the distinction is load-bearing:
-- NULL = we did not know (no course, no enrollment, no dated exams, or the
-- lookup failed — `days_until_next_exam` degrades to None rather
-- than failing a generation)
-- 0 = the exam is TODAY, which is the most actionable value the feature
-- produces. A DEFAULT 0 would make every legacy row and every
-- unknown look like exam day.
--
-- Dates only. This column is a day count derived from `assignments.due_date`,
-- which is plaintext and indexed; no grade VALUE is read, stored or prompted
-- anywhere on this path (points columns are encrypted per #521, and the audit
-- flags that the current ToS does not clearly cover feeding grades to a
-- model).
--
-- Additive and idempotent. Ordering still matters: PostgREST 400s on a column
-- its schema cache doesn't have, and `generate_quiz` includes this key in the
-- attempt INSERT — so this must be applied strictly before the code ships, as
-- with 20260814051517.
ALTER TABLE quiz_attempts
ADD COLUMN IF NOT EXISTS exam_days_away INTEGER;
137 changes: 113 additions & 24 deletions backend/routes/quiz.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -42,6 +42,7 @@
from services.encryption import encrypt_json, decrypt_json_column
from services.graph_service import apply_graph_update
from services.quiz_context_service import get_quiz_context, save_quiz_context
from services.exam_proximity import PROMPT_HORIZON_DAYS, days_until_next_exam
from services.quiz_distractors import build_distractor_profile
from services.fingerprint import fingerprint
from services.quiz_identity import question_hash, normalize_text
Expand DownExpand Up@@ -795,6 +796,50 @@ def _do_not_repeat_block(recent: list[RecentQuestion]) -> str:
)


def _insert_attempt(attempt_row: dict) -> None:
"""Write the attempt, surviving a schema that predates `exam_days_away`.

The omit-when-None rule alone does NOT make a pre-migration environment
safe, and the failure is nastier than it looks: it strikes exactly the
students the feature is FOR (the ones with a dated upcoming exam), so it
presents as a random partial outage rather than an obvious missing
migration. And it strikes late — the agent has already run and been
billed — so an unhandled 400 here loses the generated quiz, writes no
attempt row, emits no `quiz.generation_failed`, and never refunds the
rate-limit slot.
#
Ordering is still the rule (migration before code, as with
20260814051517). This is the seatbelt for the window where PostgREST has
not yet reloaded its schema cache, not a licence to deploy first.
"""
try:
table("quiz_attempts").insert(attempt_row)
return
except Exception:
if "exam_days_away" not in attempt_row:
raise
retry = {k: v for k, v in attempt_row.items() if k != "exam_days_away"}
logger.warning(
"quiz: attempt insert failed with exam_days_away present; retrying "
"without it (is 20260822090747 applied?) quiz_id=%s",
attempt_row.get("id"),
)
table("quiz_attempts").insert(retry)


class GeneratedQuiz(NamedTuple):
"""What one generation produced.

`exam_days_away` rides back with the questions rather than being resolved
again by the caller: it is used for the prompt here and stored on the
attempt there, and two lookups could disagree if an exam were entered
between them.
"""

questions: list[dict]
exam_days_away: int | None = None


async def _quiz_via_agent(
*,
user_id: str,
Expand All@@ -806,7 +851,7 @@ async def _quiz_via_agent(
use_shared_context: bool,
request_id: str,
model_pref: str | None = None,
) -> list[dict]:
) -> GeneratedQuiz:
"""Run quiz_agent and return questions in the legacy wire shape.

The agent's tools (read_concepts_for_user, read_misconceptions_for_course)
Expand DownExpand Up@@ -849,42 +894,39 @@ async def _quiz_via_agent(
difficulty_clause = (
f"Generate {num_questions} {difficulty} questions for the student."
)
routing_msg = (
f"{difficulty_clause} "
f"The target concept is '{concept_name}' "
f"(concept_node_id={concept_node_id}). Follow the workflow in your "
f"system prompt; pass concept_node_id='{concept_node_id}' to "
f"read_recent_quiz_attempts."
)
if use_shared_context:
routing_msg += (
" Also call read_misconceptions_for_course and use those misconceptions "
"as distractors and probes."
)

# Course-material grounding does blocking network I/O (a Gemini
# embedding call, bounded at 60s) plus sync Supabase reads. Run it in a
# worker thread so a slow/stalled retrieval can't freeze this worker's
# event loop for every other in-flight request. Matches the
# asyncio.to_thread pattern used by the agent read tools.
#
# E6's recently-asked read is an independent Supabase read + decrypt, so
# it runs CONCURRENTLY with grounding rather than after it — the two have
# nothing to say to each other and serializing them would add the slower
# one's latency to every generation.
# E6's recently-asked read and H3's exam-proximity lookup are independent
# Supabase reads, so all three run CONCURRENTLY rather than in sequence —
# they have nothing to say to each other, and serializing them would add
# every one of their latencies to every generation. Proximity costs
# several round-trips on its own, so running it before this block made it
# fully additive.
#
# return_exceptions=True because BOTH are best-effort context, and a bare
# gather propagates the first failure straight out of generation: an
# return_exceptions=True because ALL THREE are best-effort context, and a
# bare gather propagates the first failure straight out of generation: an
# unreadable past attempt would 502 a quiz that needed no history at all.
# Each helper already degrades internally; this is the backstop for the
# failure they cannot catch (an unexpected raise on the way in or out).
material, recent = await asyncio.gather(
material, recent, exam_days_away = await asyncio.gather(
asyncio.to_thread(_course_material, course_id, concept_name),
asyncio.to_thread(
recent_question_identities, user_id, concept_node_id
),
asyncio.to_thread(days_until_next_exam, user_id, course_id),
return_exceptions=True,
)
if isinstance(exam_days_away, BaseException):
logger.warning(
"quiz: exam-proximity lookup failed (%s); generating without it",
type(exam_days_away).__name__, exc_info=exam_days_away,
)
exam_days_away = None
if isinstance(material, BaseException):
logger.warning(
"quiz: course-material assembly failed (%s); generating ungrounded",
Expand All@@ -900,6 +942,40 @@ async def _quiz_via_agent(
"do-not-repeat list", type(recent).__name__, exc_info=recent,
)
recent = []

routing_msg = (
f"{difficulty_clause} "
f"The target concept is '{concept_name}' "
f"(concept_node_id={concept_node_id}). Follow the workflow in your "
f"system prompt; pass concept_node_id='{concept_node_id}' to "
f"read_recent_quiz_attempts."
)
if use_shared_context:
routing_msg += (
" Also call read_misconceptions_for_course and use those misconceptions "
"as distractors and probes."
)
# H3/#555: one line, dates only. Says how near the deadline is and lets
# the model decide what that implies — not "make it harder", which would
# contradict the adaptive difficulty the student actually chose. Omitted
# entirely when unknown: "next exam: unknown" is prompt tokens spent to
# say nothing.
# Bounded by a horizon: a final dated 87 days out would otherwise put
# "there is an exam coming, weight toward what an exam tests" on EVERY
# quiz for the whole semester, which carries no proximity signal and
# steers week-two practice toward exam-style questions — the opposite of
# what this line is for. The stored `exam_days_away` is NOT clamped: the
# analytics want the real distance, including the far ones.
if exam_days_away is not None and exam_days_away <= PROMPT_HORIZON_DAYS:
when = (
"TODAY" if exam_days_away == 0
else "tomorrow" if exam_days_away == 1
else f"in {exam_days_away} days"
)
routing_msg += (
f" The student's next exam in this course is {when}. Weight the"
" questions toward what an exam would actually test."
)
_log_rag_uncovered(
material,
user_id=user_id,
Expand DownExpand Up@@ -1078,7 +1154,10 @@ def _absorb(quiz: Quiz, model: str) -> None:
"quiz_agent produced no valid questions after wire-format validation"
)
# Never serve more than asked for (a generous top-up run can overshoot).
return wire_questions[:num_questions]
return GeneratedQuiz(
questions=wire_questions[:num_questions],
exam_days_away=exam_days_away,
)

@router.get("/config")
def quiz_config():
Expand DownExpand Up@@ -1169,7 +1248,7 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
# QUIZ_GENERATION_TIMEOUT_SEC (see _run) — cancelling the whole
# coroutine here would discard a partial quiz the top-up handler
# is designed to serve.
questions = await _quiz_via_agent(
generated = await _quiz_via_agent(
user_id=body.user_id,
course_id=course_id,
concept_node_id=body.concept_node_id,
Expand All@@ -1180,6 +1259,8 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
request_id=request_id,
model_pref=body.model_pref,
)
questions = generated.questions
exam_days_away = generated.exam_days_away
except HTTPException:
# The 404 for an unknown concept node is raised before the agent call;
# never swallow a known HTTP state.
Expand DownExpand Up@@ -1224,13 +1305,21 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
) from e

quiz_id = str(uuid.uuid4())
table("quiz_attempts").insert({
attempt_row = {
"id": quiz_id,
"user_id": body.user_id,
"concept_node_id": body.concept_node_id,
"difficulty": body.difficulty,
"questions_json": encrypt_json(questions),
})
}
# H3/#555: recorded so the question "do deadline-aware quizzes perform
# differently?" is answerable later. Omitted when unknown rather than
# written as an explicit null, mirroring apply_graph_update's rule: an
# environment that took this code before the migration keeps generating
# instead of 400ing on a column PostgREST's schema cache doesn't have.
if exam_days_away is not None:
attempt_row["exam_days_away"] = exam_days_away
_insert_attempt(attempt_row)
# #117: quiz.started once the attempt row exists. num_questions is the
# actual generated count (the agent may return fewer than requested).
events_service.log_event(
Expand Down
15 changes: 6 additions & 9 deletions backend/routes/study_guide.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@
)
from services.graph_service import get_courses as graph_get_courses
from services.auth_guard import require_self
from services.exam_proximity import is_exam
from services.encryption import (
decrypt_if_present,
decrypt_json,
Expand DownExpand Up@@ -303,15 +304,11 @@ def get_exams(
order="due_date.asc",
) or []

exam_keywords = ["exam", "midterm", "final", "quiz"]
exams = []
for a in all_assignments:
atype = (a.get("assignment_type") or "").lower()
title = (a.get("title") or "").lower()
if atype == "exam" or any(kw in title for kw in exam_keywords):
exams.append(a)

return {"exams": exams}
# The heuristic moved to services/exam_proximity.py (#555), which needed
# the same question answered on the quiz path. One definition, two
# callers — a second copy would drift, which is the failure #557 spent a
# workstream undoing.
return {"exams": [a for a in all_assignments if is_exam(a)]}


@router.get("/{user_id}/guide")
Expand Down
6 changes: 5 additions & 1 deletion backend/scripts/benchmark_quiz.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -147,7 +147,10 @@ async def _generate_quiz_for_async(concept_name: str) -> list[dict]:
so this exercises the real production grounding path rather than
generating an ungrounded quiz.
"""
return await _quiz_via_agent(
# `.questions`: _quiz_via_agent returns a GeneratedQuiz (#555) so the
# exam-proximity value it resolved can reach the attempt row. This bench
# only wants the questions.
generated = await _quiz_via_agent(
user_id="quizfix-user-0001",
course_id=FIXTURE_COURSE_ID,
concept_node_id="quizfix-node-0001",
Expand All@@ -157,6 +160,7 @@ async def _generate_quiz_for_async(concept_name: str) -> list[dict]:
use_shared_context=False,
request_id="quizfix-bench",
)
return generated.questions


def generate_quiz_for(concept_name: str) -> list[dict]:
Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Highlight search terms from Google/DuckDuckGo/Bing referrer (function() { var ref = document.referrer; var terms = []; if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) { var url = new URL(ref); var q = url.searchParams.get('q') || url.searchParams.get('p'); if (q) { terms = q.split(/\s+/).filter(function(t) { return t.length > 2; }); } } if (terms.length === 0) return; var style = document.createElement('style'); style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }'; document.head.appendChild(style); function highlight(node) { if (node.nodeType === 3) { // text node var text = node.textContent; var found = false; terms.forEach(function(term) { var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' feat(quiz): weight generation toward an approaching exam (#555) by AndresL230 · Pull Request #573 · SaplingLearn/Sapling · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
-- H3 (#555): how many days away the student's next exam was when this quiz
-- was generated.
--
-- Stored on the attempt, not merely used in the prompt, because the point of
-- the issue is to be able to ASK LATER whether deadline-aware quizzes perform
-- differently. A value that only ever reached a prompt leaves no way to
-- compare a quiz taken three days before a midterm against one taken in week
-- two — the measurement is the deliverable, not the prompt line.
--
-- Nullable with no default, and the distinction is load-bearing:
-- NULL = we did not know (no course, no enrollment, no dated exams, or the
-- lookup failed — `days_until_next_exam` degrades to None rather
-- than failing a generation)
-- 0 = the exam is TODAY, which is the most actionable value the feature
-- produces. A DEFAULT 0 would make every legacy row and every
-- unknown look like exam day.
--
-- Dates only. This column is a day count derived from `assignments.due_date`,
-- which is plaintext and indexed; no grade VALUE is read, stored or prompted
-- anywhere on this path (points columns are encrypted per #521, and the audit
-- flags that the current ToS does not clearly cover feeding grades to a
-- model).
--
-- Additive and idempotent. Ordering still matters: PostgREST 400s on a column
-- its schema cache doesn't have, and `generate_quiz` includes this key in the
-- attempt INSERT — so this must be applied strictly before the code ships, as
-- with 20260814051517.
ALTER TABLE quiz_attempts
ADD COLUMN IF NOT EXISTS exam_days_away INTEGER;
137 changes: 113 additions & 24 deletions backend/routes/quiz.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -42,6 +42,7 @@
from services.encryption import encrypt_json, decrypt_json_column
from services.graph_service import apply_graph_update
from services.quiz_context_service import get_quiz_context, save_quiz_context
from services.exam_proximity import PROMPT_HORIZON_DAYS, days_until_next_exam
from services.quiz_distractors import build_distractor_profile
from services.fingerprint import fingerprint
from services.quiz_identity import question_hash, normalize_text
Expand DownExpand Up@@ -795,6 +796,50 @@ def _do_not_repeat_block(recent: list[RecentQuestion]) -> str:
)


def _insert_attempt(attempt_row: dict) -> None:
"""Write the attempt, surviving a schema that predates `exam_days_away`.

The omit-when-None rule alone does NOT make a pre-migration environment
safe, and the failure is nastier than it looks: it strikes exactly the
students the feature is FOR (the ones with a dated upcoming exam), so it
presents as a random partial outage rather than an obvious missing
migration. And it strikes late — the agent has already run and been
billed — so an unhandled 400 here loses the generated quiz, writes no
attempt row, emits no `quiz.generation_failed`, and never refunds the
rate-limit slot.
#
Ordering is still the rule (migration before code, as with
20260814051517). This is the seatbelt for the window where PostgREST has
not yet reloaded its schema cache, not a licence to deploy first.
"""
try:
table("quiz_attempts").insert(attempt_row)
return
except Exception:
if "exam_days_away" not in attempt_row:
raise
retry = {k: v for k, v in attempt_row.items() if k != "exam_days_away"}
logger.warning(
"quiz: attempt insert failed with exam_days_away present; retrying "
"without it (is 20260822090747 applied?) quiz_id=%s",
attempt_row.get("id"),
)
table("quiz_attempts").insert(retry)


class GeneratedQuiz(NamedTuple):
"""What one generation produced.

`exam_days_away` rides back with the questions rather than being resolved
again by the caller: it is used for the prompt here and stored on the
attempt there, and two lookups could disagree if an exam were entered
between them.
"""

questions: list[dict]
exam_days_away: int | None = None


async def _quiz_via_agent(
*,
user_id: str,
Expand All@@ -806,7 +851,7 @@ async def _quiz_via_agent(
use_shared_context: bool,
request_id: str,
model_pref: str | None = None,
) -> list[dict]:
) -> GeneratedQuiz:
"""Run quiz_agent and return questions in the legacy wire shape.

The agent's tools (read_concepts_for_user, read_misconceptions_for_course)
Expand DownExpand Up@@ -849,42 +894,39 @@ async def _quiz_via_agent(
difficulty_clause = (
f"Generate {num_questions} {difficulty} questions for the student."
)
routing_msg = (
f"{difficulty_clause} "
f"The target concept is '{concept_name}' "
f"(concept_node_id={concept_node_id}). Follow the workflow in your "
f"system prompt; pass concept_node_id='{concept_node_id}' to "
f"read_recent_quiz_attempts."
)
if use_shared_context:
routing_msg += (
" Also call read_misconceptions_for_course and use those misconceptions "
"as distractors and probes."
)

# Course-material grounding does blocking network I/O (a Gemini
# embedding call, bounded at 60s) plus sync Supabase reads. Run it in a
# worker thread so a slow/stalled retrieval can't freeze this worker's
# event loop for every other in-flight request. Matches the
# asyncio.to_thread pattern used by the agent read tools.
#
# E6's recently-asked read is an independent Supabase read + decrypt, so
# it runs CONCURRENTLY with grounding rather than after it — the two have
# nothing to say to each other and serializing them would add the slower
# one's latency to every generation.
# E6's recently-asked read and H3's exam-proximity lookup are independent
# Supabase reads, so all three run CONCURRENTLY rather than in sequence —
# they have nothing to say to each other, and serializing them would add
# every one of their latencies to every generation. Proximity costs
# several round-trips on its own, so running it before this block made it
# fully additive.
#
# return_exceptions=True because BOTH are best-effort context, and a bare
# gather propagates the first failure straight out of generation: an
# return_exceptions=True because ALL THREE are best-effort context, and a
# bare gather propagates the first failure straight out of generation: an
# unreadable past attempt would 502 a quiz that needed no history at all.
# Each helper already degrades internally; this is the backstop for the
# failure they cannot catch (an unexpected raise on the way in or out).
material, recent = await asyncio.gather(
material, recent, exam_days_away = await asyncio.gather(
asyncio.to_thread(_course_material, course_id, concept_name),
asyncio.to_thread(
recent_question_identities, user_id, concept_node_id
),
asyncio.to_thread(days_until_next_exam, user_id, course_id),
return_exceptions=True,
)
if isinstance(exam_days_away, BaseException):
logger.warning(
"quiz: exam-proximity lookup failed (%s); generating without it",
type(exam_days_away).__name__, exc_info=exam_days_away,
)
exam_days_away = None
if isinstance(material, BaseException):
logger.warning(
"quiz: course-material assembly failed (%s); generating ungrounded",
Expand All@@ -900,6 +942,40 @@ async def _quiz_via_agent(
"do-not-repeat list", type(recent).__name__, exc_info=recent,
)
recent = []

routing_msg = (
f"{difficulty_clause} "
f"The target concept is '{concept_name}' "
f"(concept_node_id={concept_node_id}). Follow the workflow in your "
f"system prompt; pass concept_node_id='{concept_node_id}' to "
f"read_recent_quiz_attempts."
)
if use_shared_context:
routing_msg += (
" Also call read_misconceptions_for_course and use those misconceptions "
"as distractors and probes."
)
# H3/#555: one line, dates only. Says how near the deadline is and lets
# the model decide what that implies — not "make it harder", which would
# contradict the adaptive difficulty the student actually chose. Omitted
# entirely when unknown: "next exam: unknown" is prompt tokens spent to
# say nothing.
# Bounded by a horizon: a final dated 87 days out would otherwise put
# "there is an exam coming, weight toward what an exam tests" on EVERY
# quiz for the whole semester, which carries no proximity signal and
# steers week-two practice toward exam-style questions — the opposite of
# what this line is for. The stored `exam_days_away` is NOT clamped: the
# analytics want the real distance, including the far ones.
if exam_days_away is not None and exam_days_away <= PROMPT_HORIZON_DAYS:
when = (
"TODAY" if exam_days_away == 0
else "tomorrow" if exam_days_away == 1
else f"in {exam_days_away} days"
)
routing_msg += (
f" The student's next exam in this course is {when}. Weight the"
" questions toward what an exam would actually test."
)
_log_rag_uncovered(
material,
user_id=user_id,
Expand DownExpand Up@@ -1078,7 +1154,10 @@ def _absorb(quiz: Quiz, model: str) -> None:
"quiz_agent produced no valid questions after wire-format validation"
)
# Never serve more than asked for (a generous top-up run can overshoot).
return wire_questions[:num_questions]
return GeneratedQuiz(
questions=wire_questions[:num_questions],
exam_days_away=exam_days_away,
)

@router.get("/config")
def quiz_config():
Expand DownExpand Up@@ -1169,7 +1248,7 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
# QUIZ_GENERATION_TIMEOUT_SEC (see _run) — cancelling the whole
# coroutine here would discard a partial quiz the top-up handler
# is designed to serve.
questions = await _quiz_via_agent(
generated = await _quiz_via_agent(
user_id=body.user_id,
course_id=course_id,
concept_node_id=body.concept_node_id,
Expand All@@ -1180,6 +1259,8 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
request_id=request_id,
model_pref=body.model_pref,
)
questions = generated.questions
exam_days_away = generated.exam_days_away
except HTTPException:
# The 404 for an unknown concept node is raised before the agent call;
# never swallow a known HTTP state.
Expand DownExpand Up@@ -1224,13 +1305,21 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
) from e

quiz_id = str(uuid.uuid4())
table("quiz_attempts").insert({
attempt_row = {
"id": quiz_id,
"user_id": body.user_id,
"concept_node_id": body.concept_node_id,
"difficulty": body.difficulty,
"questions_json": encrypt_json(questions),
})
}
# H3/#555: recorded so the question "do deadline-aware quizzes perform
# differently?" is answerable later. Omitted when unknown rather than
# written as an explicit null, mirroring apply_graph_update's rule: an
# environment that took this code before the migration keeps generating
# instead of 400ing on a column PostgREST's schema cache doesn't have.
if exam_days_away is not None:
attempt_row["exam_days_away"] = exam_days_away
_insert_attempt(attempt_row)
# #117: quiz.started once the attempt row exists. num_questions is the
# actual generated count (the agent may return fewer than requested).
events_service.log_event(
Expand Down
15 changes: 6 additions & 9 deletions backend/routes/study_guide.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@
)
from services.graph_service import get_courses as graph_get_courses
from services.auth_guard import require_self
from services.exam_proximity import is_exam
from services.encryption import (
decrypt_if_present,
decrypt_json,
Expand DownExpand Up@@ -303,15 +304,11 @@ def get_exams(
order="due_date.asc",
) or []

exam_keywords = ["exam", "midterm", "final", "quiz"]
exams = []
for a in all_assignments:
atype = (a.get("assignment_type") or "").lower()
title = (a.get("title") or "").lower()
if atype == "exam" or any(kw in title for kw in exam_keywords):
exams.append(a)

return {"exams": exams}
# The heuristic moved to services/exam_proximity.py (#555), which needed
# the same question answered on the quiz path. One definition, two
# callers — a second copy would drift, which is the failure #557 spent a
# workstream undoing.
return {"exams": [a for a in all_assignments if is_exam(a)]}


@router.get("/{user_id}/guide")
Expand Down
6 changes: 5 additions & 1 deletion backend/scripts/benchmark_quiz.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -147,7 +147,10 @@ async def _generate_quiz_for_async(concept_name: str) -> list[dict]:
so this exercises the real production grounding path rather than
generating an ungrounded quiz.
"""
return await _quiz_via_agent(
# `.questions`: _quiz_via_agent returns a GeneratedQuiz (#555) so the
# exam-proximity value it resolved can reach the attempt row. This bench
# only wants the questions.
generated = await _quiz_via_agent(
user_id="quizfix-user-0001",
course_id=FIXTURE_COURSE_ID,
concept_node_id="quizfix-node-0001",
Expand All@@ -157,6 +160,7 @@ async def _generate_quiz_for_async(concept_name: str) -> list[dict]:
use_shared_context=False,
request_id="quizfix-bench",
)
return generated.questions


def generate_quiz_for(concept_name: str) -> list[dict]:
Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ' feat(quiz): weight generation toward an approaching exam (#555) by AndresL230 · Pull Request #573 · SaplingLearn/Sapling · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
-- H3 (#555): how many days away the student's next exam was when this quiz
-- was generated.
--
-- Stored on the attempt, not merely used in the prompt, because the point of
-- the issue is to be able to ASK LATER whether deadline-aware quizzes perform
-- differently. A value that only ever reached a prompt leaves no way to
-- compare a quiz taken three days before a midterm against one taken in week
-- two — the measurement is the deliverable, not the prompt line.
--
-- Nullable with no default, and the distinction is load-bearing:
-- NULL = we did not know (no course, no enrollment, no dated exams, or the
-- lookup failed — `days_until_next_exam` degrades to None rather
-- than failing a generation)
-- 0 = the exam is TODAY, which is the most actionable value the feature
-- produces. A DEFAULT 0 would make every legacy row and every
-- unknown look like exam day.
--
-- Dates only. This column is a day count derived from `assignments.due_date`,
-- which is plaintext and indexed; no grade VALUE is read, stored or prompted
-- anywhere on this path (points columns are encrypted per #521, and the audit
-- flags that the current ToS does not clearly cover feeding grades to a
-- model).
--
-- Additive and idempotent. Ordering still matters: PostgREST 400s on a column
-- its schema cache doesn't have, and `generate_quiz` includes this key in the
-- attempt INSERT — so this must be applied strictly before the code ships, as
-- with 20260814051517.
ALTER TABLE quiz_attempts
ADD COLUMN IF NOT EXISTS exam_days_away INTEGER;
137 changes: 113 additions & 24 deletions backend/routes/quiz.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -42,6 +42,7 @@
from services.encryption import encrypt_json, decrypt_json_column
from services.graph_service import apply_graph_update
from services.quiz_context_service import get_quiz_context, save_quiz_context
from services.exam_proximity import PROMPT_HORIZON_DAYS, days_until_next_exam
from services.quiz_distractors import build_distractor_profile
from services.fingerprint import fingerprint
from services.quiz_identity import question_hash, normalize_text
Expand DownExpand Up@@ -795,6 +796,50 @@ def _do_not_repeat_block(recent: list[RecentQuestion]) -> str:
)


def _insert_attempt(attempt_row: dict) -> None:
"""Write the attempt, surviving a schema that predates `exam_days_away`.

The omit-when-None rule alone does NOT make a pre-migration environment
safe, and the failure is nastier than it looks: it strikes exactly the
students the feature is FOR (the ones with a dated upcoming exam), so it
presents as a random partial outage rather than an obvious missing
migration. And it strikes late — the agent has already run and been
billed — so an unhandled 400 here loses the generated quiz, writes no
attempt row, emits no `quiz.generation_failed`, and never refunds the
rate-limit slot.
#
Ordering is still the rule (migration before code, as with
20260814051517). This is the seatbelt for the window where PostgREST has
not yet reloaded its schema cache, not a licence to deploy first.
"""
try:
table("quiz_attempts").insert(attempt_row)
return
except Exception:
if "exam_days_away" not in attempt_row:
raise
retry = {k: v for k, v in attempt_row.items() if k != "exam_days_away"}
logger.warning(
"quiz: attempt insert failed with exam_days_away present; retrying "
"without it (is 20260822090747 applied?) quiz_id=%s",
attempt_row.get("id"),
)
table("quiz_attempts").insert(retry)


class GeneratedQuiz(NamedTuple):
"""What one generation produced.

`exam_days_away` rides back with the questions rather than being resolved
again by the caller: it is used for the prompt here and stored on the
attempt there, and two lookups could disagree if an exam were entered
between them.
"""

questions: list[dict]
exam_days_away: int | None = None


async def _quiz_via_agent(
*,
user_id: str,
Expand All@@ -806,7 +851,7 @@ async def _quiz_via_agent(
use_shared_context: bool,
request_id: str,
model_pref: str | None = None,
) -> list[dict]:
) -> GeneratedQuiz:
"""Run quiz_agent and return questions in the legacy wire shape.

The agent's tools (read_concepts_for_user, read_misconceptions_for_course)
Expand DownExpand Up@@ -849,42 +894,39 @@ async def _quiz_via_agent(
difficulty_clause = (
f"Generate {num_questions} {difficulty} questions for the student."
)
routing_msg = (
f"{difficulty_clause} "
f"The target concept is '{concept_name}' "
f"(concept_node_id={concept_node_id}). Follow the workflow in your "
f"system prompt; pass concept_node_id='{concept_node_id}' to "
f"read_recent_quiz_attempts."
)
if use_shared_context:
routing_msg += (
" Also call read_misconceptions_for_course and use those misconceptions "
"as distractors and probes."
)

# Course-material grounding does blocking network I/O (a Gemini
# embedding call, bounded at 60s) plus sync Supabase reads. Run it in a
# worker thread so a slow/stalled retrieval can't freeze this worker's
# event loop for every other in-flight request. Matches the
# asyncio.to_thread pattern used by the agent read tools.
#
# E6's recently-asked read is an independent Supabase read + decrypt, so
# it runs CONCURRENTLY with grounding rather than after it — the two have
# nothing to say to each other and serializing them would add the slower
# one's latency to every generation.
# E6's recently-asked read and H3's exam-proximity lookup are independent
# Supabase reads, so all three run CONCURRENTLY rather than in sequence —
# they have nothing to say to each other, and serializing them would add
# every one of their latencies to every generation. Proximity costs
# several round-trips on its own, so running it before this block made it
# fully additive.
#
# return_exceptions=True because BOTH are best-effort context, and a bare
# gather propagates the first failure straight out of generation: an
# return_exceptions=True because ALL THREE are best-effort context, and a
# bare gather propagates the first failure straight out of generation: an
# unreadable past attempt would 502 a quiz that needed no history at all.
# Each helper already degrades internally; this is the backstop for the
# failure they cannot catch (an unexpected raise on the way in or out).
material, recent = await asyncio.gather(
material, recent, exam_days_away = await asyncio.gather(
asyncio.to_thread(_course_material, course_id, concept_name),
asyncio.to_thread(
recent_question_identities, user_id, concept_node_id
),
asyncio.to_thread(days_until_next_exam, user_id, course_id),
return_exceptions=True,
)
if isinstance(exam_days_away, BaseException):
logger.warning(
"quiz: exam-proximity lookup failed (%s); generating without it",
type(exam_days_away).__name__, exc_info=exam_days_away,
)
exam_days_away = None
if isinstance(material, BaseException):
logger.warning(
"quiz: course-material assembly failed (%s); generating ungrounded",
Expand All@@ -900,6 +942,40 @@ async def _quiz_via_agent(
"do-not-repeat list", type(recent).__name__, exc_info=recent,
)
recent = []

routing_msg = (
f"{difficulty_clause} "
f"The target concept is '{concept_name}' "
f"(concept_node_id={concept_node_id}). Follow the workflow in your "
f"system prompt; pass concept_node_id='{concept_node_id}' to "
f"read_recent_quiz_attempts."
)
if use_shared_context:
routing_msg += (
" Also call read_misconceptions_for_course and use those misconceptions "
"as distractors and probes."
)
# H3/#555: one line, dates only. Says how near the deadline is and lets
# the model decide what that implies — not "make it harder", which would
# contradict the adaptive difficulty the student actually chose. Omitted
# entirely when unknown: "next exam: unknown" is prompt tokens spent to
# say nothing.
# Bounded by a horizon: a final dated 87 days out would otherwise put
# "there is an exam coming, weight toward what an exam tests" on EVERY
# quiz for the whole semester, which carries no proximity signal and
# steers week-two practice toward exam-style questions — the opposite of
# what this line is for. The stored `exam_days_away` is NOT clamped: the
# analytics want the real distance, including the far ones.
if exam_days_away is not None and exam_days_away <= PROMPT_HORIZON_DAYS:
when = (
"TODAY" if exam_days_away == 0
else "tomorrow" if exam_days_away == 1
else f"in {exam_days_away} days"
)
routing_msg += (
f" The student's next exam in this course is {when}. Weight the"
" questions toward what an exam would actually test."
)
_log_rag_uncovered(
material,
user_id=user_id,
Expand DownExpand Up@@ -1078,7 +1154,10 @@ def _absorb(quiz: Quiz, model: str) -> None:
"quiz_agent produced no valid questions after wire-format validation"
)
# Never serve more than asked for (a generous top-up run can overshoot).
return wire_questions[:num_questions]
return GeneratedQuiz(
questions=wire_questions[:num_questions],
exam_days_away=exam_days_away,
)

@router.get("/config")
def quiz_config():
Expand DownExpand Up@@ -1169,7 +1248,7 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
# QUIZ_GENERATION_TIMEOUT_SEC (see _run) — cancelling the whole
# coroutine here would discard a partial quiz the top-up handler
# is designed to serve.
questions = await _quiz_via_agent(
generated = await _quiz_via_agent(
user_id=body.user_id,
course_id=course_id,
concept_node_id=body.concept_node_id,
Expand All@@ -1180,6 +1259,8 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
request_id=request_id,
model_pref=body.model_pref,
)
questions = generated.questions
exam_days_away = generated.exam_days_away
except HTTPException:
# The 404 for an unknown concept node is raised before the agent call;
# never swallow a known HTTP state.
Expand DownExpand Up@@ -1224,13 +1305,21 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
) from e

quiz_id = str(uuid.uuid4())
table("quiz_attempts").insert({
attempt_row = {
"id": quiz_id,
"user_id": body.user_id,
"concept_node_id": body.concept_node_id,
"difficulty": body.difficulty,
"questions_json": encrypt_json(questions),
})
}
# H3/#555: recorded so the question "do deadline-aware quizzes perform
# differently?" is answerable later. Omitted when unknown rather than
# written as an explicit null, mirroring apply_graph_update's rule: an
# environment that took this code before the migration keeps generating
# instead of 400ing on a column PostgREST's schema cache doesn't have.
if exam_days_away is not None:
attempt_row["exam_days_away"] = exam_days_away
_insert_attempt(attempt_row)
# #117: quiz.started once the attempt row exists. num_questions is the
# actual generated count (the agent may return fewer than requested).
events_service.log_event(
Expand Down
15 changes: 6 additions & 9 deletions backend/routes/study_guide.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@
)
from services.graph_service import get_courses as graph_get_courses
from services.auth_guard import require_self
from services.exam_proximity import is_exam
from services.encryption import (
decrypt_if_present,
decrypt_json,
Expand DownExpand Up@@ -303,15 +304,11 @@ def get_exams(
order="due_date.asc",
) or []

exam_keywords = ["exam", "midterm", "final", "quiz"]
exams = []
for a in all_assignments:
atype = (a.get("assignment_type") or "").lower()
title = (a.get("title") or "").lower()
if atype == "exam" or any(kw in title for kw in exam_keywords):
exams.append(a)

return {"exams": exams}
# The heuristic moved to services/exam_proximity.py (#555), which needed
# the same question answered on the quiz path. One definition, two
# callers — a second copy would drift, which is the failure #557 spent a
# workstream undoing.
return {"exams": [a for a in all_assignments if is_exam(a)]}


@router.get("/{user_id}/guide")
Expand Down
6 changes: 5 additions & 1 deletion backend/scripts/benchmark_quiz.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -147,7 +147,10 @@ async def _generate_quiz_for_async(concept_name: str) -> list[dict]:
so this exercises the real production grounding path rather than
generating an ungrounded quiz.
"""
return await _quiz_via_agent(
# `.questions`: _quiz_via_agent returns a GeneratedQuiz (#555) so the
# exam-proximity value it resolved can reach the attempt row. This bench
# only wants the questions.
generated = await _quiz_via_agent(
user_id="quizfix-user-0001",
course_id=FIXTURE_COURSE_ID,
concept_node_id="quizfix-node-0001",
Expand All@@ -157,6 +160,7 @@ async def _generate_quiz_for_async(concept_name: str) -> list[dict]:
use_shared_context=False,
request_id="quizfix-bench",
)
return generated.questions


def generate_quiz_for(concept_name: str) -> list[dict]:
Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' feat(quiz): weight generation toward an approaching exam (#555) by AndresL230 · Pull Request #573 · SaplingLearn/Sapling · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
-- H3 (#555): how many days away the student's next exam was when this quiz
-- was generated.
--
-- Stored on the attempt, not merely used in the prompt, because the point of
-- the issue is to be able to ASK LATER whether deadline-aware quizzes perform
-- differently. A value that only ever reached a prompt leaves no way to
-- compare a quiz taken three days before a midterm against one taken in week
-- two — the measurement is the deliverable, not the prompt line.
--
-- Nullable with no default, and the distinction is load-bearing:
-- NULL = we did not know (no course, no enrollment, no dated exams, or the
-- lookup failed — `days_until_next_exam` degrades to None rather
-- than failing a generation)
-- 0 = the exam is TODAY, which is the most actionable value the feature
-- produces. A DEFAULT 0 would make every legacy row and every
-- unknown look like exam day.
--
-- Dates only. This column is a day count derived from `assignments.due_date`,
-- which is plaintext and indexed; no grade VALUE is read, stored or prompted
-- anywhere on this path (points columns are encrypted per #521, and the audit
-- flags that the current ToS does not clearly cover feeding grades to a
-- model).
--
-- Additive and idempotent. Ordering still matters: PostgREST 400s on a column
-- its schema cache doesn't have, and `generate_quiz` includes this key in the
-- attempt INSERT — so this must be applied strictly before the code ships, as
-- with 20260814051517.
ALTER TABLE quiz_attempts
ADD COLUMN IF NOT EXISTS exam_days_away INTEGER;
137 changes: 113 additions & 24 deletions backend/routes/quiz.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -42,6 +42,7 @@
from services.encryption import encrypt_json, decrypt_json_column
from services.graph_service import apply_graph_update
from services.quiz_context_service import get_quiz_context, save_quiz_context
from services.exam_proximity import PROMPT_HORIZON_DAYS, days_until_next_exam
from services.quiz_distractors import build_distractor_profile
from services.fingerprint import fingerprint
from services.quiz_identity import question_hash, normalize_text
Expand DownExpand Up@@ -795,6 +796,50 @@ def _do_not_repeat_block(recent: list[RecentQuestion]) -> str:
)


def _insert_attempt(attempt_row: dict) -> None:
"""Write the attempt, surviving a schema that predates `exam_days_away`.

The omit-when-None rule alone does NOT make a pre-migration environment
safe, and the failure is nastier than it looks: it strikes exactly the
students the feature is FOR (the ones with a dated upcoming exam), so it
presents as a random partial outage rather than an obvious missing
migration. And it strikes late — the agent has already run and been
billed — so an unhandled 400 here loses the generated quiz, writes no
attempt row, emits no `quiz.generation_failed`, and never refunds the
rate-limit slot.
#
Ordering is still the rule (migration before code, as with
20260814051517). This is the seatbelt for the window where PostgREST has
not yet reloaded its schema cache, not a licence to deploy first.
"""
try:
table("quiz_attempts").insert(attempt_row)
return
except Exception:
if "exam_days_away" not in attempt_row:
raise
retry = {k: v for k, v in attempt_row.items() if k != "exam_days_away"}
logger.warning(
"quiz: attempt insert failed with exam_days_away present; retrying "
"without it (is 20260822090747 applied?) quiz_id=%s",
attempt_row.get("id"),
)
table("quiz_attempts").insert(retry)


class GeneratedQuiz(NamedTuple):
"""What one generation produced.

`exam_days_away` rides back with the questions rather than being resolved
again by the caller: it is used for the prompt here and stored on the
attempt there, and two lookups could disagree if an exam were entered
between them.
"""

questions: list[dict]
exam_days_away: int | None = None


async def _quiz_via_agent(
*,
user_id: str,
Expand All@@ -806,7 +851,7 @@ async def _quiz_via_agent(
use_shared_context: bool,
request_id: str,
model_pref: str | None = None,
) -> list[dict]:
) -> GeneratedQuiz:
"""Run quiz_agent and return questions in the legacy wire shape.

The agent's tools (read_concepts_for_user, read_misconceptions_for_course)
Expand DownExpand Up@@ -849,42 +894,39 @@ async def _quiz_via_agent(
difficulty_clause = (
f"Generate {num_questions} {difficulty} questions for the student."
)
routing_msg = (
f"{difficulty_clause} "
f"The target concept is '{concept_name}' "
f"(concept_node_id={concept_node_id}). Follow the workflow in your "
f"system prompt; pass concept_node_id='{concept_node_id}' to "
f"read_recent_quiz_attempts."
)
if use_shared_context:
routing_msg += (
" Also call read_misconceptions_for_course and use those misconceptions "
"as distractors and probes."
)

# Course-material grounding does blocking network I/O (a Gemini
# embedding call, bounded at 60s) plus sync Supabase reads. Run it in a
# worker thread so a slow/stalled retrieval can't freeze this worker's
# event loop for every other in-flight request. Matches the
# asyncio.to_thread pattern used by the agent read tools.
#
# E6's recently-asked read is an independent Supabase read + decrypt, so
# it runs CONCURRENTLY with grounding rather than after it — the two have
# nothing to say to each other and serializing them would add the slower
# one's latency to every generation.
# E6's recently-asked read and H3's exam-proximity lookup are independent
# Supabase reads, so all three run CONCURRENTLY rather than in sequence —
# they have nothing to say to each other, and serializing them would add
# every one of their latencies to every generation. Proximity costs
# several round-trips on its own, so running it before this block made it
# fully additive.
#
# return_exceptions=True because BOTH are best-effort context, and a bare
# gather propagates the first failure straight out of generation: an
# return_exceptions=True because ALL THREE are best-effort context, and a
# bare gather propagates the first failure straight out of generation: an
# unreadable past attempt would 502 a quiz that needed no history at all.
# Each helper already degrades internally; this is the backstop for the
# failure they cannot catch (an unexpected raise on the way in or out).
material, recent = await asyncio.gather(
material, recent, exam_days_away = await asyncio.gather(
asyncio.to_thread(_course_material, course_id, concept_name),
asyncio.to_thread(
recent_question_identities, user_id, concept_node_id
),
asyncio.to_thread(days_until_next_exam, user_id, course_id),
return_exceptions=True,
)
if isinstance(exam_days_away, BaseException):
logger.warning(
"quiz: exam-proximity lookup failed (%s); generating without it",
type(exam_days_away).__name__, exc_info=exam_days_away,
)
exam_days_away = None
if isinstance(material, BaseException):
logger.warning(
"quiz: course-material assembly failed (%s); generating ungrounded",
Expand All@@ -900,6 +942,40 @@ async def _quiz_via_agent(
"do-not-repeat list", type(recent).__name__, exc_info=recent,
)
recent = []

routing_msg = (
f"{difficulty_clause} "
f"The target concept is '{concept_name}' "
f"(concept_node_id={concept_node_id}). Follow the workflow in your "
f"system prompt; pass concept_node_id='{concept_node_id}' to "
f"read_recent_quiz_attempts."
)
if use_shared_context:
routing_msg += (
" Also call read_misconceptions_for_course and use those misconceptions "
"as distractors and probes."
)
# H3/#555: one line, dates only. Says how near the deadline is and lets
# the model decide what that implies — not "make it harder", which would
# contradict the adaptive difficulty the student actually chose. Omitted
# entirely when unknown: "next exam: unknown" is prompt tokens spent to
# say nothing.
# Bounded by a horizon: a final dated 87 days out would otherwise put
# "there is an exam coming, weight toward what an exam tests" on EVERY
# quiz for the whole semester, which carries no proximity signal and
# steers week-two practice toward exam-style questions — the opposite of
# what this line is for. The stored `exam_days_away` is NOT clamped: the
# analytics want the real distance, including the far ones.
if exam_days_away is not None and exam_days_away <= PROMPT_HORIZON_DAYS:
when = (
"TODAY" if exam_days_away == 0
else "tomorrow" if exam_days_away == 1
else f"in {exam_days_away} days"
)
routing_msg += (
f" The student's next exam in this course is {when}. Weight the"
" questions toward what an exam would actually test."
)
_log_rag_uncovered(
material,
user_id=user_id,
Expand DownExpand Up@@ -1078,7 +1154,10 @@ def _absorb(quiz: Quiz, model: str) -> None:
"quiz_agent produced no valid questions after wire-format validation"
)
# Never serve more than asked for (a generous top-up run can overshoot).
return wire_questions[:num_questions]
return GeneratedQuiz(
questions=wire_questions[:num_questions],
exam_days_away=exam_days_away,
)

@router.get("/config")
def quiz_config():
Expand DownExpand Up@@ -1169,7 +1248,7 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
# QUIZ_GENERATION_TIMEOUT_SEC (see _run) — cancelling the whole
# coroutine here would discard a partial quiz the top-up handler
# is designed to serve.
questions = await _quiz_via_agent(
generated = await _quiz_via_agent(
user_id=body.user_id,
course_id=course_id,
concept_node_id=body.concept_node_id,
Expand All@@ -1180,6 +1259,8 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
request_id=request_id,
model_pref=body.model_pref,
)
questions = generated.questions
exam_days_away = generated.exam_days_away
except HTTPException:
# The 404 for an unknown concept node is raised before the agent call;
# never swallow a known HTTP state.
Expand DownExpand Up@@ -1224,13 +1305,21 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
) from e

quiz_id = str(uuid.uuid4())
table("quiz_attempts").insert({
attempt_row = {
"id": quiz_id,
"user_id": body.user_id,
"concept_node_id": body.concept_node_id,
"difficulty": body.difficulty,
"questions_json": encrypt_json(questions),
})
}
# H3/#555: recorded so the question "do deadline-aware quizzes perform
# differently?" is answerable later. Omitted when unknown rather than
# written as an explicit null, mirroring apply_graph_update's rule: an
# environment that took this code before the migration keeps generating
# instead of 400ing on a column PostgREST's schema cache doesn't have.
if exam_days_away is not None:
attempt_row["exam_days_away"] = exam_days_away
_insert_attempt(attempt_row)
# #117: quiz.started once the attempt row exists. num_questions is the
# actual generated count (the agent may return fewer than requested).
events_service.log_event(
Expand Down
15 changes: 6 additions & 9 deletions backend/routes/study_guide.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@
)
from services.graph_service import get_courses as graph_get_courses
from services.auth_guard import require_self
from services.exam_proximity import is_exam
from services.encryption import (
decrypt_if_present,
decrypt_json,
Expand DownExpand Up@@ -303,15 +304,11 @@ def get_exams(
order="due_date.asc",
) or []

exam_keywords = ["exam", "midterm", "final", "quiz"]
exams = []
for a in all_assignments:
atype = (a.get("assignment_type") or "").lower()
title = (a.get("title") or "").lower()
if atype == "exam" or any(kw in title for kw in exam_keywords):
exams.append(a)

return {"exams": exams}
# The heuristic moved to services/exam_proximity.py (#555), which needed
# the same question answered on the quiz path. One definition, two
# callers — a second copy would drift, which is the failure #557 spent a
# workstream undoing.
return {"exams": [a for a in all_assignments if is_exam(a)]}


@router.get("/{user_id}/guide")
Expand Down
6 changes: 5 additions & 1 deletion backend/scripts/benchmark_quiz.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -147,7 +147,10 @@ async def _generate_quiz_for_async(concept_name: str) -> list[dict]:
so this exercises the real production grounding path rather than
generating an ungrounded quiz.
"""
return await _quiz_via_agent(
# `.questions`: _quiz_via_agent returns a GeneratedQuiz (#555) so the
# exam-proximity value it resolved can reach the attempt row. This bench
# only wants the questions.
generated = await _quiz_via_agent(
user_id="quizfix-user-0001",
course_id=FIXTURE_COURSE_ID,
concept_node_id="quizfix-node-0001",
Expand All@@ -157,6 +160,7 @@ async def _generate_quiz_for_async(concept_name: str) -> list[dict]:
use_shared_context=False,
request_id="quizfix-bench",
)
return generated.questions


def generate_quiz_for(concept_name: str) -> list[dict]:
Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' feat(quiz): weight generation toward an approaching exam (#555) by AndresL230 · Pull Request #573 · SaplingLearn/Sapling · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
-- H3 (#555): how many days away the student's next exam was when this quiz
-- was generated.
--
-- Stored on the attempt, not merely used in the prompt, because the point of
-- the issue is to be able to ASK LATER whether deadline-aware quizzes perform
-- differently. A value that only ever reached a prompt leaves no way to
-- compare a quiz taken three days before a midterm against one taken in week
-- two — the measurement is the deliverable, not the prompt line.
--
-- Nullable with no default, and the distinction is load-bearing:
-- NULL = we did not know (no course, no enrollment, no dated exams, or the
-- lookup failed — `days_until_next_exam` degrades to None rather
-- than failing a generation)
-- 0 = the exam is TODAY, which is the most actionable value the feature
-- produces. A DEFAULT 0 would make every legacy row and every
-- unknown look like exam day.
--
-- Dates only. This column is a day count derived from `assignments.due_date`,
-- which is plaintext and indexed; no grade VALUE is read, stored or prompted
-- anywhere on this path (points columns are encrypted per #521, and the audit
-- flags that the current ToS does not clearly cover feeding grades to a
-- model).
--
-- Additive and idempotent. Ordering still matters: PostgREST 400s on a column
-- its schema cache doesn't have, and `generate_quiz` includes this key in the
-- attempt INSERT — so this must be applied strictly before the code ships, as
-- with 20260814051517.
ALTER TABLE quiz_attempts
ADD COLUMN IF NOT EXISTS exam_days_away INTEGER;
137 changes: 113 additions & 24 deletions backend/routes/quiz.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -42,6 +42,7 @@
from services.encryption import encrypt_json, decrypt_json_column
from services.graph_service import apply_graph_update
from services.quiz_context_service import get_quiz_context, save_quiz_context
from services.exam_proximity import PROMPT_HORIZON_DAYS, days_until_next_exam
from services.quiz_distractors import build_distractor_profile
from services.fingerprint import fingerprint
from services.quiz_identity import question_hash, normalize_text
Expand DownExpand Up@@ -795,6 +796,50 @@ def _do_not_repeat_block(recent: list[RecentQuestion]) -> str:
)


def _insert_attempt(attempt_row: dict) -> None:
"""Write the attempt, surviving a schema that predates `exam_days_away`.

The omit-when-None rule alone does NOT make a pre-migration environment
safe, and the failure is nastier than it looks: it strikes exactly the
students the feature is FOR (the ones with a dated upcoming exam), so it
presents as a random partial outage rather than an obvious missing
migration. And it strikes late — the agent has already run and been
billed — so an unhandled 400 here loses the generated quiz, writes no
attempt row, emits no `quiz.generation_failed`, and never refunds the
rate-limit slot.
#
Ordering is still the rule (migration before code, as with
20260814051517). This is the seatbelt for the window where PostgREST has
not yet reloaded its schema cache, not a licence to deploy first.
"""
try:
table("quiz_attempts").insert(attempt_row)
return
except Exception:
if "exam_days_away" not in attempt_row:
raise
retry = {k: v for k, v in attempt_row.items() if k != "exam_days_away"}
logger.warning(
"quiz: attempt insert failed with exam_days_away present; retrying "
"without it (is 20260822090747 applied?) quiz_id=%s",
attempt_row.get("id"),
)
table("quiz_attempts").insert(retry)


class GeneratedQuiz(NamedTuple):
"""What one generation produced.

`exam_days_away` rides back with the questions rather than being resolved
again by the caller: it is used for the prompt here and stored on the
attempt there, and two lookups could disagree if an exam were entered
between them.
"""

questions: list[dict]
exam_days_away: int | None = None


async def _quiz_via_agent(
*,
user_id: str,
Expand All@@ -806,7 +851,7 @@ async def _quiz_via_agent(
use_shared_context: bool,
request_id: str,
model_pref: str | None = None,
) -> list[dict]:
) -> GeneratedQuiz:
"""Run quiz_agent and return questions in the legacy wire shape.

The agent's tools (read_concepts_for_user, read_misconceptions_for_course)
Expand DownExpand Up@@ -849,42 +894,39 @@ async def _quiz_via_agent(
difficulty_clause = (
f"Generate {num_questions} {difficulty} questions for the student."
)
routing_msg = (
f"{difficulty_clause} "
f"The target concept is '{concept_name}' "
f"(concept_node_id={concept_node_id}). Follow the workflow in your "
f"system prompt; pass concept_node_id='{concept_node_id}' to "
f"read_recent_quiz_attempts."
)
if use_shared_context:
routing_msg += (
" Also call read_misconceptions_for_course and use those misconceptions "
"as distractors and probes."
)

# Course-material grounding does blocking network I/O (a Gemini
# embedding call, bounded at 60s) plus sync Supabase reads. Run it in a
# worker thread so a slow/stalled retrieval can't freeze this worker's
# event loop for every other in-flight request. Matches the
# asyncio.to_thread pattern used by the agent read tools.
#
# E6's recently-asked read is an independent Supabase read + decrypt, so
# it runs CONCURRENTLY with grounding rather than after it — the two have
# nothing to say to each other and serializing them would add the slower
# one's latency to every generation.
# E6's recently-asked read and H3's exam-proximity lookup are independent
# Supabase reads, so all three run CONCURRENTLY rather than in sequence —
# they have nothing to say to each other, and serializing them would add
# every one of their latencies to every generation. Proximity costs
# several round-trips on its own, so running it before this block made it
# fully additive.
#
# return_exceptions=True because BOTH are best-effort context, and a bare
# gather propagates the first failure straight out of generation: an
# return_exceptions=True because ALL THREE are best-effort context, and a
# bare gather propagates the first failure straight out of generation: an
# unreadable past attempt would 502 a quiz that needed no history at all.
# Each helper already degrades internally; this is the backstop for the
# failure they cannot catch (an unexpected raise on the way in or out).
material, recent = await asyncio.gather(
material, recent, exam_days_away = await asyncio.gather(
asyncio.to_thread(_course_material, course_id, concept_name),
asyncio.to_thread(
recent_question_identities, user_id, concept_node_id
),
asyncio.to_thread(days_until_next_exam, user_id, course_id),
return_exceptions=True,
)
if isinstance(exam_days_away, BaseException):
logger.warning(
"quiz: exam-proximity lookup failed (%s); generating without it",
type(exam_days_away).__name__, exc_info=exam_days_away,
)
exam_days_away = None
if isinstance(material, BaseException):
logger.warning(
"quiz: course-material assembly failed (%s); generating ungrounded",
Expand All@@ -900,6 +942,40 @@ async def _quiz_via_agent(
"do-not-repeat list", type(recent).__name__, exc_info=recent,
)
recent = []

routing_msg = (
f"{difficulty_clause} "
f"The target concept is '{concept_name}' "
f"(concept_node_id={concept_node_id}). Follow the workflow in your "
f"system prompt; pass concept_node_id='{concept_node_id}' to "
f"read_recent_quiz_attempts."
)
if use_shared_context:
routing_msg += (
" Also call read_misconceptions_for_course and use those misconceptions "
"as distractors and probes."
)
# H3/#555: one line, dates only. Says how near the deadline is and lets
# the model decide what that implies — not "make it harder", which would
# contradict the adaptive difficulty the student actually chose. Omitted
# entirely when unknown: "next exam: unknown" is prompt tokens spent to
# say nothing.
# Bounded by a horizon: a final dated 87 days out would otherwise put
# "there is an exam coming, weight toward what an exam tests" on EVERY
# quiz for the whole semester, which carries no proximity signal and
# steers week-two practice toward exam-style questions — the opposite of
# what this line is for. The stored `exam_days_away` is NOT clamped: the
# analytics want the real distance, including the far ones.
if exam_days_away is not None and exam_days_away <= PROMPT_HORIZON_DAYS:
when = (
"TODAY" if exam_days_away == 0
else "tomorrow" if exam_days_away == 1
else f"in {exam_days_away} days"
)
routing_msg += (
f" The student's next exam in this course is {when}. Weight the"
" questions toward what an exam would actually test."
)
_log_rag_uncovered(
material,
user_id=user_id,
Expand DownExpand Up@@ -1078,7 +1154,10 @@ def _absorb(quiz: Quiz, model: str) -> None:
"quiz_agent produced no valid questions after wire-format validation"
)
# Never serve more than asked for (a generous top-up run can overshoot).
return wire_questions[:num_questions]
return GeneratedQuiz(
questions=wire_questions[:num_questions],
exam_days_away=exam_days_away,
)

@router.get("/config")
def quiz_config():
Expand DownExpand Up@@ -1169,7 +1248,7 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
# QUIZ_GENERATION_TIMEOUT_SEC (see _run) — cancelling the whole
# coroutine here would discard a partial quiz the top-up handler
# is designed to serve.
questions = await _quiz_via_agent(
generated = await _quiz_via_agent(
user_id=body.user_id,
course_id=course_id,
concept_node_id=body.concept_node_id,
Expand All@@ -1180,6 +1259,8 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
request_id=request_id,
model_pref=body.model_pref,
)
questions = generated.questions
exam_days_away = generated.exam_days_away
except HTTPException:
# The 404 for an unknown concept node is raised before the agent call;
# never swallow a known HTTP state.
Expand DownExpand Up@@ -1224,13 +1305,21 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
) from e

quiz_id = str(uuid.uuid4())
table("quiz_attempts").insert({
attempt_row = {
"id": quiz_id,
"user_id": body.user_id,
"concept_node_id": body.concept_node_id,
"difficulty": body.difficulty,
"questions_json": encrypt_json(questions),
})
}
# H3/#555: recorded so the question "do deadline-aware quizzes perform
# differently?" is answerable later. Omitted when unknown rather than
# written as an explicit null, mirroring apply_graph_update's rule: an
# environment that took this code before the migration keeps generating
# instead of 400ing on a column PostgREST's schema cache doesn't have.
if exam_days_away is not None:
attempt_row["exam_days_away"] = exam_days_away
_insert_attempt(attempt_row)
# #117: quiz.started once the attempt row exists. num_questions is the
# actual generated count (the agent may return fewer than requested).
events_service.log_event(
Expand Down
15 changes: 6 additions & 9 deletions backend/routes/study_guide.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@
)
from services.graph_service import get_courses as graph_get_courses
from services.auth_guard import require_self
from services.exam_proximity import is_exam
from services.encryption import (
decrypt_if_present,
decrypt_json,
Expand DownExpand Up@@ -303,15 +304,11 @@ def get_exams(
order="due_date.asc",
) or []

exam_keywords = ["exam", "midterm", "final", "quiz"]
exams = []
for a in all_assignments:
atype = (a.get("assignment_type") or "").lower()
title = (a.get("title") or "").lower()
if atype == "exam" or any(kw in title for kw in exam_keywords):
exams.append(a)

return {"exams": exams}
# The heuristic moved to services/exam_proximity.py (#555), which needed
# the same question answered on the quiz path. One definition, two
# callers — a second copy would drift, which is the failure #557 spent a
# workstream undoing.
return {"exams": [a for a in all_assignments if is_exam(a)]}


@router.get("/{user_id}/guide")
Expand Down
6 changes: 5 additions & 1 deletion backend/scripts/benchmark_quiz.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -147,7 +147,10 @@ async def _generate_quiz_for_async(concept_name: str) -> list[dict]:
so this exercises the real production grounding path rather than
generating an ungrounded quiz.
"""
return await _quiz_via_agent(
# `.questions`: _quiz_via_agent returns a GeneratedQuiz (#555) so the
# exam-proximity value it resolved can reach the attempt row. This bench
# only wants the questions.
generated = await _quiz_via_agent(
user_id="quizfix-user-0001",
course_id=FIXTURE_COURSE_ID,
concept_node_id="quizfix-node-0001",
Expand All@@ -157,6 +160,7 @@ async def _generate_quiz_for_async(concept_name: str) -> list[dict]:
use_shared_context=False,
request_id="quizfix-bench",
)
return generated.questions


def generate_quiz_for(concept_name: str) -> list[dict]:
Expand Down
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Universal Dark Mode - works on any site (function() { var enabled = true; function applyDarkMode() { if (!enabled) return; // Create style element if it doesn't exist var style = document.getElementById('universal-dark-mode-style'); if (!style) { style = document.createElement('style'); style.id = 'universal-dark-mode-style'; document.head.appendChild(style); } // Dark mode CSS - inverts colors but preserves images/video style.textContent = ' /* Invert everything except media */ html { filter: invert(1) hue-rotate(180deg) !important; background: #1a1a2e !important; } /* Restore images, videos, iframes, canvas */ img, video, iframe, canvas, svg, picture, [style*="background-image"] { filter: invert(1) hue-rotate(180deg) !important; } /* Preserve specific elements that should not be inverted */ .no-dark-mode, .no-dark-mode *, [data-theme="light"], [data-theme="light"], .ace_editor, .ace_editor *, .CodeMirror, .CodeMirror *, .monaco-editor, .monaco-editor *, .markdown-body pre, .markdown-body pre *, .highlight, .highlight *, pre code, pre code * { filter: none !important; } /* Fix common UI elements */ .modal, .popup, .dropdown-menu, .tooltip, .popover { filter: invert(1) hue-rotate(180deg) !important; background: #2d2d44 !important; border-color: #444 !important; } /* Scrollbars */ ::-webkit-scrollbar { background: #1a1a2e !important; } ::-webkit-scrollbar-thumb { background: #444 !important; } ::-webkit-scrollbar-thumb:hover { background: #555 !important; } /* Selection */ ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; } ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; } '; } function removeDarkMode() { var style = document.getElementById('universal-dark-mode-style'); if (style) style.remove(); } // Toggle with Alt+Shift+D document.addEventListener('keydown', function(e) { if (e.altKey && e.shiftKey && e.key === 'D') { e.preventDefault(); enabled = !enabled; if (enabled) { applyDarkMode(); console.log('[Universal Dark Mode] Enabled'); } else { removeDarkMode(); console.log('[Universal Dark Mode] Disabled'); } } }); // Apply on load applyDarkMode(); // Re-apply on dynamic content var observer = new MutationObserver(function(mutations) { if (enabled && !document.getElementById('universal-dark-mode-style')) { applyDarkMode(); } }); observer.observe(document.head, { childList: true }); console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle'); })(); } } catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); } })(); })(); feat(quiz): weight generation toward an approaching exam (#555) by AndresL230 · Pull Request #573 · SaplingLearn/Sapling · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
-- H3 (#555): how many days away the student's next exam was when this quiz
-- was generated.
--
-- Stored on the attempt, not merely used in the prompt, because the point of
-- the issue is to be able to ASK LATER whether deadline-aware quizzes perform
-- differently. A value that only ever reached a prompt leaves no way to
-- compare a quiz taken three days before a midterm against one taken in week
-- two — the measurement is the deliverable, not the prompt line.
--
-- Nullable with no default, and the distinction is load-bearing:
-- NULL = we did not know (no course, no enrollment, no dated exams, or the
-- lookup failed — `days_until_next_exam` degrades to None rather
-- than failing a generation)
-- 0 = the exam is TODAY, which is the most actionable value the feature
-- produces. A DEFAULT 0 would make every legacy row and every
-- unknown look like exam day.
--
-- Dates only. This column is a day count derived from `assignments.due_date`,
-- which is plaintext and indexed; no grade VALUE is read, stored or prompted
-- anywhere on this path (points columns are encrypted per #521, and the audit
-- flags that the current ToS does not clearly cover feeding grades to a
-- model).
--
-- Additive and idempotent. Ordering still matters: PostgREST 400s on a column
-- its schema cache doesn't have, and `generate_quiz` includes this key in the
-- attempt INSERT — so this must be applied strictly before the code ships, as
-- with 20260814051517.
ALTER TABLE quiz_attempts
ADD COLUMN IF NOT EXISTS exam_days_away INTEGER;
137 changes: 113 additions & 24 deletions backend/routes/quiz.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -42,6 +42,7 @@
from services.encryption import encrypt_json, decrypt_json_column
from services.graph_service import apply_graph_update
from services.quiz_context_service import get_quiz_context, save_quiz_context
from services.exam_proximity import PROMPT_HORIZON_DAYS, days_until_next_exam
from services.quiz_distractors import build_distractor_profile
from services.fingerprint import fingerprint
from services.quiz_identity import question_hash, normalize_text
Expand DownExpand Up@@ -795,6 +796,50 @@ def _do_not_repeat_block(recent: list[RecentQuestion]) -> str:
)


def _insert_attempt(attempt_row: dict) -> None:
"""Write the attempt, surviving a schema that predates `exam_days_away`.

The omit-when-None rule alone does NOT make a pre-migration environment
safe, and the failure is nastier than it looks: it strikes exactly the
students the feature is FOR (the ones with a dated upcoming exam), so it
presents as a random partial outage rather than an obvious missing
migration. And it strikes late — the agent has already run and been
billed — so an unhandled 400 here loses the generated quiz, writes no
attempt row, emits no `quiz.generation_failed`, and never refunds the
rate-limit slot.
#
Ordering is still the rule (migration before code, as with
20260814051517). This is the seatbelt for the window where PostgREST has
not yet reloaded its schema cache, not a licence to deploy first.
"""
try:
table("quiz_attempts").insert(attempt_row)
return
except Exception:
if "exam_days_away" not in attempt_row:
raise
retry = {k: v for k, v in attempt_row.items() if k != "exam_days_away"}
logger.warning(
"quiz: attempt insert failed with exam_days_away present; retrying "
"without it (is 20260822090747 applied?) quiz_id=%s",
attempt_row.get("id"),
)
table("quiz_attempts").insert(retry)


class GeneratedQuiz(NamedTuple):
"""What one generation produced.

`exam_days_away` rides back with the questions rather than being resolved
again by the caller: it is used for the prompt here and stored on the
attempt there, and two lookups could disagree if an exam were entered
between them.
"""

questions: list[dict]
exam_days_away: int | None = None


async def _quiz_via_agent(
*,
user_id: str,
Expand All@@ -806,7 +851,7 @@ async def _quiz_via_agent(
use_shared_context: bool,
request_id: str,
model_pref: str | None = None,
) -> list[dict]:
) -> GeneratedQuiz:
"""Run quiz_agent and return questions in the legacy wire shape.

The agent's tools (read_concepts_for_user, read_misconceptions_for_course)
Expand DownExpand Up@@ -849,42 +894,39 @@ async def _quiz_via_agent(
difficulty_clause = (
f"Generate {num_questions} {difficulty} questions for the student."
)
routing_msg = (
f"{difficulty_clause} "
f"The target concept is '{concept_name}' "
f"(concept_node_id={concept_node_id}). Follow the workflow in your "
f"system prompt; pass concept_node_id='{concept_node_id}' to "
f"read_recent_quiz_attempts."
)
if use_shared_context:
routing_msg += (
" Also call read_misconceptions_for_course and use those misconceptions "
"as distractors and probes."
)

# Course-material grounding does blocking network I/O (a Gemini
# embedding call, bounded at 60s) plus sync Supabase reads. Run it in a
# worker thread so a slow/stalled retrieval can't freeze this worker's
# event loop for every other in-flight request. Matches the
# asyncio.to_thread pattern used by the agent read tools.
#
# E6's recently-asked read is an independent Supabase read + decrypt, so
# it runs CONCURRENTLY with grounding rather than after it — the two have
# nothing to say to each other and serializing them would add the slower
# one's latency to every generation.
# E6's recently-asked read and H3's exam-proximity lookup are independent
# Supabase reads, so all three run CONCURRENTLY rather than in sequence —
# they have nothing to say to each other, and serializing them would add
# every one of their latencies to every generation. Proximity costs
# several round-trips on its own, so running it before this block made it
# fully additive.
#
# return_exceptions=True because BOTH are best-effort context, and a bare
# gather propagates the first failure straight out of generation: an
# return_exceptions=True because ALL THREE are best-effort context, and a
# bare gather propagates the first failure straight out of generation: an
# unreadable past attempt would 502 a quiz that needed no history at all.
# Each helper already degrades internally; this is the backstop for the
# failure they cannot catch (an unexpected raise on the way in or out).
material, recent = await asyncio.gather(
material, recent, exam_days_away = await asyncio.gather(
asyncio.to_thread(_course_material, course_id, concept_name),
asyncio.to_thread(
recent_question_identities, user_id, concept_node_id
),
asyncio.to_thread(days_until_next_exam, user_id, course_id),
return_exceptions=True,
)
if isinstance(exam_days_away, BaseException):
logger.warning(
"quiz: exam-proximity lookup failed (%s); generating without it",
type(exam_days_away).__name__, exc_info=exam_days_away,
)
exam_days_away = None
if isinstance(material, BaseException):
logger.warning(
"quiz: course-material assembly failed (%s); generating ungrounded",
Expand All@@ -900,6 +942,40 @@ async def _quiz_via_agent(
"do-not-repeat list", type(recent).__name__, exc_info=recent,
)
recent = []

routing_msg = (
f"{difficulty_clause} "
f"The target concept is '{concept_name}' "
f"(concept_node_id={concept_node_id}). Follow the workflow in your "
f"system prompt; pass concept_node_id='{concept_node_id}' to "
f"read_recent_quiz_attempts."
)
if use_shared_context:
routing_msg += (
" Also call read_misconceptions_for_course and use those misconceptions "
"as distractors and probes."
)
# H3/#555: one line, dates only. Says how near the deadline is and lets
# the model decide what that implies — not "make it harder", which would
# contradict the adaptive difficulty the student actually chose. Omitted
# entirely when unknown: "next exam: unknown" is prompt tokens spent to
# say nothing.
# Bounded by a horizon: a final dated 87 days out would otherwise put
# "there is an exam coming, weight toward what an exam tests" on EVERY
# quiz for the whole semester, which carries no proximity signal and
# steers week-two practice toward exam-style questions — the opposite of
# what this line is for. The stored `exam_days_away` is NOT clamped: the
# analytics want the real distance, including the far ones.
if exam_days_away is not None and exam_days_away <= PROMPT_HORIZON_DAYS:
when = (
"TODAY" if exam_days_away == 0
else "tomorrow" if exam_days_away == 1
else f"in {exam_days_away} days"
)
routing_msg += (
f" The student's next exam in this course is {when}. Weight the"
" questions toward what an exam would actually test."
)
_log_rag_uncovered(
material,
user_id=user_id,
Expand DownExpand Up@@ -1078,7 +1154,10 @@ def _absorb(quiz: Quiz, model: str) -> None:
"quiz_agent produced no valid questions after wire-format validation"
)
# Never serve more than asked for (a generous top-up run can overshoot).
return wire_questions[:num_questions]
return GeneratedQuiz(
questions=wire_questions[:num_questions],
exam_days_away=exam_days_away,
)

@router.get("/config")
def quiz_config():
Expand DownExpand Up@@ -1169,7 +1248,7 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
# QUIZ_GENERATION_TIMEOUT_SEC (see _run) — cancelling the whole
# coroutine here would discard a partial quiz the top-up handler
# is designed to serve.
questions = await _quiz_via_agent(
generated = await _quiz_via_agent(
user_id=body.user_id,
course_id=course_id,
concept_node_id=body.concept_node_id,
Expand All@@ -1180,6 +1259,8 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
request_id=request_id,
model_pref=body.model_pref,
)
questions = generated.questions
exam_days_away = generated.exam_days_away
except HTTPException:
# The 404 for an unknown concept node is raised before the agent call;
# never swallow a known HTTP state.
Expand DownExpand Up@@ -1224,13 +1305,21 @@ async def generate_quiz(body: GenerateQuizBody, request: Request):
) from e

quiz_id = str(uuid.uuid4())
table("quiz_attempts").insert({
attempt_row = {
"id": quiz_id,
"user_id": body.user_id,
"concept_node_id": body.concept_node_id,
"difficulty": body.difficulty,
"questions_json": encrypt_json(questions),
})
}
# H3/#555: recorded so the question "do deadline-aware quizzes perform
# differently?" is answerable later. Omitted when unknown rather than
# written as an explicit null, mirroring apply_graph_update's rule: an
# environment that took this code before the migration keeps generating
# instead of 400ing on a column PostgREST's schema cache doesn't have.
if exam_days_away is not None:
attempt_row["exam_days_away"] = exam_days_away
_insert_attempt(attempt_row)
# #117: quiz.started once the attempt row exists. num_questions is the
# actual generated count (the agent may return fewer than requested).
events_service.log_event(
Expand Down
15 changes: 6 additions & 9 deletions backend/routes/study_guide.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@
)
from services.graph_service import get_courses as graph_get_courses
from services.auth_guard import require_self
from services.exam_proximity import is_exam
from services.encryption import (
decrypt_if_present,
decrypt_json,
Expand DownExpand Up@@ -303,15 +304,11 @@ def get_exams(
order="due_date.asc",
) or []

exam_keywords = ["exam", "midterm", "final", "quiz"]
exams = []
for a in all_assignments:
atype = (a.get("assignment_type") or "").lower()
title = (a.get("title") or "").lower()
if atype == "exam" or any(kw in title for kw in exam_keywords):
exams.append(a)

return {"exams": exams}
# The heuristic moved to services/exam_proximity.py (#555), which needed
# the same question answered on the quiz path. One definition, two
# callers — a second copy would drift, which is the failure #557 spent a
# workstream undoing.
return {"exams": [a for a in all_assignments if is_exam(a)]}


@router.get("/{user_id}/guide")
Expand Down
6 changes: 5 additions & 1 deletion backend/scripts/benchmark_quiz.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -147,7 +147,10 @@ async def _generate_quiz_for_async(concept_name: str) -> list[dict]:
so this exercises the real production grounding path rather than
generating an ungrounded quiz.
"""
return await _quiz_via_agent(
# `.questions`: _quiz_via_agent returns a GeneratedQuiz (#555) so the
# exam-proximity value it resolved can reach the attempt row. This bench
# only wants the questions.
generated = await _quiz_via_agent(
user_id="quizfix-user-0001",
course_id=FIXTURE_COURSE_ID,
concept_node_id="quizfix-node-0001",
Expand All@@ -157,6 +160,7 @@ async def _generate_quiz_for_async(concept_name: str) -> list[dict]:
use_shared_context=False,
request_id="quizfix-bench",
)
return generated.questions


def generate_quiz_for(concept_name: str) -> list[dict]:
Expand Down
Loading
Loading