Add score trace URL to evaluation run

backend/app/alembic/versions/044_add_score_trace_url_to_evaluation_run.py

@@ -0,0 +1,32 @@
+"""Add score_trace_url to evaluation_run
+
+Revision ID: 044
+Revises: 043
+Create Date: 2026-01-24 19:34:46.763908
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel.sql.sqltypes
+
+
+revision = "044"
+down_revision = "043"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ op.add_column(
+ "evaluation_run",
+ sa.Column(
+ "score_trace_url",
+ sqlmodel.sql.sqltypes.AutoString(),
+ nullable=True,
+ comment="S3 URL where per-trace evaluation scores are stored",
+ ),
+ )
+
+
+def downgrade():
+ op.drop_column("evaluation_run", "score_trace_url")

backend/app/core/storage_utils.py

@@ -15,6 +15,7 @@
 from starlette.datastructures import Headers, UploadFile
 
 from app.core.cloud.storage import CloudStorage, CloudStorageError
+from typing import Literal
 
 logger = logging.getLogger(__name__)
 
@@ -88,6 +89,7 @@ def upload_jsonl_to_object_store(
  results: list[dict],
  filename: str,
  subdirectory: str,
+ format: Literal["json", "jsonl"] = "jsonl",
 ) -> str | None:
  """
  Upload JSONL (JSON Lines) content to object store.
@@ -114,12 +116,20 @@ def upload_jsonl_to_object_store(
  # Create file path
  file_path = Path(subdirectory) / filename
 
- # Convert results to JSONL
- jsonl_content = "\n".join([json.dumps(result) for result in results])
+ if format == "jsonl":
+ jsonl_content = (
+ "\n".join(json.dumps(result, ensure_ascii=False) for result in results)
+ + "\n"
+ )
+ content_type = {"content-type": "application/jsonl"}
+ else:
+ jsonl_content = json.dumps(results, ensure_ascii=False)
+ content_type = {"content-type": "application/json"}
+
  content_bytes = jsonl_content.encode("utf-8")
 
  # Create UploadFile-like object
- headers = Headers({"content-type": "application/jsonl"})
+ headers = Headers(content_type)
  upload_file = UploadFile(
  filename=filename,
  file=BytesIO(content_bytes),
@@ -152,6 +162,37 @@ def upload_jsonl_to_object_store(
  return None
 
 
+def load_json_from_object_store(storage: CloudStorage, url: str) -> list | dict | None:
+ logger.info(f"[load_json_from_object_store] Loading JSON from '{url}")
+ try:
+ body = storage.stream(url)
+ content = body.read()
+
+ data = json.loads(content.decode("utf-8"))
+
+ logger.info(
+ f"[load_json_from_object_store] Download successful | "
+ f"url='{url}', size={len(content)} bytes"
+ )
+ return data
+ except CloudStorageError as e:
+ logger.warning(
+ f"[load_json_from_object_store] failed to load JSON from '{url}': {e}",
+ )
+ return None
+ except json.JSONDecodeError as e:
+ logger.warning(
+ f"[load_json_from_object_store] JSON decode error loading JSON from '{url}': {e}",
+ )
+ return None
+ except Exception as e:
+ logger.warning(
+ f"[load_json_from_object_store] unexpected error loading JSON from '{url}': {e}",
+ exc_info=True,
+ )
+ return None
+
+
 def generate_timestamped_filename(base_name: str, extension: str = "csv") -> str:
  """
  Generate a filename with timestamp.

backend/app/crud/evaluations/core.py

@@ -13,6 +13,10 @@
 from app.models.llm.request import ConfigBlob, LLMCallConfig
 from app.services.llm.jobs import resolve_config_blob
 
+from app.core.db import engine
+from app.core.cloud.storage import get_cloud_storage
+from app.core.storage_utils import upload_jsonl_to_object_store
+
 logger = logging.getLogger(__name__)
 
 
@@ -187,6 +191,7 @@ def update_evaluation_run(
  status: str | None = None,
  error_message: str | None = None,
  object_store_url: str | None = None,
+ score_trace_url: str | None = None,
  score: dict | None = None,
  embedding_batch_job_id: int | None = None,
 ) -> EvaluationRun:
@@ -219,6 +224,8 @@ def update_evaluation_run(
  eval_run.score = score
  if embedding_batch_job_id is not None:
  eval_run.embedding_batch_job_id = embedding_batch_job_id
+ if score_trace_url is not None:
+ eval_run.score_trace_url = score_trace_url or None
 
  # Always update timestamp
  eval_run.updated_at = now()
@@ -335,7 +342,6 @@ def save_score(
  Returns:
  Updated EvaluationRun instance, or None if not found
  """
- from app.core.db import engine
 
  with Session(engine) as session:
  eval_run = get_evaluation_run_by_id(
@@ -344,12 +350,61 @@ def save_score(
  organization_id=organization_id,
  project_id=project_id,
  )
- if eval_run:
- update_evaluation_run(session=session, eval_run=eval_run, score=score)
- logger.info(
- f"[save_score] Saved score | evaluation_id={eval_run_id} | "
- f"traces={len(score.get('traces', []))}"
- )
+ if not eval_run:
+ return None
+
+ traces = score.get("traces", [])
+ summary_score = score.get("summary_scores", [])
+ score_trace_url: str | None = "" if not traces else None
+
+ if traces:
+ try:
+ storage = get_cloud_storage(session=session, project_id=project_id)
+ score_trace_url = upload_jsonl_to_object_store(
+ storage=storage,
+ results=traces,
+ filename=f"traces_{eval_run_id}.json",
+ subdirectory=f"evaluations/score/{eval_run_id}",
+ format="json",
+ )
+ if score_trace_url:
+ logger.info(
+ f"[save_score] uploaded traces to S3 | "
+ f"evaluation_id={eval_run_id} | url={score_trace_url} | "
+ f"traces_count={len(traces)}"
+ )
+ else:
+ logger.warning(
+ f"[save_score] failed to upload traces to S3, "
+ f"falling back to DB storage | evaluation_id={eval_run_id}"
+ )
+ except Exception as e:
+ logger.error(
+ f"[save_score] Error uploading traces to S3: {e} | "
+ f"evaluation_id={eval_run_id}",
+ exc_info=True,
+ )
+
+ # IF TRACES DATA IS STORED IN S3 URL THEN HERE WE ARE JUST STORING THE SUMMARY SCORE
+ # TODO: Evaluate whether this behaviour is needed or completely discard the storing data in db
+ if score_trace_url:
+ db_score = {"summary_scores": summary_score}
+ else:
+ # fallback to store data in db if failed to store in s3
+ db_score = score
+
+ update_evaluation_run(
+ session=session,
+ eval_run=eval_run,
+ score=db_score,
+ score_trace_url=score_trace_url,
+ )
+
+ logger.info(
+ f"[save_score] Saved score | evaluation_id={eval_run_id} | "
+ f"traces={len(score.get('traces', []))}"
+ )
+
  return eval_run
 
 

backend/app/models/evaluation.py

@@ -250,6 +250,13 @@ class EvaluationRun(SQLModel, table=True):
  description="Object store URL of processed evaluation results for future reference",
  sa_column_kwargs={"comment": "S3 URL of processed evaluation results"},
  )
+ score_trace_url: str | None = SQLField(
+ default=None,
+ description="S3 URL per-trace score data is stored",
+ sa_column_kwargs={
+ "comment": "S3 URL where per-trace evaluation scores are stored"
+ },
+ )
  total_items: int = SQLField(
  default=0,
  description="Total number of items evaluated (set during processing)",

backend/app/services/evaluations/evaluation.py

@@ -18,6 +18,8 @@
 from app.models.evaluation import EvaluationRun
 from app.services.llm.providers import LLMProvider
 from app.utils import get_langfuse_client, get_openai_client
+from app.core.cloud.storage import get_cloud_storage
+from app.core.storage_utils import load_json_from_object_store
 
 logger = logging.getLogger(__name__)
 
@@ -189,6 +191,7 @@ def get_evaluation_with_scores(
  Returns:
  Tuple of (EvaluationRun or None, error_message or None)
  """
+
  logger.info(
  f"[get_evaluation_with_scores] Fetching status for evaluation run | "
  f"evaluation_id={evaluation_id} | "
@@ -227,9 +230,41 @@ def get_evaluation_with_scores(
  return eval_run, None
 
  # Check if we already have cached traces
- has_cached_traces = eval_run.score is not None and "traces" in eval_run.score
- if not resync_score and has_cached_traces:
- return eval_run, None
+ has_cached_traces_s3 = eval_run.score_trace_url is not None
+ has_cached_traces_db = eval_run.score is not None and "traces" in eval_run.score
+ if not resync_score:
+ if has_cached_traces_s3:
+ try:
+ storage = get_cloud_storage(session=session, project_id=project_id)
+ traces = load_json_from_object_store(
+ storage=storage, url=eval_run.score_trace_url
+ )
+ if traces is not None:
+ eval_run.score = {
+ "summary_scores": (eval_run.score or {}).get(
+ "summary_scores", []
+ ),
+ "traces": traces,
+ }
+ logger.info(
+ f"[get_evaluation_with_scores] Loaded traces from S3 | "
+ f"evaluation_id={evaluation_id} | "
+ f"traces_count={len(traces)}"
+ )
+ return eval_run, None
+ except Exception as e:
+ logger.error(
+ f"[get_evaluation_with_scores] Error loading traces from S3: {e} | "
+ f"evaluation_id={evaluation_id}",
+ exc_info=True,
+ )
+
+ if has_cached_traces_db:
+ logger.info(
+ f"[get_evaluation_with_scores] Returning traces from DB | "
+ f"evaluation_id={evaluation_id}"
+ )
+ return eval_run, None
 
  langfuse = get_langfuse_client(
  session=session,
@@ -289,4 +324,7 @@ def get_evaluation_with_scores(
  score=score,
  )
 
+ if eval_run:
+ eval_run.score = score
+
  return eval_run, None