Evaluation: Use Config Management

backend/app/alembic/versions/041_add_config_in_evals_run_table.py

@@ -0,0 +1,59 @@
+"""add config in evals run table
+
+Revision ID: 041
+Revises: 040
+Create Date: 2025-12-15 14:03:22.082746
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "041"
+down_revision = "040"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ op.add_column(
+ "evaluation_run",
+ sa.Column(
+ "config_id",
+ sa.Uuid(),
+ nullable=True,
+ comment="Reference to the stored config used",
+ ),
+ )
+ op.add_column(
+ "evaluation_run",
+ sa.Column(
+ "config_version",
+ sa.Integer(),
+ nullable=True,
+ comment="Version of the config used",
+ ),
+ )
+ op.create_foreign_key(
+ "fk_evaluation_run_config_id", "evaluation_run", "config", ["config_id"], ["id"]
+ )
+ op.drop_column("evaluation_run", "config")
+
+
+def downgrade():
+ op.add_column(
+ "evaluation_run",
+ sa.Column(
+ "config",
+ postgresql.JSONB(astext_type=sa.Text()),
+ autoincrement=False,
+ nullable=False,
+ comment="Evaluation configuration (model, instructions, etc.)",
+ ),
+ )
+ op.drop_constraint(
+ "fk_evaluation_run_config_id", "evaluation_run", type_="foreignkey"
+ )
+ op.drop_column("evaluation_run", "config_version")
+ op.drop_column("evaluation_run", "config_id")

backend/app/api/routes/evaluations/dataset.py

@@ -48,7 +48,7 @@ def _dataset_to_response(dataset: EvaluationDataset) -> DatasetUploadResponse:
 
 
 @router.post(
- "/",
+ "",
  description=load_description("evaluation/upload_dataset.md"),
  response_model=APIResponse[DatasetUploadResponse],
  dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))],
@@ -87,7 +87,7 @@ async def upload_dataset(
 
 
 @router.get(
- "/",
+ "",
  description=load_description("evaluation/list_datasets.md"),
  response_model=APIResponse[list[DatasetUploadResponse]],
  dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))],

backend/app/api/routes/evaluations/evaluation.py

@@ -1,6 +1,7 @@
 """Evaluation run API routes."""
 
 import logging
+from uuid import UUID
 
 from fastapi import (
  APIRouter,
@@ -29,7 +30,7 @@
 
 
 @router.post(
- "/",
+ "",
  description=load_description("evaluation/create_evaluation.md"),
  response_model=APIResponse[EvaluationRunPublic],
  dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))],
@@ -41,19 +42,16 @@ def evaluate(
  experiment_name: str = Body(
  ..., description="Name for this evaluation experiment/run"
  ),
- config: dict = Body(default_factory=dict, description="Evaluation configuration"),
- assistant_id: str
- | None = Body(
- None, description="Optional assistant ID to fetch configuration from"
- ),
+ config_id: UUID = Body(..., description="Stored config ID"),
+ config_version: int = Body(..., ge=1, description="Stored config version"),
 ) -> APIResponse[EvaluationRunPublic]:
  """Start an evaluation run."""
  eval_run = start_evaluation(
  session=_session,
  dataset_id=dataset_id,
  experiment_name=experiment_name,
-config=config,
-assistant_id=assistant_id,
+config_id=config_id,
+config_version=config_version,
  organization_id=auth_context.organization_.id,
  project_id=auth_context.project_.id,
  )
@@ -68,7 +66,7 @@ def evaluate(
 
 
 @router.get(
- "/",
+ "",
  description=load_description("evaluation/list_evaluations.md"),
  response_model=APIResponse[list[EvaluationRunPublic]],
  dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))],

backend/app/crud/evaluations/__init__.py

@@ -5,6 +5,8 @@
  create_evaluation_run,
  get_evaluation_run_by_id,
  list_evaluation_runs,
+ resolve_evaluation_config,
+ resolve_model_from_config,
  save_score,
 )
 from app.crud.evaluations.cron import (
@@ -43,3 +45,45 @@
  TraceData,
  TraceScore,
 )
+
+__all__ = [
+ # Core
+ "create_evaluation_run",
+ "get_evaluation_run_by_id",
+ "list_evaluation_runs",
+ "resolve_evaluation_config",
+ "resolve_model_from_config",
+ "save_score",
+ # Cron
+ "process_all_pending_evaluations",
+ "process_all_pending_evaluations_sync",
+ # Dataset
+ "create_evaluation_dataset",
+ "delete_dataset",
+ "get_dataset_by_id",
+ "list_datasets",
+ "upload_csv_to_object_store",
+ # Batch
+ "start_evaluation_batch",
+ # Processing
+ "check_and_process_evaluation",
+ "poll_all_pending_evaluations",
+ "process_completed_embedding_batch",
+ "process_completed_evaluation",
+ # Embeddings
+ "calculate_average_similarity",
+ "calculate_cosine_similarity",
+ "start_embedding_batch",
+ # Langfuse
+ "create_langfuse_dataset_run",
+ "fetch_trace_scores_from_langfuse",
+ "update_traces_with_cosine_scores",
+ "upload_dataset_to_langfuse",
+ # Score types
+ "CategoricalSummaryScore",
+ "EvaluationScore",
+ "NumericSummaryScore",
+ "SummaryScore",
+ "TraceData",
+ "TraceScore",
+]

backend/app/crud/evaluations/batch.py

@@ -16,6 +16,7 @@
 
 from app.core.batch import OpenAIBatchProvider, start_batch_job
 from app.models import EvaluationRun
+from app.models.llm.request import KaapiLLMParams
 
 logger = logging.getLogger(__name__)
 
@@ -59,7 +60,7 @@ def fetch_dataset_items(langfuse: Langfuse, dataset_name: str) -> list[dict[str,
 
 
 def build_evaluation_jsonl(
- dataset_items: list[dict[str, Any]], config: dict[str, Any]
+ dataset_items: list[dict[str, Any]], config: KaapiLLMParams
 ) -> list[dict[str, Any]]:
  """
  Build JSONL data for evaluation batch using OpenAI Responses API.
@@ -88,7 +89,6 @@ def build_evaluation_jsonl(
  List of dictionaries (JSONL data)
  """
  jsonl_data = []
-
  for item in dataset_items:
  # Extract question from input
  question = item["input"].get("question", "")
@@ -100,14 +100,34 @@ def build_evaluation_jsonl(
 
  # Build the batch request object for Responses API
  # Use config as-is and only add the input field
+ body: dict[str, Any] = {
+ "model": config.model,
+ "instructions": config.instructions,
+ "temperature": config.temperature
+ if config.temperature is not None
+ else 0.01,
+ "input": question, # Add input from dataset
+ }
+
+ # Add reasoning only if provided
+ if config.reasoning:
+ body["reasoning"] = {"effort": config.reasoning}
+
+ # Add tools only if knowledge_base_ids are provided
+ if config.knowledge_base_ids:
+ body["tools"] = [
+ {
+ "type": "file_search",
+ "vector_store_ids": config.knowledge_base_ids,
+ "max_num_results": config.max_num_results or 20,
+ }
+ ]
+
  batch_request = {
  "custom_id": item["id"],
  "method": "POST",
  "url": "/v1/responses",
- "body": {
- **config, # Use config as-is
- "input": question, # Add input from dataset
- },
+ "body": body,
  }
 
  jsonl_data.append(batch_request)
@@ -119,7 +139,7 @@ def start_evaluation_batch(
  openai_client: OpenAI,
  session: Session,
  eval_run: EvaluationRun,
- config: dict[str, Any],
+ config: KaapiLLMParams,
 ) -> EvaluationRun:
  """
  Fetch data, build JSONL, and start evaluation batch.
@@ -132,7 +152,7 @@ def start_evaluation_batch(
  openai_client: Configured OpenAI client
  session: Database session
  eval_run: EvaluationRun database object (with run_name, dataset_name, config)
- config: Evaluation configuration dict with llm, instructions, vector_store_ids
+ config: KaapiLLMParams with model, instructions, knowledge_base_ids, etc.
 
  Returns:
  Updated EvaluationRun with batch_job_id populated
@@ -166,7 +186,7 @@ def start_evaluation_batch(
  "description": f"Evaluation: {eval_run.run_name}",
  "completion_window": "24h",
  # Store complete config for reference
- "evaluation_config": config,
+ "evaluation_config": config.model_dump(exclude_none=True),
  }
 
  # Step 5: Start batch job using generic infrastructure