From 2834727c133acefbd51280c2f4f7535917d9a5af Mon Sep 17 00:00:00 2001 From: Ondrej Metelka Date: Tue, 8 Sep 2026 11:56:24 +0200 Subject: [PATCH] OLS-3995 add token usage E2E coverage --- test/agent/main.go | 22 ++++++ test/e2e/token_usage_test.go | 130 +++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 test/e2e/token_usage_test.go diff --git a/test/agent/main.go b/test/agent/main.go index 0034344a..a7047be1 100644 --- a/test/agent/main.go +++ b/test/agent/main.go @@ -219,6 +219,24 @@ func newResultCR(step string, tmplRaw []byte) client.Object { } } +// mockTokenUsage returns deterministic values for the operator E2E tests. +// These values model non-zero usage from a sandbox without depending on an LLM. +func mockTokenUsage(step string) agenticv1alpha1.TokenUsage { + var input, output int64 + switch step { + case "analysis": + input, output = 100, 200 + case "execution": + input, output = 110, 210 + case "verification": + input, output = 120, 220 + case "escalation": + input, output = 130, 230 + } + return agenticv1alpha1.TokenUsage{InputTokens: &input, OutputTokens: &output} +} + +// setStatus fills a Result CR with the canned outcome for its workflow step. func setStatus(obj client.Object, targetNS string, verifyFail bool) { now := metav1.Now() completed := []metav1.Condition{{ @@ -230,6 +248,7 @@ func setStatus(obj client.Object, targetNS string, verifyFail bool) { switch cr := obj.(type) { case *agenticv1alpha1.AnalysisResult: + cr.Status.TokenUsage = mockTokenUsage("analysis") cr.Status.Conditions = completed cr.Status.ActionRequired = agenticv1alpha1.ActionRequiredTrue cr.Status.Diagnosis = agenticv1alpha1.DiagnosisResult{ @@ -277,6 +296,7 @@ func setStatus(obj client.Object, targetNS string, verifyFail bool) { }} case *agenticv1alpha1.ExecutionResult: + cr.Status.TokenUsage = mockTokenUsage("execution") cr.Status.Conditions = completed cr.Status.ActionsTaken = []agenticv1alpha1.ExecutionAction{{ Type: "mock", @@ -285,6 +305,7 @@ func setStatus(obj client.Object, targetNS string, verifyFail bool) { }} case *agenticv1alpha1.VerificationResult: + cr.Status.TokenUsage = mockTokenUsage("verification") if verifyFail { // Objective verification failure (OLS-3817): the agent ran and // reports the remediation did not work. Signal it via the @@ -314,6 +335,7 @@ func setStatus(obj client.Object, targetNS string, verifyFail bool) { cr.Status.Summary = "mock verification summary" case *agenticv1alpha1.EscalationResult: + cr.Status.TokenUsage = mockTokenUsage("escalation") cr.Status.Conditions = completed cr.Status.Summary = "mock escalation summary" cr.Status.Content = "mock escalation content" diff --git a/test/e2e/token_usage_test.go b/test/e2e/token_usage_test.go new file mode 100644 index 00000000..5cd9bfdb --- /dev/null +++ b/test/e2e/token_usage_test.go @@ -0,0 +1,130 @@ +//go:build e2e + +package e2e + +import ( + "context" + "os" + "testing" + "time" + + "sigs.k8s.io/controller-runtime/pkg/client" + + agenticv1alpha1 "github.com/openshift/lightspeed-agentic-operator/api/v1alpha1" +) + +// TestTokenUsageFlow_AggregatesResultCRs verifies token usage across the full +// analysis -> execution -> verification workflow. The test intentionally reads +// usage from the Result CRs and compares the sum with AgenticRun.status rather +// than relying only on a hard-coded total. This also runs against each real +// provider in the product E2E pipeline. +func TestTokenUsageFlow_AggregatesResultCRs(t *testing.T) { + t.Log("=== TestTokenUsageFlow_AggregatesResultCRs: validates Result CR token usage and AgenticRun aggregation ===") + c := newClient(t) + ctx := context.Background() + + run := createAgenticRun(t, c, "e2e-token-usage") + t.Logf("AgenticRun created: %s/%s", testNS, run.Name) + + t.Log("Waiting for phase: Proposed (analysis complete)") + proposed := waitForPhase(t, c, run.Name, agenticv1alpha1.AgenticRunPhaseProposed) + runUID := string(proposed.UID) + + t.Log("Approving execution with option 0") + approveExecution(t, c, run.Name, 0) + + t.Log("Waiting for phase: Verifying (execution complete)") + waitForPhase(t, c, run.Name, agenticv1alpha1.AgenticRunPhaseVerifying) + + t.Log("Approving verification") + approveVerification(t, c, run.Name) + + t.Log("Waiting for phase: Completed (verification complete)") + updated := waitForPhase(t, c, run.Name, agenticv1alpha1.AgenticRunPhaseCompleted) + + listCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + var analysisList agenticv1alpha1.AnalysisResultList + if err := c.List(listCtx, &analysisList, client.InNamespace(testNS), client.MatchingLabels{"agentic.openshift.io/run": runUID}); err != nil { + t.Fatalf("list AnalysisResult: %v", err) + } + if len(analysisList.Items) != 1 { + t.Fatalf("expected exactly 1 AnalysisResult, got %d", len(analysisList.Items)) + } + + var executionList agenticv1alpha1.ExecutionResultList + if err := c.List(listCtx, &executionList, client.InNamespace(testNS), client.MatchingLabels{"agentic.openshift.io/run": runUID}); err != nil { + t.Fatalf("list ExecutionResult: %v", err) + } + if len(executionList.Items) != 1 { + t.Fatalf("expected exactly 1 ExecutionResult, got %d", len(executionList.Items)) + } + + var verificationList agenticv1alpha1.VerificationResultList + if err := c.List(listCtx, &verificationList, client.InNamespace(testNS), client.MatchingLabels{"agentic.openshift.io/run": runUID}); err != nil { + t.Fatalf("list VerificationResult: %v", err) + } + if len(verificationList.Items) != 1 { + t.Fatalf("expected exactly 1 VerificationResult, got %d", len(verificationList.Items)) + } + + inputTokens := int64(0) + outputTokens := int64(0) + for _, result := range []struct { + name string + usage agenticv1alpha1.TokenUsage + }{ + {name: "AnalysisResult", usage: analysisList.Items[0].Status.TokenUsage}, + {name: "ExecutionResult", usage: executionList.Items[0].Status.TokenUsage}, + {name: "VerificationResult", usage: verificationList.Items[0].Status.TokenUsage}, + } { + input, output := requireTokenUsage(t, result.name, result.usage) + inputTokens += input + outputTokens += output + } + + if updated.Status.TokenUsage.InputTokens == nil { + t.Fatal("AgenticRun status.tokenUsage.inputTokens is nil") + } + if updated.Status.TokenUsage.OutputTokens == nil { + t.Fatal("AgenticRun status.tokenUsage.outputTokens is nil") + } + if *updated.Status.TokenUsage.InputTokens != inputTokens { + t.Errorf("AgenticRun input token total = %d, want %d", *updated.Status.TokenUsage.InputTokens, inputTokens) + } + if *updated.Status.TokenUsage.OutputTokens != outputTokens { + t.Errorf("AgenticRun output token total = %d, want %d", *updated.Status.TokenUsage.OutputTokens, outputTokens) + } + t.Logf("Verified: AgenticRun token usage totals input=%d output=%d", inputTokens, outputTokens) + + // The mock lane has deterministic values. Keep an exact assertion there so + // a stale or incorrectly configured mock image cannot make this test pass + // with arbitrary non-zero values. Product E2E uses real provider counts and + // therefore only asserts the aggregate relationship above. + if os.Getenv("E2E_PROVIDER") == "" { + if inputTokens != 330 || outputTokens != 630 { + t.Errorf("mock token usage total = input=%d output=%d, want input=330 output=630", inputTokens, outputTokens) + } + } + + t.Log("PASS: all Result CRs contain non-zero token usage and AgenticRun contains their exact aggregate") +} + +// requireTokenUsage verifies that a Result CR reports positive input and output usage. +func requireTokenUsage(t *testing.T, resultName string, usage agenticv1alpha1.TokenUsage) (int64, int64) { + t.Helper() + if usage.InputTokens == nil { + t.Fatalf("%s status.tokenUsage.inputTokens is nil", resultName) + } + if usage.OutputTokens == nil { + t.Fatalf("%s status.tokenUsage.outputTokens is nil", resultName) + } + if *usage.InputTokens <= 0 { + t.Errorf("%s input token count = %d, want > 0", resultName, *usage.InputTokens) + } + if *usage.OutputTokens <= 0 { + t.Errorf("%s output token count = %d, want > 0", resultName, *usage.OutputTokens) + } + return *usage.InputTokens, *usage.OutputTokens +}