From 46a0f5ce588784c4d1f6bdf19338053c3a519b3a Mon Sep 17 00:00:00 2001 From: Gage Krumbach Date: Tue, 20 Jan 2026 17:53:17 -0600 Subject: [PATCH 1/3] Add Jira logging command and enhance AG-UI feedback handling - Introduced a new command for logging Jira issues with pre-filled fields for the Ambient team and Agentic component. - Updated the AG-UI feedback handling to include trace ID for better association of feedback with specific interactions. - Enhanced the feedback submission process to utilize AG-UI META events, improving observability and user experience. - Streamlined the frontend feedback modal to construct and send structured META events, including optional fields like trace ID and context. These changes improve the overall feedback mechanism and facilitate better tracking of user interactions. --- .claude/commands/jira.log.md | 273 ++++++++++++++++++ components/backend/websocket/agui.go | 10 + components/backend/websocket/agui_proxy.go | 96 ++---- .../[name]/sessions/[sessionName]/page.tsx | 6 +- .../src/components/feedback/FeedbackModal.tsx | 45 ++- .../frontend/src/hooks/use-agui-stream.ts | 9 + .../runners/claude-code-runner/adapter.py | 14 + components/runners/claude-code-runner/main.py | 33 ++- .../claude-code-runner/observability.py | 17 +- e2e/scripts/deploy-langfuse.sh | 1 - 10 files changed, 412 insertions(+), 92 deletions(-) create mode 100644 .claude/commands/jira.log.md diff --git a/.claude/commands/jira.log.md b/.claude/commands/jira.log.md new file mode 100644 index 0000000000..bd49c533e9 --- /dev/null +++ b/.claude/commands/jira.log.md @@ -0,0 +1,273 @@ +--- +description: Log a new Jira issue to RHOAIENG with Team (Ambient team) and Component (Agentic) pre-filled. +--- + +## User Input + +```text +$ARGUMENTS +``` + +You **MUST** consider the user input before proceeding (if not empty). + +## Goal + +Create a new Jira Story in the RHOAIENG project with the correct Team and Component pre-filled for the Ambient team. + +## Execution Steps + +### 1. Parse User Input + +Extract the following from `$ARGUMENTS`: + +- **Summary** (required): The title/summary of the issue +- **Description** (optional): Detailed description of the work +- **Issue Type** (optional): Defaults to `Story`, but can be `Bug` or `Task`. Tasks are tech debt related and not user facing +- **Priority** (optional): Defaults to `Normal` + +If the user provides a simple sentence, use it as the summary. If they provide multiple lines, use the first line as summary and the rest as description. + +### 2. Gather Cold-Start Context + +**IMPORTANT**: To make this Jira actionable by an agent, gather the following context. Ask the user for any missing critical info: + +**Required for Stories:** +- What is the user-facing goal? (As a [user], I want [X], so that [Y]) +- What are the acceptance criteria? (How do we know it's done) +- Which repo/codebase? (e.g., `vTeam`, `ambient-cli`) + +**Required for Bugs:** +- Steps to reproduce +- Expected vs actual behaviour +- Environment/browser info if relevant + +**Helpful for all types:** +- Relevant file paths or components (e.g., `components/frontend/src/...`) +- Related issues/PRs/design docs +- Screenshots or mockups (as links) +- Constraints or out-of-scope items +- Testing requirements + +### 3. Build Structured Description + +Format the description using this **agent-friendly template**: + +```markdown +## Overview +[One paragraph summary of what needs to be done and why] + +## User Story (for Stories) +As a [type of user], I want [goal], so that [benefit]. + +## Acceptance Criteria +- [ ] [Criterion 1] +- [ ] [Criterion 2] +- [ ] [Criterion 3] + +## Technical Context +**Repo**: [repo name or URL] +**Relevant Paths**: +- `path/to/relevant/file.ts` +- `path/to/another/area/` + +## Related Links +- Design: [link if any] +- Related Issues: [RHOAIENG-XXXX] +- PR: [link if any] + +## Constraints +- [What NOT to do] +- [Boundaries to respect] + +## Testing Requirements +- [ ] Unit tests for [X] +- [ ] E2E test for [Y] + +## Bug Details (for Bugs only) +**Steps to Reproduce**: +1. Step 1 +2. Step 2 + +**Expected**: [what should happen] +**Actual**: [what actually happens] +**Environment**: [browser/OS if relevant] +``` + +### 4. Confirm Details + +Before creating the issue, confirm with the user: + +``` +📋 About to create RHOAIENG Jira: + +**Summary**: [extracted summary] +**Type**: Story +**Component**: Agentic +**Team**: Ambient team + +**Description Preview**: +[Show first 500 chars of formatted description] + +This description is structured for agent cold-start. Shall I create this issue? (yes/no/edit) +``` + +### 5. Create the Jira Issue + +Use the `mcp_mcp-atlassian_jira_create_issue` tool with: + +```json +{ + "project_key": "RHOAIENG", + "summary": "[user provided summary]", + "issue_type": "Story", + "description": "[structured description from template]", + "components": "Agentic" +} +``` + +Then **update the issue** to set the Team field (must be done as a separate update call): + +```json +{ + "issue_key": "[CREATED_ISSUE_KEY]", + "fields": { + "customfield_12313240": "6290" + } +} +``` + +### 6. Report Success + +After creation, report: + +``` +✅ Created: [ISSUE_KEY] +🔗 Link: https://issues.redhat.com/browse/[ISSUE_KEY] + +Summary: [summary] +Component: Agentic +Team: Ambient team + +📋 Agent Cold-Start Ready: Yes +``` + +## Examples + +### Quick Story (will prompt for more context) + +``` +/jira.log Add dark mode toggle to session viewer +``` + +The command will then ask you for acceptance criteria, relevant files, etc. + +### Detailed Story (agent-ready) + +``` +/jira.log Add dark mode toggle to session viewer + +As a user, I want to toggle dark mode in the session viewer, so that I can reduce eye strain during long sessions. + +Acceptance: +- Toggle persists across sessions (localStorage) +- Respects system preference by default +- Smooth transition animation + +Repo: vTeam +Files: components/frontend/src/components/session-viewer/ +Related: RHOAIENG-38000 (design system tokens) + +Constraints: +- Use existing Shadcn theme tokens, don't create new colours +- Must work with existing syntax highlighting + +Tests: +- Unit test for toggle logic +- E2E test for persistence +``` + +### Bug Report + +``` +/jira.log [Bug] Session list doesn't refresh after deletion + +Steps: +1. Create a session +2. Delete the session via UI +3. Observe the list + +Expected: Session disappears from list +Actual: Session remains until page refresh + +Repo: vTeam +Files: components/frontend/src/components/session-list/ +Browser: Chrome 120, Firefox 121 + +Fix should invalidate the React Query cache after mutation. +``` + +### Tech Debt Task + +``` +/jira.log [Task] Migrate session queries to use React Query v5 patterns + +Current queries use deprecated `onSuccess` callbacks. +Need to migrate to the new `select` and mutation patterns. + +Repo: vTeam +Files: +- components/frontend/src/services/queries/sessions.ts +- components/frontend/src/hooks/ + +Constraints: +- Don't change API contracts +- Maintain backwards compatibility with existing components + +Tests: +- Existing tests should pass +- Add test for cache invalidation edge case +``` + +## Field Reference + +| Field | Value | Notes | +|-------|-------|-------| +| Project | RHOAIENG | Red Hat OpenShift AI Engineering | +| Component | Agentic | Pre-filled | +| Team | Ambient team | Custom field `customfield_12313240` = `6290` | +| Issue Type | Story | Default, can override with [Bug], [Task] | +| Priority | Normal | Default | + +## Agent Cold-Start Checklist + +For a Jira to be immediately actionable by an agent, ensure: + +| Element | Why It Matters | +|---------|----------------| +| **User Story** | Agent understands the "who" and "why" | +| **Acceptance Criteria** | Clear definition of done, testable outcomes | +| **Repo + File Paths** | Agent knows where to look/edit | +| **Related Links** | Context from design docs, related PRs | +| **Constraints** | Prevents agent from over-engineering or going off-piste | +| **Testing Requirements** | Agent knows what coverage is expected | +| **Bug Repro Steps** | Agent can verify the fix works | + +### What Makes a Good vs Bad Jira for Agents + +**❌ Bad (vague, agent will struggle):** +> "Fix the login bug" + +**✅ Good (agent can start immediately):** +> "Fix login redirect loop on Safari" +> +> **Steps**: 1. Open Safari 2. Click Login 3. Observe infinite redirect +> **Expected**: Redirect to dashboard +> **Actual**: Loops back to login +> **Repo**: vTeam +> **Files**: `components/frontend/src/app/auth/callback/` +> **Constraint**: Don't break Chrome/Firefox flows +> **Test**: Add E2E test for Safari user-agent + +## Context + +$ARGUMENTS diff --git a/components/backend/websocket/agui.go b/components/backend/websocket/agui.go index ce505e1692..2acbc969a5 100644 --- a/components/backend/websocket/agui.go +++ b/components/backend/websocket/agui.go @@ -406,6 +406,16 @@ func streamThreadEvents(c *gin.Context, projectName, sessionName string) { writeSSEEvent(c.Writer, snapshot) c.Writer.(http.Flusher).Flush() } + + // Replay META events from completed runs (feedback, tags, annotations) + // META events are not part of MESSAGES_SNAPSHOT, so replay them separately + for _, event := range completedEvents { + eventType, _ := event["type"].(string) + if eventType == types.EventTypeMeta { + writeSSEEvent(c.Writer, event) + } + } + c.Writer.(http.Flusher).Flush() } } else if err != nil { log.Printf("AGUI: Failed to load events: %v", err) diff --git a/components/backend/websocket/agui_proxy.go b/components/backend/websocket/agui_proxy.go index 8055c26158..e5e6634969 100644 --- a/components/backend/websocket/agui_proxy.go +++ b/components/backend/websocket/agui_proxy.go @@ -586,28 +586,9 @@ func truncateForLog(s string, maxLen int) string { return s[:maxLen] + "..." } -// FeedbackRequest represents the input for submitting user feedback -type FeedbackRequest struct { - // Type of feedback: "thumbs_up" or "thumbs_down" - FeedbackType string `json:"feedbackType" binding:"required,oneof=thumbs_up thumbs_down"` - // Optional message ID being rated - MessageID string `json:"messageId,omitempty"` - // Optional reason for negative feedback - Reason string `json:"reason,omitempty"` - // Optional additional comment - Comment string `json:"comment,omitempty"` - // Optional workflow name - Workflow string `json:"workflow,omitempty"` - // Optional context about what user was working on - Context string `json:"context,omitempty"` - // Whether to include transcript - IncludeTranscript bool `json:"includeTranscript,omitempty"` - // Optional transcript of conversation - Transcript []types.FeedbackTranscriptItem `json:"transcript,omitempty"` -} - -// HandleAGUIFeedback sends user feedback as a META event to the runner +// HandleAGUIFeedback forwards AG-UI META events (user feedback) to the runner // POST /api/projects/:projectName/agentic-sessions/:sessionName/agui/feedback +// Frontend constructs the full META event, backend validates and forwards // See: https://docs.ag-ui.com/drafts/meta-events#user-feedback func HandleAGUIFeedback(c *gin.Context) { // SECURITY: Sanitize URL path params to prevent log injection @@ -643,23 +624,28 @@ func HandleAGUIFeedback(c *gin.Context) { return } - // Extract username from request (forwarded by auth proxy) - // SECURITY: Sanitize to prevent log injection via control characters (newlines, etc.) - username := handlers.SanitizeForLog(c.GetHeader("X-Forwarded-User")) - if username == "" { - username = "unknown" + // Parse AG-UI META event from frontend + // Frontend constructs the full event, we just validate and forward + var metaEvent map[string]interface{} + if err := c.ShouldBindJSON(&metaEvent); err != nil { + log.Printf("AGUI Feedback: Failed to parse META event: %v", err) + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("invalid META event: %v", err)}) + return } - var input FeedbackRequest - if err := c.ShouldBindJSON(&input); err != nil { - log.Printf("AGUI Feedback: Failed to parse input: %v", err) - c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("invalid input: %v", err)}) + // Validate it's a META event + eventType, ok := metaEvent["type"].(string) + if !ok || eventType != types.EventTypeMeta { + log.Printf("AGUI Feedback: Invalid event type: %v", eventType) + c.JSON(http.StatusBadRequest, gin.H{"error": "Expected META event type"}) return } - // SECURITY: Sanitize user input before logging (FeedbackType is validated but sanitize for defense-in-depth) + // Extract metaType for logging + metaType, _ := metaEvent["metaType"].(string) + username := handlers.SanitizeForLog(c.GetHeader("X-Forwarded-User")) log.Printf("AGUI Feedback: Received %s feedback from %s for session %s/%s", - handlers.SanitizeForLog(input.FeedbackType), username, projectName, sessionName) + handlers.SanitizeForLog(metaType), username, projectName, sessionName) // Get runner endpoint runnerURL, err := getRunnerEndpoint(projectName, sessionName) @@ -669,42 +655,7 @@ func HandleAGUIFeedback(c *gin.Context) { return } - // Build AG-UI META event payload - payload := map[string]interface{}{ - "userId": username, - "projectName": projectName, - "sessionName": sessionName, - } - if input.MessageID != "" { - payload["messageId"] = input.MessageID - } - if input.Reason != "" { - payload["reason"] = input.Reason - } - if input.Comment != "" { - payload["comment"] = input.Comment - } - if input.Workflow != "" { - payload["workflow"] = input.Workflow - } - if input.Context != "" { - payload["context"] = input.Context - } - if input.IncludeTranscript && len(input.Transcript) > 0 { - payload["includeTranscript"] = true - payload["transcript"] = input.Transcript - } - - // Create META event following AG-UI spec - metaEvent := map[string]interface{}{ - "type": types.EventTypeMeta, - "metaType": input.FeedbackType, - "payload": payload, - "threadId": sessionName, - "ts": time.Now().UnixMilli(), - } - - // Serialize event for POST to runner + // Serialize event for POST to runner (forward as-is) bodyBytes, err := json.Marshal(metaEvent) if err != nil { log.Printf("AGUI Feedback: Failed to serialize META event: %v", err) @@ -714,7 +665,7 @@ func HandleAGUIFeedback(c *gin.Context) { // POST to runner's feedback endpoint feedbackURL := strings.TrimSuffix(runnerURL, "/") + "/feedback" - log.Printf("AGUI Feedback: Forwarding to runner: %s", feedbackURL) + log.Printf("AGUI Feedback: Forwarding META event to runner: %s", feedbackURL) req, err := http.NewRequest("POST", feedbackURL, bytes.NewReader(bodyBytes)) if err != nil { @@ -744,7 +695,12 @@ func HandleAGUIFeedback(c *gin.Context) { return } - log.Printf("AGUI Feedback: Successfully sent %s feedback to runner", handlers.SanitizeForLog(input.FeedbackType)) + log.Printf("AGUI Feedback: Successfully forwarded %s feedback to runner", handlers.SanitizeForLog(metaType)) + + // Broadcast the META event on the event stream so UI can see feedback submissions + // This allows the frontend to display "Feedback submitted" or track which traces have feedback + broadcastToThread(sessionName, metaEvent) + c.JSON(http.StatusOK, gin.H{ "message": "Feedback submitted successfully", "status": "sent", diff --git a/components/frontend/src/app/projects/[name]/sessions/[sessionName]/page.tsx b/components/frontend/src/app/projects/[name]/sessions/[sessionName]/page.tsx index d86052df65..a80cce0e01 100644 --- a/components/frontend/src/app/projects/[name]/sessions/[sessionName]/page.tsx +++ b/components/frontend/src/app/projects/[name]/sessions/[sessionName]/page.tsx @@ -203,6 +203,9 @@ export default function ProjectSessionDetailPage({ phase === "Running" // Only poll when session is running ); + // Track the current Langfuse trace ID for feedback association + const [langfuseTraceId, setLangfuseTraceId] = useState(null); + // AG-UI streaming hook - replaces useSessionMessages and useSendChatMessage // Note: autoConnect is intentionally false to avoid SSR hydration mismatch // Connection is triggered manually in useEffect after client hydration @@ -211,6 +214,7 @@ export default function ProjectSessionDetailPage({ sessionName: sessionName || "", autoConnect: false, // Manual connection after hydration onError: (err) => console.error("AG-UI stream error:", err), + onTraceId: (traceId) => setLangfuseTraceId(traceId), // Capture Langfuse trace ID for feedback }); const aguiState = aguiStream.state; const aguiSendMessage = aguiStream.sendMessage; @@ -1978,7 +1982,7 @@ export default function ProjectSessionDetailPage({ initialPrompt={session?.spec?.initialPrompt} activeWorkflow={workflowManagement.activeWorkflow || undefined} messages={streamMessages} - traceId={session?.status?.sdkSessionId} + traceId={langfuseTraceId || undefined} > = { + userId: feedbackContext.username, + projectName: feedbackContext.projectName, + sessionName: feedbackContext.sessionName, + }; + + if (feedbackContext.traceId) { + payload.traceId = feedbackContext.traceId; + } + if (comment) { + payload.comment = comment; + } + if (feedbackContext.activeWorkflow) { + payload.workflow = feedbackContext.activeWorkflow; + } + if (contextParts.length > 0) { + payload.context = contextParts.join("; "); + } + if (includeTranscript && transcript && transcript.length > 0) { + payload.includeTranscript = true; + payload.transcript = transcript; + } + + const metaEvent = { + type: "META", + metaType: feedbackType === "positive" ? "thumbs_up" : "thumbs_down", + payload, + threadId: feedbackContext.sessionName, + ts: Date.now(), + }; + + // Send to backend (which forwards to runner and broadcasts on event stream) const feedbackUrl = `/api/projects/${encodeURIComponent(feedbackContext.projectName)}/agentic-sessions/${encodeURIComponent(feedbackContext.sessionName)}/agui/feedback`; const response = await fetch(feedbackUrl, { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - feedbackType: feedbackType === "positive" ? "thumbs_up" : "thumbs_down", - comment: comment || undefined, - workflow: feedbackContext.activeWorkflow || undefined, - context: contextParts.join("; "), - includeTranscript, - transcript, - }), + body: JSON.stringify(metaEvent), }); if (!response.ok) { diff --git a/components/frontend/src/hooks/use-agui-stream.ts b/components/frontend/src/hooks/use-agui-stream.ts index a2152f3ae0..28cb743ad4 100644 --- a/components/frontend/src/hooks/use-agui-stream.ts +++ b/components/frontend/src/hooks/use-agui-stream.ts @@ -41,6 +41,7 @@ type UseAGUIStreamOptions = { onError?: (error: string) => void onConnected?: () => void onDisconnected?: () => void + onTraceId?: (traceId: string) => void // Called when Langfuse trace_id is received } type UseAGUIStreamReturn = { @@ -81,6 +82,7 @@ export function useAGUIStream(options: UseAGUIStreamOptions): UseAGUIStreamRetur onError, onConnected, onDisconnected, + onTraceId, } = options const [state, setState] = useState(initialState) @@ -518,6 +520,13 @@ export function useAGUIStream(options: UseAGUIStreamOptions): UseAGUIStreamRetur return newState } + // Handle Langfuse trace_id for feedback association + if (rawData?.type === 'langfuse_trace' && rawData?.traceId) { + const traceId = rawData.traceId as string + onTraceId?.(traceId) + return newState + } + const actualRawData = rawData // Handle thinking blocks from Claude SDK diff --git a/components/runners/claude-code-runner/adapter.py b/components/runners/claude-code-runner/adapter.py index f48f4f8347..7c9da0a5d8 100644 --- a/components/runners/claude-code-runner/adapter.py +++ b/components/runners/claude-code-runner/adapter.py @@ -600,6 +600,20 @@ def create_sdk_client(opts, disable_continue=False): if isinstance(message, AssistantMessage): current_message = message obs.start_turn(configured_model, user_input=prompt) + + # Emit trace_id for feedback association + # Frontend can use this to link feedback to specific Langfuse traces + trace_id = obs.get_current_trace_id() + if trace_id: + yield RawEvent( + type=EventType.RAW, + thread_id=thread_id, + run_id=run_id, + event={ + "type": "langfuse_trace", + "traceId": trace_id, + } + ) # Process all blocks in the message for block in getattr(message, 'content', []) or []: diff --git a/components/runners/claude-code-runner/main.py b/components/runners/claude-code-runner/main.py index 1fcf9c07ff..eb585efd94 100644 --- a/components/runners/claude-code-runner/main.py +++ b/components/runners/claude-code-runner/main.py @@ -318,6 +318,7 @@ async def handle_feedback(event: FeedbackEvent): project_name = payload.get("projectName", "") session_name = payload.get("sessionName", "") message_id = payload.get("messageId", "") + trace_id = payload.get("traceId", "") # Langfuse trace ID for specific turn association comment = payload.get("comment", "") reason = payload.get("reason", "") workflow = payload.get("workflow", "") @@ -363,10 +364,6 @@ async def handle_feedback(event: FeedbackEvent): host=host, ) - # Generate trace ID if not provided - # Use session name as base for trace ID to group feedback with session traces - trace_id = f"feedback-{session_name}-{event.ts or int(__import__('time').time() * 1000)}" - # Build metadata for structured filtering in Langfuse UI metadata = { "project": project_name, @@ -379,25 +376,43 @@ async def handle_feedback(event: FeedbackEvent): if message_id: metadata["messageId"] = message_id - # Send score to Langfuse - langfuse.score( - trace_id=trace_id, + # Create score directly using create_score() API + # Prefer trace_id (specific turn) over session_id (whole session) + # Langfuse expects trace_id OR session_id, not both + # if trace_id: + langfuse.create_score( name="user-feedback", value=value, + trace_id=trace_id, + session_id=session_name, + data_type="NUMERIC", comment=feedback_comment, metadata=metadata, ) + # else: + # langfuse.create_score( + # name="user-feedback", + # value=value, + # session_id=session_name, # Fallback to session-level + # data_type="NUMERIC", + # comment=feedback_comment, + # metadata=metadata, + # ) # Flush immediately to ensure feedback is sent langfuse.flush() - logger.info(f"Langfuse: Feedback score sent (trace_id={trace_id}, value={value})") + # Log success after flush completes + if trace_id: + logger.info(f"Langfuse: Feedback score sent successfully (trace_id={trace_id}, value={value})") + else: + logger.info(f"Langfuse: Feedback score sent successfully (session={session_name}, value={value})") else: logger.warning("Langfuse enabled but missing credentials") except ImportError: logger.warning("Langfuse not available - feedback will not be recorded") except Exception as e: - logger.error(f"Failed to send feedback to Langfuse: {e}") + logger.error(f"Failed to send feedback to Langfuse: {e}", exc_info=True) else: logger.info("Langfuse not enabled - feedback logged but not sent to Langfuse") diff --git a/components/runners/claude-code-runner/observability.py b/components/runners/claude-code-runner/observability.py index 5ff085bc96..b8be5bac1b 100644 --- a/components/runners/claude-code-runner/observability.py +++ b/components/runners/claude-code-runner/observability.py @@ -323,11 +323,26 @@ def start_turn(self, model: str, user_input: str | None = None) -> None: metadata={}, # Turn number will be added in end_turn() ) self._current_turn_generation = self._current_turn_ctx.__enter__() - logging.info(f"Langfuse: Created new trace (model={model})") + logging.info(f"Langfuse: Created new trace (model={model}, trace_id={self.get_current_trace_id()})") except Exception as e: logging.error(f"Langfuse: Failed to start turn: {e}", exc_info=True) + def get_current_trace_id(self) -> str | None: + """Get the current turn's trace ID for feedback association. + + Returns: + The Langfuse trace ID if a turn is active, None otherwise. + """ + if not self._current_turn_generation: + return None + + # The generation object has a trace_id attribute + try: + return getattr(self._current_turn_generation, 'trace_id', None) + except Exception: + return None + def end_turn(self, turn_count: int, message: Any, usage: dict | None = None) -> None: """Complete turn tracking with output and usage data (called when ResultMessage arrives). diff --git a/e2e/scripts/deploy-langfuse.sh b/e2e/scripts/deploy-langfuse.sh index 08e396a4a9..2bfd0e5713 100755 --- a/e2e/scripts/deploy-langfuse.sh +++ b/e2e/scripts/deploy-langfuse.sh @@ -162,7 +162,6 @@ VALUES_FILE="$SCRIPT_DIR/langfuse-values-clickhouse-minimal-logging.yaml" helm upgrade --install langfuse langfuse/langfuse \ --namespace langfuse \ - --version ">= 3.63.0" \ --values "$VALUES_FILE" \ --set langfuse.nextauth.secret.value="$NEXTAUTH_SECRET" \ --set langfuse.salt.value="$SALT" \ From 5cf39c510e1bc70988eb69801a835c871b2d954d Mon Sep 17 00:00:00 2001 From: Gage Krumbach Date: Wed, 21 Jan 2026 11:52:13 -0600 Subject: [PATCH 2/3] Refactor feedback handling to use boolean values - Updated the feedback value mapping from numeric to boolean, improving clarity in the representation of user feedback (thumbs up/down). - Changed the data type for feedback submissions from "NUMERIC" to "BOOLEAN" to align with the new mapping. - Removed obsolete code related to fallback handling for feedback submissions. These changes enhance the feedback processing logic and ensure consistency in data representation. --- components/runners/claude-code-runner/main.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/components/runners/claude-code-runner/main.py b/components/runners/claude-code-runner/main.py index eb585efd94..e13931a426 100644 --- a/components/runners/claude-code-runner/main.py +++ b/components/runners/claude-code-runner/main.py @@ -326,8 +326,8 @@ async def handle_feedback(event: FeedbackEvent): include_transcript = payload.get("includeTranscript", False) transcript = payload.get("transcript", []) - # Map metaType to numeric value (1 = positive, 0 = negative) - value = 1 if event.metaType == "thumbs_up" else 0 + # Map metaType to boolean value (True = positive, False = negative) + value = True if event.metaType == "thumbs_up" else False # Build comment string with context comment_parts = [] @@ -385,19 +385,10 @@ async def handle_feedback(event: FeedbackEvent): value=value, trace_id=trace_id, session_id=session_name, - data_type="NUMERIC", + data_type="BOOLEAN", comment=feedback_comment, metadata=metadata, ) - # else: - # langfuse.create_score( - # name="user-feedback", - # value=value, - # session_id=session_name, # Fallback to session-level - # data_type="NUMERIC", - # comment=feedback_comment, - # metadata=metadata, - # ) # Flush immediately to ensure feedback is sent langfuse.flush() From 71b0ae760c1d9f31e4f73e90344fd2de37c05892 Mon Sep 17 00:00:00 2001 From: Gage Krumbach Date: Wed, 21 Jan 2026 11:59:32 -0600 Subject: [PATCH 3/3] Remove session_id from feedback score creation in Langfuse integration - Eliminated the session_id parameter from the create_score() API call, simplifying the feedback submission process. - This change aligns with the existing logic to prioritize trace_id for user feedback tracking, enhancing clarity and consistency in data handling. These modifications improve the feedback processing workflow and ensure adherence to the updated data requirements. --- components/runners/claude-code-runner/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/runners/claude-code-runner/main.py b/components/runners/claude-code-runner/main.py index e13931a426..88b7acd205 100644 --- a/components/runners/claude-code-runner/main.py +++ b/components/runners/claude-code-runner/main.py @@ -379,12 +379,10 @@ async def handle_feedback(event: FeedbackEvent): # Create score directly using create_score() API # Prefer trace_id (specific turn) over session_id (whole session) # Langfuse expects trace_id OR session_id, not both - # if trace_id: langfuse.create_score( name="user-feedback", value=value, trace_id=trace_id, - session_id=session_name, data_type="BOOLEAN", comment=feedback_comment, metadata=metadata,