diff --git a/.claude/commands/jira.log.md b/.claude/commands/jira.log.md new file mode 100644 index 0000000000..bd49c533e9 --- /dev/null +++ b/.claude/commands/jira.log.md @@ -0,0 +1,273 @@ +--- +description: Log a new Jira issue to RHOAIENG with Team (Ambient team) and Component (Agentic) pre-filled. +--- + +## User Input + +```text +$ARGUMENTS +``` + +You **MUST** consider the user input before proceeding (if not empty). + +## Goal + +Create a new Jira Story in the RHOAIENG project with the correct Team and Component pre-filled for the Ambient team. + +## Execution Steps + +### 1. Parse User Input + +Extract the following from `$ARGUMENTS`: + +- **Summary** (required): The title/summary of the issue +- **Description** (optional): Detailed description of the work +- **Issue Type** (optional): Defaults to `Story`, but can be `Bug` or `Task`. Tasks are tech debt related and not user facing +- **Priority** (optional): Defaults to `Normal` + +If the user provides a simple sentence, use it as the summary. If they provide multiple lines, use the first line as summary and the rest as description. + +### 2. Gather Cold-Start Context + +**IMPORTANT**: To make this Jira actionable by an agent, gather the following context. Ask the user for any missing critical info: + +**Required for Stories:** +- What is the user-facing goal? (As a [user], I want [X], so that [Y]) +- What are the acceptance criteria? (How do we know it's done) +- Which repo/codebase? (e.g., `vTeam`, `ambient-cli`) + +**Required for Bugs:** +- Steps to reproduce +- Expected vs actual behaviour +- Environment/browser info if relevant + +**Helpful for all types:** +- Relevant file paths or components (e.g., `components/frontend/src/...`) +- Related issues/PRs/design docs +- Screenshots or mockups (as links) +- Constraints or out-of-scope items +- Testing requirements + +### 3. Build Structured Description + +Format the description using this **agent-friendly template**: + +```markdown +## Overview +[One paragraph summary of what needs to be done and why] + +## User Story (for Stories) +As a [type of user], I want [goal], so that [benefit]. + +## Acceptance Criteria +- [ ] [Criterion 1] +- [ ] [Criterion 2] +- [ ] [Criterion 3] + +## Technical Context +**Repo**: [repo name or URL] +**Relevant Paths**: +- `path/to/relevant/file.ts` +- `path/to/another/area/` + +## Related Links +- Design: [link if any] +- Related Issues: [RHOAIENG-XXXX] +- PR: [link if any] + +## Constraints +- [What NOT to do] +- [Boundaries to respect] + +## Testing Requirements +- [ ] Unit tests for [X] +- [ ] E2E test for [Y] + +## Bug Details (for Bugs only) +**Steps to Reproduce**: +1. Step 1 +2. Step 2 + +**Expected**: [what should happen] +**Actual**: [what actually happens] +**Environment**: [browser/OS if relevant] +``` + +### 4. Confirm Details + +Before creating the issue, confirm with the user: + +``` +📋 About to create RHOAIENG Jira: + +**Summary**: [extracted summary] +**Type**: Story +**Component**: Agentic +**Team**: Ambient team + +**Description Preview**: +[Show first 500 chars of formatted description] + +This description is structured for agent cold-start. Shall I create this issue? (yes/no/edit) +``` + +### 5. Create the Jira Issue + +Use the `mcp_mcp-atlassian_jira_create_issue` tool with: + +```json +{ + "project_key": "RHOAIENG", + "summary": "[user provided summary]", + "issue_type": "Story", + "description": "[structured description from template]", + "components": "Agentic" +} +``` + +Then **update the issue** to set the Team field (must be done as a separate update call): + +```json +{ + "issue_key": "[CREATED_ISSUE_KEY]", + "fields": { + "customfield_12313240": "6290" + } +} +``` + +### 6. Report Success + +After creation, report: + +``` +✅ Created: [ISSUE_KEY] +🔗 Link: https://issues.redhat.com/browse/[ISSUE_KEY] + +Summary: [summary] +Component: Agentic +Team: Ambient team + +📋 Agent Cold-Start Ready: Yes +``` + +## Examples + +### Quick Story (will prompt for more context) + +``` +/jira.log Add dark mode toggle to session viewer +``` + +The command will then ask you for acceptance criteria, relevant files, etc. + +### Detailed Story (agent-ready) + +``` +/jira.log Add dark mode toggle to session viewer + +As a user, I want to toggle dark mode in the session viewer, so that I can reduce eye strain during long sessions. + +Acceptance: +- Toggle persists across sessions (localStorage) +- Respects system preference by default +- Smooth transition animation + +Repo: vTeam +Files: components/frontend/src/components/session-viewer/ +Related: RHOAIENG-38000 (design system tokens) + +Constraints: +- Use existing Shadcn theme tokens, don't create new colours +- Must work with existing syntax highlighting + +Tests: +- Unit test for toggle logic +- E2E test for persistence +``` + +### Bug Report + +``` +/jira.log [Bug] Session list doesn't refresh after deletion + +Steps: +1. Create a session +2. Delete the session via UI +3. Observe the list + +Expected: Session disappears from list +Actual: Session remains until page refresh + +Repo: vTeam +Files: components/frontend/src/components/session-list/ +Browser: Chrome 120, Firefox 121 + +Fix should invalidate the React Query cache after mutation. +``` + +### Tech Debt Task + +``` +/jira.log [Task] Migrate session queries to use React Query v5 patterns + +Current queries use deprecated `onSuccess` callbacks. +Need to migrate to the new `select` and mutation patterns. + +Repo: vTeam +Files: +- components/frontend/src/services/queries/sessions.ts +- components/frontend/src/hooks/ + +Constraints: +- Don't change API contracts +- Maintain backwards compatibility with existing components + +Tests: +- Existing tests should pass +- Add test for cache invalidation edge case +``` + +## Field Reference + +| Field | Value | Notes | +|-------|-------|-------| +| Project | RHOAIENG | Red Hat OpenShift AI Engineering | +| Component | Agentic | Pre-filled | +| Team | Ambient team | Custom field `customfield_12313240` = `6290` | +| Issue Type | Story | Default, can override with [Bug], [Task] | +| Priority | Normal | Default | + +## Agent Cold-Start Checklist + +For a Jira to be immediately actionable by an agent, ensure: + +| Element | Why It Matters | +|---------|----------------| +| **User Story** | Agent understands the "who" and "why" | +| **Acceptance Criteria** | Clear definition of done, testable outcomes | +| **Repo + File Paths** | Agent knows where to look/edit | +| **Related Links** | Context from design docs, related PRs | +| **Constraints** | Prevents agent from over-engineering or going off-piste | +| **Testing Requirements** | Agent knows what coverage is expected | +| **Bug Repro Steps** | Agent can verify the fix works | + +### What Makes a Good vs Bad Jira for Agents + +**❌ Bad (vague, agent will struggle):** +> "Fix the login bug" + +**✅ Good (agent can start immediately):** +> "Fix login redirect loop on Safari" +> +> **Steps**: 1. Open Safari 2. Click Login 3. Observe infinite redirect +> **Expected**: Redirect to dashboard +> **Actual**: Loops back to login +> **Repo**: vTeam +> **Files**: `components/frontend/src/app/auth/callback/` +> **Constraint**: Don't break Chrome/Firefox flows +> **Test**: Add E2E test for Safari user-agent + +## Context + +$ARGUMENTS diff --git a/components/backend/websocket/agui.go b/components/backend/websocket/agui.go index ce505e1692..2acbc969a5 100644 --- a/components/backend/websocket/agui.go +++ b/components/backend/websocket/agui.go @@ -406,6 +406,16 @@ func streamThreadEvents(c *gin.Context, projectName, sessionName string) { writeSSEEvent(c.Writer, snapshot) c.Writer.(http.Flusher).Flush() } + + // Replay META events from completed runs (feedback, tags, annotations) + // META events are not part of MESSAGES_SNAPSHOT, so replay them separately + for _, event := range completedEvents { + eventType, _ := event["type"].(string) + if eventType == types.EventTypeMeta { + writeSSEEvent(c.Writer, event) + } + } + c.Writer.(http.Flusher).Flush() } } else if err != nil { log.Printf("AGUI: Failed to load events: %v", err) diff --git a/components/backend/websocket/agui_proxy.go b/components/backend/websocket/agui_proxy.go index 8055c26158..e5e6634969 100644 --- a/components/backend/websocket/agui_proxy.go +++ b/components/backend/websocket/agui_proxy.go @@ -586,28 +586,9 @@ func truncateForLog(s string, maxLen int) string { return s[:maxLen] + "..." } -// FeedbackRequest represents the input for submitting user feedback -type FeedbackRequest struct { - // Type of feedback: "thumbs_up" or "thumbs_down" - FeedbackType string `json:"feedbackType" binding:"required,oneof=thumbs_up thumbs_down"` - // Optional message ID being rated - MessageID string `json:"messageId,omitempty"` - // Optional reason for negative feedback - Reason string `json:"reason,omitempty"` - // Optional additional comment - Comment string `json:"comment,omitempty"` - // Optional workflow name - Workflow string `json:"workflow,omitempty"` - // Optional context about what user was working on - Context string `json:"context,omitempty"` - // Whether to include transcript - IncludeTranscript bool `json:"includeTranscript,omitempty"` - // Optional transcript of conversation - Transcript []types.FeedbackTranscriptItem `json:"transcript,omitempty"` -} - -// HandleAGUIFeedback sends user feedback as a META event to the runner +// HandleAGUIFeedback forwards AG-UI META events (user feedback) to the runner // POST /api/projects/:projectName/agentic-sessions/:sessionName/agui/feedback +// Frontend constructs the full META event, backend validates and forwards // See: https://docs.ag-ui.com/drafts/meta-events#user-feedback func HandleAGUIFeedback(c *gin.Context) { // SECURITY: Sanitize URL path params to prevent log injection @@ -643,23 +624,28 @@ func HandleAGUIFeedback(c *gin.Context) { return } - // Extract username from request (forwarded by auth proxy) - // SECURITY: Sanitize to prevent log injection via control characters (newlines, etc.) - username := handlers.SanitizeForLog(c.GetHeader("X-Forwarded-User")) - if username == "" { - username = "unknown" + // Parse AG-UI META event from frontend + // Frontend constructs the full event, we just validate and forward + var metaEvent map[string]interface{} + if err := c.ShouldBindJSON(&metaEvent); err != nil { + log.Printf("AGUI Feedback: Failed to parse META event: %v", err) + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("invalid META event: %v", err)}) + return } - var input FeedbackRequest - if err := c.ShouldBindJSON(&input); err != nil { - log.Printf("AGUI Feedback: Failed to parse input: %v", err) - c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("invalid input: %v", err)}) + // Validate it's a META event + eventType, ok := metaEvent["type"].(string) + if !ok || eventType != types.EventTypeMeta { + log.Printf("AGUI Feedback: Invalid event type: %v", eventType) + c.JSON(http.StatusBadRequest, gin.H{"error": "Expected META event type"}) return } - // SECURITY: Sanitize user input before logging (FeedbackType is validated but sanitize for defense-in-depth) + // Extract metaType for logging + metaType, _ := metaEvent["metaType"].(string) + username := handlers.SanitizeForLog(c.GetHeader("X-Forwarded-User")) log.Printf("AGUI Feedback: Received %s feedback from %s for session %s/%s", - handlers.SanitizeForLog(input.FeedbackType), username, projectName, sessionName) + handlers.SanitizeForLog(metaType), username, projectName, sessionName) // Get runner endpoint runnerURL, err := getRunnerEndpoint(projectName, sessionName) @@ -669,42 +655,7 @@ func HandleAGUIFeedback(c *gin.Context) { return } - // Build AG-UI META event payload - payload := map[string]interface{}{ - "userId": username, - "projectName": projectName, - "sessionName": sessionName, - } - if input.MessageID != "" { - payload["messageId"] = input.MessageID - } - if input.Reason != "" { - payload["reason"] = input.Reason - } - if input.Comment != "" { - payload["comment"] = input.Comment - } - if input.Workflow != "" { - payload["workflow"] = input.Workflow - } - if input.Context != "" { - payload["context"] = input.Context - } - if input.IncludeTranscript && len(input.Transcript) > 0 { - payload["includeTranscript"] = true - payload["transcript"] = input.Transcript - } - - // Create META event following AG-UI spec - metaEvent := map[string]interface{}{ - "type": types.EventTypeMeta, - "metaType": input.FeedbackType, - "payload": payload, - "threadId": sessionName, - "ts": time.Now().UnixMilli(), - } - - // Serialize event for POST to runner + // Serialize event for POST to runner (forward as-is) bodyBytes, err := json.Marshal(metaEvent) if err != nil { log.Printf("AGUI Feedback: Failed to serialize META event: %v", err) @@ -714,7 +665,7 @@ func HandleAGUIFeedback(c *gin.Context) { // POST to runner's feedback endpoint feedbackURL := strings.TrimSuffix(runnerURL, "/") + "/feedback" - log.Printf("AGUI Feedback: Forwarding to runner: %s", feedbackURL) + log.Printf("AGUI Feedback: Forwarding META event to runner: %s", feedbackURL) req, err := http.NewRequest("POST", feedbackURL, bytes.NewReader(bodyBytes)) if err != nil { @@ -744,7 +695,12 @@ func HandleAGUIFeedback(c *gin.Context) { return } - log.Printf("AGUI Feedback: Successfully sent %s feedback to runner", handlers.SanitizeForLog(input.FeedbackType)) + log.Printf("AGUI Feedback: Successfully forwarded %s feedback to runner", handlers.SanitizeForLog(metaType)) + + // Broadcast the META event on the event stream so UI can see feedback submissions + // This allows the frontend to display "Feedback submitted" or track which traces have feedback + broadcastToThread(sessionName, metaEvent) + c.JSON(http.StatusOK, gin.H{ "message": "Feedback submitted successfully", "status": "sent", diff --git a/components/frontend/src/app/projects/[name]/sessions/[sessionName]/page.tsx b/components/frontend/src/app/projects/[name]/sessions/[sessionName]/page.tsx index d86052df65..a80cce0e01 100644 --- a/components/frontend/src/app/projects/[name]/sessions/[sessionName]/page.tsx +++ b/components/frontend/src/app/projects/[name]/sessions/[sessionName]/page.tsx @@ -203,6 +203,9 @@ export default function ProjectSessionDetailPage({ phase === "Running" // Only poll when session is running ); + // Track the current Langfuse trace ID for feedback association + const [langfuseTraceId, setLangfuseTraceId] = useState(null); + // AG-UI streaming hook - replaces useSessionMessages and useSendChatMessage // Note: autoConnect is intentionally false to avoid SSR hydration mismatch // Connection is triggered manually in useEffect after client hydration @@ -211,6 +214,7 @@ export default function ProjectSessionDetailPage({ sessionName: sessionName || "", autoConnect: false, // Manual connection after hydration onError: (err) => console.error("AG-UI stream error:", err), + onTraceId: (traceId) => setLangfuseTraceId(traceId), // Capture Langfuse trace ID for feedback }); const aguiState = aguiStream.state; const aguiSendMessage = aguiStream.sendMessage; @@ -1978,7 +1982,7 @@ export default function ProjectSessionDetailPage({ initialPrompt={session?.spec?.initialPrompt} activeWorkflow={workflowManagement.activeWorkflow || undefined} messages={streamMessages} - traceId={session?.status?.sdkSessionId} + traceId={langfuseTraceId || undefined} > = { + userId: feedbackContext.username, + projectName: feedbackContext.projectName, + sessionName: feedbackContext.sessionName, + }; + + if (feedbackContext.traceId) { + payload.traceId = feedbackContext.traceId; + } + if (comment) { + payload.comment = comment; + } + if (feedbackContext.activeWorkflow) { + payload.workflow = feedbackContext.activeWorkflow; + } + if (contextParts.length > 0) { + payload.context = contextParts.join("; "); + } + if (includeTranscript && transcript && transcript.length > 0) { + payload.includeTranscript = true; + payload.transcript = transcript; + } + + const metaEvent = { + type: "META", + metaType: feedbackType === "positive" ? "thumbs_up" : "thumbs_down", + payload, + threadId: feedbackContext.sessionName, + ts: Date.now(), + }; + + // Send to backend (which forwards to runner and broadcasts on event stream) const feedbackUrl = `/api/projects/${encodeURIComponent(feedbackContext.projectName)}/agentic-sessions/${encodeURIComponent(feedbackContext.sessionName)}/agui/feedback`; const response = await fetch(feedbackUrl, { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - feedbackType: feedbackType === "positive" ? "thumbs_up" : "thumbs_down", - comment: comment || undefined, - workflow: feedbackContext.activeWorkflow || undefined, - context: contextParts.join("; "), - includeTranscript, - transcript, - }), + body: JSON.stringify(metaEvent), }); if (!response.ok) { diff --git a/components/frontend/src/hooks/use-agui-stream.ts b/components/frontend/src/hooks/use-agui-stream.ts index a2152f3ae0..28cb743ad4 100644 --- a/components/frontend/src/hooks/use-agui-stream.ts +++ b/components/frontend/src/hooks/use-agui-stream.ts @@ -41,6 +41,7 @@ type UseAGUIStreamOptions = { onError?: (error: string) => void onConnected?: () => void onDisconnected?: () => void + onTraceId?: (traceId: string) => void // Called when Langfuse trace_id is received } type UseAGUIStreamReturn = { @@ -81,6 +82,7 @@ export function useAGUIStream(options: UseAGUIStreamOptions): UseAGUIStreamRetur onError, onConnected, onDisconnected, + onTraceId, } = options const [state, setState] = useState(initialState) @@ -518,6 +520,13 @@ export function useAGUIStream(options: UseAGUIStreamOptions): UseAGUIStreamRetur return newState } + // Handle Langfuse trace_id for feedback association + if (rawData?.type === 'langfuse_trace' && rawData?.traceId) { + const traceId = rawData.traceId as string + onTraceId?.(traceId) + return newState + } + const actualRawData = rawData // Handle thinking blocks from Claude SDK diff --git a/components/runners/claude-code-runner/adapter.py b/components/runners/claude-code-runner/adapter.py index f48f4f8347..7c9da0a5d8 100644 --- a/components/runners/claude-code-runner/adapter.py +++ b/components/runners/claude-code-runner/adapter.py @@ -600,6 +600,20 @@ def create_sdk_client(opts, disable_continue=False): if isinstance(message, AssistantMessage): current_message = message obs.start_turn(configured_model, user_input=prompt) + + # Emit trace_id for feedback association + # Frontend can use this to link feedback to specific Langfuse traces + trace_id = obs.get_current_trace_id() + if trace_id: + yield RawEvent( + type=EventType.RAW, + thread_id=thread_id, + run_id=run_id, + event={ + "type": "langfuse_trace", + "traceId": trace_id, + } + ) # Process all blocks in the message for block in getattr(message, 'content', []) or []: diff --git a/components/runners/claude-code-runner/main.py b/components/runners/claude-code-runner/main.py index 1fcf9c07ff..88b7acd205 100644 --- a/components/runners/claude-code-runner/main.py +++ b/components/runners/claude-code-runner/main.py @@ -318,6 +318,7 @@ async def handle_feedback(event: FeedbackEvent): project_name = payload.get("projectName", "") session_name = payload.get("sessionName", "") message_id = payload.get("messageId", "") + trace_id = payload.get("traceId", "") # Langfuse trace ID for specific turn association comment = payload.get("comment", "") reason = payload.get("reason", "") workflow = payload.get("workflow", "") @@ -325,8 +326,8 @@ async def handle_feedback(event: FeedbackEvent): include_transcript = payload.get("includeTranscript", False) transcript = payload.get("transcript", []) - # Map metaType to numeric value (1 = positive, 0 = negative) - value = 1 if event.metaType == "thumbs_up" else 0 + # Map metaType to boolean value (True = positive, False = negative) + value = True if event.metaType == "thumbs_up" else False # Build comment string with context comment_parts = [] @@ -363,10 +364,6 @@ async def handle_feedback(event: FeedbackEvent): host=host, ) - # Generate trace ID if not provided - # Use session name as base for trace ID to group feedback with session traces - trace_id = f"feedback-{session_name}-{event.ts or int(__import__('time').time() * 1000)}" - # Build metadata for structured filtering in Langfuse UI metadata = { "project": project_name, @@ -379,11 +376,14 @@ async def handle_feedback(event: FeedbackEvent): if message_id: metadata["messageId"] = message_id - # Send score to Langfuse - langfuse.score( - trace_id=trace_id, + # Create score directly using create_score() API + # Prefer trace_id (specific turn) over session_id (whole session) + # Langfuse expects trace_id OR session_id, not both + langfuse.create_score( name="user-feedback", value=value, + trace_id=trace_id, + data_type="BOOLEAN", comment=feedback_comment, metadata=metadata, ) @@ -391,13 +391,17 @@ async def handle_feedback(event: FeedbackEvent): # Flush immediately to ensure feedback is sent langfuse.flush() - logger.info(f"Langfuse: Feedback score sent (trace_id={trace_id}, value={value})") + # Log success after flush completes + if trace_id: + logger.info(f"Langfuse: Feedback score sent successfully (trace_id={trace_id}, value={value})") + else: + logger.info(f"Langfuse: Feedback score sent successfully (session={session_name}, value={value})") else: logger.warning("Langfuse enabled but missing credentials") except ImportError: logger.warning("Langfuse not available - feedback will not be recorded") except Exception as e: - logger.error(f"Failed to send feedback to Langfuse: {e}") + logger.error(f"Failed to send feedback to Langfuse: {e}", exc_info=True) else: logger.info("Langfuse not enabled - feedback logged but not sent to Langfuse") diff --git a/components/runners/claude-code-runner/observability.py b/components/runners/claude-code-runner/observability.py index 5ff085bc96..b8be5bac1b 100644 --- a/components/runners/claude-code-runner/observability.py +++ b/components/runners/claude-code-runner/observability.py @@ -323,11 +323,26 @@ def start_turn(self, model: str, user_input: str | None = None) -> None: metadata={}, # Turn number will be added in end_turn() ) self._current_turn_generation = self._current_turn_ctx.__enter__() - logging.info(f"Langfuse: Created new trace (model={model})") + logging.info(f"Langfuse: Created new trace (model={model}, trace_id={self.get_current_trace_id()})") except Exception as e: logging.error(f"Langfuse: Failed to start turn: {e}", exc_info=True) + def get_current_trace_id(self) -> str | None: + """Get the current turn's trace ID for feedback association. + + Returns: + The Langfuse trace ID if a turn is active, None otherwise. + """ + if not self._current_turn_generation: + return None + + # The generation object has a trace_id attribute + try: + return getattr(self._current_turn_generation, 'trace_id', None) + except Exception: + return None + def end_turn(self, turn_count: int, message: Any, usage: dict | None = None) -> None: """Complete turn tracking with output and usage data (called when ResultMessage arrives). diff --git a/e2e/scripts/deploy-langfuse.sh b/e2e/scripts/deploy-langfuse.sh index 08e396a4a9..2bfd0e5713 100755 --- a/e2e/scripts/deploy-langfuse.sh +++ b/e2e/scripts/deploy-langfuse.sh @@ -162,7 +162,6 @@ VALUES_FILE="$SCRIPT_DIR/langfuse-values-clickhouse-minimal-logging.yaml" helm upgrade --install langfuse langfuse/langfuse \ --namespace langfuse \ - --version ">= 3.63.0" \ --values "$VALUES_FILE" \ --set langfuse.nextauth.secret.value="$NEXTAUTH_SECRET" \ --set langfuse.salt.value="$SALT" \