feat(langsmith): add run update, get run, and feedback tools

apps/sim/blocks/blocks/langsmith.ts

@@ -23,6 +23,9 @@ export const LangsmithBlock: BlockConfig<LangsmithResponse> = {
  options: [
  { label: 'Create Run', id: 'langsmith_create_run' },
  { label: 'Create Runs Batch', id: 'langsmith_create_runs_batch' },
+ { label: 'Update Run', id: 'langsmith_update_run' },
+ { label: 'Get Run', id: 'langsmith_get_run' },
+ { label: 'Create Feedback', id: 'langsmith_create_feedback' },
  ],
  value: () => 'langsmith_create_run',
  },
@@ -41,13 +44,27 @@ export const LangsmithBlock: BlockConfig<LangsmithResponse> = {
  placeholder: 'Auto-generated if blank',
  condition: { field: 'operation', value: 'langsmith_create_run' },
  },
+ {
+ id: 'runId',
+ title: 'Run ID',
+ type: 'short-input',
+ placeholder: 'ID of the run to update, retrieve, or attach feedback to',
+ required: {
+ field: 'operation',
+ value: ['langsmith_update_run', 'langsmith_get_run', 'langsmith_create_feedback'],
+ },
+ condition: {
+ field: 'operation',
+ value: ['langsmith_update_run', 'langsmith_get_run', 'langsmith_create_feedback'],
+ },
+ },
  {
  id: 'name',
  title: 'Name',
  type: 'short-input',
  placeholder: 'Run name',
  required: { field: 'operation', value: 'langsmith_create_run' },
- condition: { field: 'operation', value: 'langsmith_create_run' },
+ condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] },
  },
  {
  id: 'run_type',
@@ -78,7 +95,7 @@ export const LangsmithBlock: BlockConfig<LangsmithResponse> = {
  title: 'End Time',
  type: 'short-input',
  placeholder: '2025-01-01T12:00:30Z',
- condition: { field: 'operation', value: 'langsmith_create_run' },
+ condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] },
  mode: 'advanced',
  },
  {
@@ -94,23 +111,23 @@ export const LangsmithBlock: BlockConfig<LangsmithResponse> = {
  title: 'Outputs',
  type: 'code',
  placeholder: '{"output":"value"}',
- condition: { field: 'operation', value: 'langsmith_create_run' },
+ condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] },
  mode: 'advanced',
  },
  {
  id: 'extra',
  title: 'Metadata',
  type: 'code',
  placeholder: '{"ls_model":"gpt-4"}',
- condition: { field: 'operation', value: 'langsmith_create_run' },
+ condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] },
  mode: 'advanced',
  },
  {
  id: 'tags',
  title: 'Tags',
  type: 'code',
  placeholder: '["production","workflow"]',
- condition: { field: 'operation', value: 'langsmith_create_run' },
+ condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] },
  mode: 'advanced',
  },
  {
@@ -150,15 +167,15 @@ export const LangsmithBlock: BlockConfig<LangsmithResponse> = {
  title: 'Status',
  type: 'short-input',
  placeholder: 'success',
- condition: { field: 'operation', value: 'langsmith_create_run' },
+ condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] },
  mode: 'advanced',
  },
  {
  id: 'error',
  title: 'Error',
  type: 'long-input',
  placeholder: 'Error message',
- condition: { field: 'operation', value: 'langsmith_create_run' },
+ condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] },
  mode: 'advanced',
  },
  {
@@ -174,7 +191,7 @@ export const LangsmithBlock: BlockConfig<LangsmithResponse> = {
  title: 'Events',
  type: 'code',
  placeholder: '[{"event":"token","value":1}]',
- condition: { field: 'operation', value: 'langsmith_create_run' },
+ condition: { field: 'operation', value: ['langsmith_create_run', 'langsmith_update_run'] },
  mode: 'advanced',
  },
  {
@@ -207,9 +224,66 @@ Required: id (existing run UUID), name, run_type ("tool"|"chain"|"llm"|"retrieve
 Common patch fields: outputs, end_time, status, error`,
  },
  },
+ {
+ id: 'key',
+ title: 'Feedback Key',
+ type: 'short-input',
+ placeholder: 'e.g. correctness, user_score',
+ required: { field: 'operation', value: 'langsmith_create_feedback' },
+ condition: { field: 'operation', value: 'langsmith_create_feedback' },
+ },
+ {
+ id: 'score',
+ title: 'Score',
+ type: 'short-input',
+ placeholder: 'e.g. 1, 0.5, 0',
+ condition: { field: 'operation', value: 'langsmith_create_feedback' },
+ },
+ {
+ id: 'value',
+ title: 'Value',
+ type: 'short-input',
+ placeholder: 'e.g. good, bad',
+ condition: { field: 'operation', value: 'langsmith_create_feedback' },
+ mode: 'advanced',
+ },
+ {
+ id: 'comment',
+ title: 'Comment',
+ type: 'long-input',
+ placeholder: 'Explanation for the feedback',
+ condition: { field: 'operation', value: 'langsmith_create_feedback' },
+ mode: 'advanced',
+ },
+ {
+ id: 'correction',
+ title: 'Correction',
+ type: 'code',
+ placeholder: '{"output":"the corrected value"}',
+ condition: { field: 'operation', value: 'langsmith_create_feedback' },
+ mode: 'advanced',
+ },
+ {
+ id: 'feedbackSourceType',
+ title: 'Feedback Source',
+ type: 'dropdown',
+ options: [
+ { label: 'API', id: 'api' },
+ { label: 'App', id: 'app' },
+ { label: 'Model', id: 'model' },
+ ],
+ condition: { field: 'operation', value: 'langsmith_create_feedback' },
+ mode: 'advanced',
+ },
  ],
  tools: {
- access: ['langsmith_create_run', 'langsmith_create_runs_batch'],
+ access: [
+ 'langsmith_create_run',
+ 'langsmith_create_runs_batch',
+ 'langsmith_update_run',
+ 'langsmith_get_run',
+ 'langsmith_create_feedback',
+ ],
  config: {
  tool: (params) => params.operation,
  params: (params) => {
@@ -227,6 +301,8 @@ Common patch fields: outputs, end_time, status, error`,
  return value
  }
 
+ const emptyToUndefined = (value: unknown) => (value === '' ? undefined : value)
+
  if (params.operation === 'langsmith_create_runs_batch') {
  const post = parseJsonValue(params.post, 'post runs')
  const patch = parseJsonValue(params.patch, 'patch runs')
@@ -242,6 +318,69 @@ Common patch fields: outputs, end_time, status, error`,
  }
  }
 
+ if (params.operation === 'langsmith_update_run') {
+ const name = emptyToUndefined(params.name)
+ const end_time = emptyToUndefined(params.end_time)
+ const outputs = parseJsonValue(params.outputs, 'outputs')
+ const extra = parseJsonValue(params.extra, 'metadata')
+ const tags = parseJsonValue(params.tags, 'tags')
+ const status = emptyToUndefined(params.status)
+ const error = emptyToUndefined(params.error)
+ const events = parseJsonValue(params.events, 'events')
+
+ if (
+ [name, end_time, outputs, extra, tags, status, error, events].every(
+ (value) => value === undefined
+ )
+ ) {
+ throw new Error('Provide at least one field to update')
+ }
+
+ return {
+ apiKey: params.apiKey,
+ runId: params.runId,
+ name,
+ end_time,
+ outputs,
+ extra,
+ tags,
+ status,
+ error,
+ events,
+ }
+ }
+
+ if (params.operation === 'langsmith_get_run') {
+ return {
+ apiKey: params.apiKey,
+ runId: params.runId,
+ }
+ }
+
+ if (params.operation === 'langsmith_create_feedback') {
+ const parseScore = (value: unknown) => {
+ if (value === undefined || value === null || value === '') {
+ return undefined
+ }
+ const parsed = Number(value)
+ if (Number.isNaN(parsed)) {
+ throw new Error(`Invalid score: "${value}" is not a number`)
+ }
+ return parsed
+ }
+
+ return {
+ apiKey: params.apiKey,
+ runId: params.runId,
+ key: params.key,
+ score: parseScore(params.score),
+ value: params.value,
+ comment: params.comment,
+ correction: parseJsonValue(params.correction, 'correction'),
+ feedbackSourceType: params.feedbackSourceType || undefined,
+ }
+ }
+
  return {
  apiKey: params.apiKey,
  id: params.id,
@@ -269,6 +408,10 @@ Common patch fields: outputs, end_time, status, error`,
  operation: { type: 'string', description: 'Operation to perform' },
  apiKey: { type: 'string', description: 'LangSmith API key' },
  id: { type: 'string', description: 'Run identifier' },
+ runId: {
+ type: 'string',
+ description: 'ID of the run to update, retrieve, or attach feedback to',
+ },
  name: { type: 'string', description: 'Run name' },
  run_type: { type: 'string', description: 'Run type' },
  start_time: { type: 'string', description: 'Run start time (ISO)' },
@@ -287,13 +430,45 @@ Common patch fields: outputs, end_time, status, error`,
  events: { type: 'json', description: 'Events array' },
  post: { type: 'json', description: 'Runs to ingest in batch' },
  patch: { type: 'json', description: 'Runs to update in batch' },
+ key: { type: 'string', description: 'Feedback metric name' },
+ score: { type: 'string', description: 'Numeric score for the feedback metric' },
+ value: { type: 'string', description: 'Categorical value for the feedback metric' },
+ comment: { type: 'string', description: 'Comment explaining the feedback' },
+ correction: { type: 'json', description: 'Corrected output for the run' },
+ feedbackSourceType: {
+ type: 'string',
+ description: 'Origin of the feedback (api, app, or model)',
+ },
  },
  outputs: {
- accepted: { type: 'boolean', description: 'Whether ingestion was accepted' },
- runId: { type: 'string', description: 'Run ID for singlerun' },
+ accepted: { type: 'boolean', description: 'Whether ingestion or the update was accepted' },
+ runId: { type: 'string', description: 'Run ID for single-run operations' },
  runIds: { type: 'array', description: 'Run IDs for batch ingest' },
  message: { type: 'string', description: 'LangSmith response message' },
  messages: { type: 'array', description: 'Per-run response messages' },
+ id: { type: 'string', description: 'Run ID (get run) or feedback ID (create feedback)' },
+ name: { type: 'string', description: 'Run name (get run)' },
+ runType: { type: 'string', description: 'Run type (get run)' },
+ status: { type: 'string', description: 'Run status (get run)' },
+ startTime: { type: 'string', description: 'Run start time (get run)' },
+ endTime: { type: 'string', description: 'Run end time (get run)' },
+ inputs: { type: 'json', description: 'Run inputs payload (get run)' },
+ outputs: { type: 'json', description: 'Run outputs payload (get run)' },
+ error: { type: 'string', description: 'Error details (get run)' },
+ tags: { type: 'array', description: 'Tags attached to the run (get run)' },
+ sessionId: { type: 'string', description: 'Project (session) ID the run belongs to (get run)' },
+ traceId: { type: 'string', description: 'Trace ID (get run)' },
+ parentRunId: { type: 'string', description: 'Parent run ID (get run)' },
+ totalTokens: { type: 'number', description: 'Total tokens consumed by the run (get run)' },
+ totalCost: { type: 'string', description: 'Total cost of the run (get run)' },
+ key: { type: 'string', description: 'Feedback metric name (create feedback)' },
+ score: { type: 'number', description: 'Score recorded for the feedback (create feedback)' },
+ value: {
+ type: 'string',
+ description: 'Categorical value recorded for the feedback (create feedback)',
+ },
+ comment: { type: 'string', description: 'Comment recorded for the feedback (create feedback)' },
+ createdAt: { type: 'string', description: 'When the feedback was created (create feedback)' },
  },
 }
 
@@ -324,11 +499,20 @@ export const LangsmithBlockMeta = {
  icon: LangsmithIcon,
  title: 'LangSmith feedback capture',
  prompt:
- 'Build a workflow that collects user-reported agent failures from a table and forwards each as a tagged LangSmith run with the inputs and expected output for later review.',
+ 'Build a workflow that collects user-reported agent failures from a table and attaches each as scored LangSmith feedback on the originating run for later review.',
  modules: ['tables', 'agent', 'workflows'],
  category: 'engineering',
  tags: ['engineering', 'automation'],
  },
+ {
+ icon: LangsmithIcon,
+ title: 'LangSmith run completion',
+ prompt:
+ 'Build a workflow that creates a LangSmith run when an agent step starts, then updates it with outputs, status, and end time once the step finishes so traces always show the full lifecycle.',
+ modules: ['agent', 'workflows'],
+ category: 'engineering',
+ tags: ['engineering', 'monitoring'],
+ },
  {
  icon: LangsmithIcon,
  title: 'LangSmith batch run shipper',
@@ -381,5 +565,12 @@ export const LangsmithBlockMeta = {
  content:
  '# Batch Export Runs\n\nShip multiple completed runs to LangSmith at once instead of one by one.\n\n## Steps\n1. Collect the runs to export, each with name, type, inputs, outputs, and timing.\n2. Assign a shared project so the runs land together.\n3. Submit them as a single batch.\n\n## Output\nReturn how many runs were exported, the project they landed in, and any runs that failed validation.',
  },
+ {
+ name: 'attach-feedback-to-run',
+ description:
+ 'Attach a score, categorical value, or correction to an existing LangSmith run for evaluation.',
+ content:
+ '# Attach Feedback to a Run\n\nRecord a human or automated judgment on a run that already exists in LangSmith.\n\n## Steps\n1. Identify the run ID the feedback applies to.\n2. Choose a feedback key (e.g. "correctness", "user_score") and a score, value, or comment.\n3. Include a correction if the expected output is known.\n4. Submit the feedback.\n\n## Output\nConfirm the feedback ID and the run it was attached to.',
+ },
  ],
 } as const satisfies BlockMeta