Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
165 changes: 165 additions & 0 deletions apps/sim/lib/logs/execution/trace-store.test.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@ vi.mock('@/lib/execution/payloads/store', () => ({
import {
externalizeExecutionData,
materializeExecutionData,
materializeExecutionDataForDisplayWithBlockOutputs,
projectExecutionDataForDisplay,
RESOLVED_SECRET_PROVENANCE_KEY,
SECRET_PROJECTION_VERSION,
Expand DownExpand Up@@ -92,6 +93,170 @@ describe('execution data storage', () => {
})

describe('projectExecutionDataForDisplay', () => {
it('projects authoritative state-only block outputs without mutating execution state', async () => {
const executionData = {
secretProjectionVersion: SECRET_PROJECTION_VERSION,
traceSpans: [],
executionState: {
resolvedSecretTraceProvenance: {
version: 1 as const,
complete: true,
entries: [{ name: 'OPENAI_API_KEY', encryptedValue: 'ciphertext' }],
scope: { userId: 'user-1', workspaceId: 'workspace-1' },
},
blockStates: {
'function-1': {
output: { token: 12345678, derived: 12345683 },
resolvedSecretTraceProvenance: {
version: 1 as const,
complete: true,
entries: [{ name: 'OPENAI_API_KEY', encryptedValue: 'ciphertext' }],
scope: { userId: 'user-1', workspaceId: 'workspace-1' },
},
},
},
},
}

const materialized = await materializeExecutionDataForDisplayWithBlockOutputs(
executionData,
CONTEXT,
['function-1']
)

expect(materialized.executionData).not.toHaveProperty('executionState')
expect(materialized.blockOutputs).toEqual(
new Map([['function-1', { token: '{{OPENAI_API_KEY}}', derived: 12345683 }]])
)
expect(executionData.executionState.blockStates['function-1'].output).toEqual({
token: 12345678,
derived: 12345683,
})
expect(JSON.stringify(materialized.executionData)).not.toContain('12345678')
expect(JSON.stringify([...materialized.blockOutputs])).not.toContain('12345678')
})

it('does not use trace output for a requested block missing from partial state', async () => {
const emptyProvenance = {
version: 1 as const,
complete: true,
entries: [],
scope: { userId: 'user-1', workspaceId: 'workspace-1' },
}
const materialized = await materializeExecutionDataForDisplayWithBlockOutputs(
{
secretProjectionVersion: SECRET_PROJECTION_VERSION,
traceSpans: [
{
id: 'span-1',
blockId: 'trace-only',
name: 'Trace-only block',
type: 'function',
duration: 1,
startTime: '2026-08-11T00:00:00.000Z',
endTime: '2026-08-11T00:00:00.001Z',
output: { result: 'trace-output' },
},
],
executionState: {
resolvedSecretTraceProvenance: emptyProvenance,
blockStates: {
'state-only': {
output: { result: 'state-output' },
resolvedSecretTraceProvenance: emptyProvenance,
},
},
},
},
CONTEXT,
['state-only', 'trace-only']
)

expect(materialized.blockOutputs).toEqual(new Map([['state-only', { result: 'state-output' }]]))
})

it('does not derive block outputs from legacy trace spans', async () => {
const materialized = await materializeExecutionDataForDisplayWithBlockOutputs(
{
traceSpans: [
{
id: 'span-1',
blockId: 'function-1',
name: 'Function 1',
type: 'function',
duration: 1,
startTime: '2026-08-11T00:00:00.000Z',
endTime: '2026-08-11T00:00:00.001Z',
output: { token: 'raw-legacy-secret' },
},
],
},
CONTEXT,
['function-1']
)

expect(materialized.blockOutputs).toEqual(new Map())
})

it('does not mix legacy trace output into partial execution state', async () => {
const materialized = await materializeExecutionDataForDisplayWithBlockOutputs(
{
traceSpans: [
{
id: 'span-1',
blockId: 'trace-only',
name: 'Trace-only block',
type: 'function',
duration: 1,
startTime: '2026-08-11T00:00:00.000Z',
endTime: '2026-08-11T00:00:00.001Z',
output: { token: 'raw-legacy-secret' },
},
],
executionState: {
blockStates: {
'state-only': { output: { result: 'unproven-state-output' } },
},
},
},
CONTEXT,
['state-only', 'trace-only']
)

expect(materialized.blockOutputs).toEqual(new Map())
expect(JSON.stringify([...materialized.blockOutputs])).not.toContain('raw-legacy-secret')
})

it('omits state-only block outputs that lack usable secret provenance', async () => {
const materialized = await materializeExecutionDataForDisplayWithBlockOutputs(
{
secretProjectionVersion: SECRET_PROJECTION_VERSION,
traceSpans: [
{
id: 'span-1',
blockId: 'function-1',
name: 'Function 1',
type: 'function',
duration: 1,
startTime: '2026-08-11T00:00:00.000Z',
endTime: '2026-08-11T00:00:00.001Z',
output: { token: 'trace-fallback' },
},
],
executionState: {
blockStates: {
'function-1': { output: { token: 'unproven-secret' } },
},
},
},
CONTEXT,
['function-1']
)

expect(materialized.blockOutputs).toEqual(new Map())
expect(JSON.stringify(materialized)).not.toContain('unproven-secret')
})

it('retains run-global projection for legacy rows without exact value sidecars', async () => {
const executionData = {
finalOutput: { result: 12345678, derived: 12345683 },
Expand Down
125 changes: 103 additions & 22 deletions apps/sim/lib/logs/execution/trace-store.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,6 +3,7 @@ import { toError } from '@sim/utils/errors'
import { omit } from '@sim/utils/object'
import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref'
import { materializeLargeValueRef, storeLargeValue } from '@/lib/execution/payloads/store'
import { FunctionalOutputsUnavailableError } from '@/lib/logs/execution/functional-outputs'
import { projectTraceSpansForSecrets } from '@/lib/logs/execution/trace-secret-projection'
import type { TraceSpan } from '@/lib/logs/types'
import {
Expand DownExpand Up@@ -72,6 +73,11 @@ export interface TraceStoreReadContext {
userId?: string
}

export interface DisplayExecutionDataWithBlockOutputs {
executionData: Record<string, unknown>
blockOutputs: Map<string, unknown>
}

/**
* Write-path context. Requires the execution owner's `userId`: the externalized
* object is tracked in `workspace_files`, whose `user_id` column is NOT NULL
Expand DownExpand Up@@ -269,6 +275,100 @@ export async function materializeExecutionDataForDisplay(
return projectExecutionDataForDisplay(materialized, context)
}

/**
* Materializes one trusted row into its display envelope plus secret-safe functional outputs.
* Only requested execution-state outputs are projected and returned; trace spans remain display
* data and the raw execution state never crosses the display boundary.
*/
export async function materializeExecutionDataForDisplayWithBlockOutputs(
executionData: Record<string, unknown> | null | undefined,
context: TraceStoreReadContext,
blockIds: readonly string[]
): Promise<DisplayExecutionDataWithBlockOutputs> {
const materialized = await materializeExecutionData(executionData, context)
const displayData = await projectExecutionDataForDisplay(materialized, context)
if (blockIds.length === 0) {
return { executionData: displayData, blockOutputs: new Map() }
}

const executionState = readRecord(materialized.executionState)
const blockStates = readRecord(executionState?.blockStates)
if (!blockStates) {
if (materialized.executionDataTruncated === true) {
throw new FunctionalOutputsUnavailableError()
}
return { executionData: displayData, blockOutputs: new Map() }
}

const runRegistry = await importResolvedSecretTraceRegistry(
materialized[RESOLVED_SECRET_PROVENANCE_KEY] ??
executionState?.[RESOLVED_SECRET_PROVENANCE_KEY],
'traceStore.blockOutputRunProvenance'
)
const blockOutputs = new Map<string, unknown>()
const projectionStore = createReadOnlyProjectionStore(context)

for (const blockId of new Set(blockIds)) {
const blockState = readRecord(blockStates[blockId])
if (!blockState || blockState.output === undefined) continue
Comment thread
TheodoreSpeaks marked this conversation as resolved.

const hasExactProvenance = Object.hasOwn(blockState, RESOLVED_SECRET_PROVENANCE_KEY)
const registry = hasExactProvenance
? await importResolvedSecretTraceRegistry(
blockState[RESOLVED_SECRET_PROVENANCE_KEY],
'traceStore.blockOutputExactProvenance'
)
: runRegistry
const now = new Date().toISOString()
const [projected] = await projectTraceSpansForSecrets(
[
{
id: `${LOG_DISPLAY_PROJECTION_SPAN_ID}-block-output`,
name: 'Block Output Display Projection',
type: 'display',
duration: 0,
startTime: now,
endTime: now,
output: { value: blockState.output },
},
],
{ registry, allowLargeValueWrites: false, store: projectionStore }
)
if (projected?.output && Object.hasOwn(projected.output, 'value')) {
blockOutputs.set(blockId, projected.output.value)
}
}

return { executionData: displayData, blockOutputs }
}

function readRecord(value: unknown): Record<string, unknown> | undefined {
return value && typeof value === 'object' && !Array.isArray(value)
? (value as Record<string, unknown>)
: undefined
}

async function importResolvedSecretTraceRegistry(
provenance: unknown,
origin: string
): Promise<ResolvedSecretTraceRegistry | undefined> {
if (!isResolvedSecretTraceProvenanceV1(provenance)) return undefined

const registry = new ResolvedSecretTraceRegistry([], provenance.scope)
await registry.importProvenance(provenance, { trusted: true, origin })
return registry
}

function createReadOnlyProjectionStore(context: TraceStoreReadContext) {
return {
workspaceId: context.workspaceId ?? undefined,
workflowId: context.workflowId ?? undefined,
executionId: context.executionId,
userId: context.userId,
trackReference: false,
}
}

/**
* Projects execution-log content with the encrypted provenance saved by the
* trusted executor. Current workflow input and final output values use their
Expand All@@ -284,12 +384,7 @@ export async function projectExecutionDataForDisplay(
executionData: Record<string, unknown>,
context: TraceStoreReadContext
): Promise<Record<string, unknown>> {
const executionState =
executionData.executionState &&
typeof executionData.executionState === 'object' &&
!Array.isArray(executionData.executionState)
? (executionData.executionState as Record<string, unknown>)
: undefined
const executionState = readRecord(executionData.executionState)
const hasTopLevelProvenance = Object.hasOwn(executionData, RESOLVED_SECRET_PROVENANCE_KEY)
const stateProvenance = executionState?.[RESOLVED_SECRET_PROVENANCE_KEY]
const provenance = executionData[RESOLVED_SECRET_PROVENANCE_KEY] ?? stateProvenance
Expand All@@ -302,15 +397,7 @@ export async function projectExecutionDataForDisplay(
return projectLegacyExecutionDataForDisplay(executionData)
}

let registry: ResolvedSecretTraceRegistry | undefined

if (isResolvedSecretTraceProvenanceV1(provenance)) {
registry = new ResolvedSecretTraceRegistry([], provenance.scope)
await registry.importProvenance(provenance, {
trusted: true,
origin: 'traceStore.spanProvenance',
})
}
Comment thread
cursor[bot] marked this conversation as resolved.
const registry = await importResolvedSecretTraceRegistry(provenance, 'traceStore.spanProvenance')

/**
* Compaction drops `executionState`, and with it the only copy of the
Expand DownExpand Up@@ -339,13 +426,7 @@ export async function projectExecutionDataForDisplay(
})
}

const projectionStore = {
workspaceId: context.workspaceId ?? undefined,
workflowId: context.workflowId ?? undefined,
executionId: context.executionId,
userId: context.userId,
trackReference: false,
}
const projectionStore = createReadOnlyProjectionStore(context)

const exactValueProjections = new Map<string, unknown>()
for (const [valueKey, provenanceKey] of Object.entries(EXACT_LOG_VALUE_PROVENANCE_KEYS)) {
Expand Down
Loading
Loading