fix(traces): support larger execution trace archives

apps/sim/lib/execution/payloads/limits.ts

@@ -1,4 +1,5 @@
 export const MAX_DURABLE_LARGE_VALUE_BYTES = 64 * 1024 * 1024
+export const MAX_TRACE_ARCHIVE_BYTES = 512 * 1024 * 1024
 export const MAX_INLINE_MATERIALIZATION_BYTES = 16 * 1024 * 1024
 export const MAX_FUNCTION_FILE_BYTES = 64 * 1024 * 1024
 export const MAX_FUNCTION_INLINE_BYTES = 10 * 1024 * 1024

apps/sim/lib/execution/payloads/materialization.server.ts

@@ -75,12 +75,15 @@ function getLogger(options: ExecutionMaterializationContext): Logger {
   return options.logger ?? logger
 }
 
-export function assertDurableLargeValueSize(size: number): void {
-  if (size > MAX_DURABLE_LARGE_VALUE_BYTES) {
+export function assertDurableLargeValueSize(
+  size: number,
+  limitBytes = MAX_DURABLE_LARGE_VALUE_BYTES
+): void {
+  if (size > limitBytes) {
     throw new ExecutionResourceLimitError({
       resource: 'execution_payload_bytes',
       attemptedBytes: size,
-      limitBytes: MAX_DURABLE_LARGE_VALUE_BYTES,
+      limitBytes,
     })
   }
 }

apps/sim/lib/execution/payloads/store.test.ts

@@ -8,12 +8,19 @@ import {
   clearLargeValueCacheForTests,
   materializeLargeValueRefSync,
 } from '@/lib/execution/payloads/cache'
-import { MAX_DURABLE_LARGE_VALUE_BYTES } from '@/lib/execution/payloads/limits'
+import {
+  MAX_DURABLE_LARGE_VALUE_BYTES,
+  MAX_TRACE_ARCHIVE_BYTES,
+} from '@/lib/execution/payloads/limits'
 import {
   readLargeValueRefFromStorage,
   readUserFileContent,
 } from '@/lib/execution/payloads/materialization.server'
-import { materializeLargeValueRef, storeLargeValue } from '@/lib/execution/payloads/store'
+import {
+  materializeLargeValueRef,
+  storeExecutionTraceArchive,
+  storeLargeValue,
+} from '@/lib/execution/payloads/store'
 import { EXECUTION_RESOURCE_LIMIT_CODE } from '@/lib/execution/resource-errors'
 
 const {
@@ -350,6 +357,51 @@ describe('large execution payload store', () => {
         requireDurable: true,
       })
     ).rejects.toMatchObject({ code: EXECUTION_RESOURCE_LIMIT_CODE })
+    expect(mockUploadFile).not.toHaveBeenCalled()
+  })
+
+  it('admits a trace archive at its separate size cap with durable ownership', async () => {
+    const ref = await storeExecutionTraceArchive({}, '{}', MAX_TRACE_ARCHIVE_BYTES, {
+      workspaceId: 'workspace-1',
+      workflowId: 'workflow-1',
+      executionId: 'execution-1',
+      userId: 'user-1',
+    })
+
+    expect(mockUploadFile).toHaveBeenCalledOnce()
+    expect(mockRegisterLargeValueOwner).toHaveBeenCalledWith(
+      expect.objectContaining({ key: ref.key, size: MAX_TRACE_ARCHIVE_BYTES }),
+      []
+    )
+    expect(materializeLargeValueRefSync(ref, { executionId: 'execution-1' })).toBeUndefined()
+  })
+
+  it('rejects archives above the trace cap before upload or metadata writes', async () => {
+    await expect(
+      storeExecutionTraceArchive({}, '{}', MAX_TRACE_ARCHIVE_BYTES + 1, {
+        workspaceId: 'workspace-1',
+        workflowId: 'workflow-1',
+        executionId: 'execution-1',
+        userId: 'user-1',
+      })
+    ).rejects.toMatchObject({ code: EXECUTION_RESOURCE_LIMIT_CODE })
+    expect(mockUploadFile).not.toHaveBeenCalled()
+    expect(mockRegisterLargeValueOwner).not.toHaveBeenCalled()
+  })
+
+  it('requires durable storage for trace archives even if the caller disables it', async () => {
+    mockUploadFile.mockRejectedValueOnce(new Error('storage unavailable'))
+
+    await expect(
+      storeExecutionTraceArchive({}, '{}', 2, {
+        workspaceId: 'workspace-1',
+        workflowId: 'workflow-1',
+        executionId: 'execution-1',
+        userId: 'user-1',
+        requireDurable: false,
+      })
+    ).rejects.toThrow('storage unavailable')
+    expect(mockRegisterLargeValueOwner).not.toHaveBeenCalled()
   })
 
   it('bounds explicit server-side materialization', async () => {

apps/sim/lib/execution/payloads/store.ts

@@ -9,6 +9,10 @@ import {
   type LargeValueKind,
   type LargeValueRef,
 } from '@/lib/execution/payloads/large-value-ref'
+import {
+  MAX_DURABLE_LARGE_VALUE_BYTES,
+  MAX_TRACE_ARCHIVE_BYTES,
+} from '@/lib/execution/payloads/limits'
 import {
   assertDurableLargeValueSize,
   assertInlineMaterializationSize,
@@ -164,7 +168,33 @@ export async function storeLargeValue(
   size: number,
   context: LargeValueStoreContext
 ): Promise<LargeValueRef> {
-  assertDurableLargeValueSize(size)
+  return persistLargeValue(value, json, size, context, MAX_DURABLE_LARGE_VALUE_BYTES)
+}
+
+/** Stores a completed execution archive with a larger cap than individual workflow values. */
+export async function storeExecutionTraceArchive(
+  value: Record<string, unknown>,
+  json: string,
+  size: number,
+  context: LargeValueStoreContext
+): Promise<LargeValueRef> {
+  return persistLargeValue(
+    value,
+    json,
+    size,
+    { ...context, requireDurable: true },
+    MAX_TRACE_ARCHIVE_BYTES
+  )
+}
+
+async function persistLargeValue(
+  value: unknown,
+  json: string,
+  size: number,
+  context: LargeValueStoreContext,
+  limitBytes: number
+): Promise<LargeValueRef> {
+  assertDurableLargeValueSize(size, limitBytes)
   const referencedKeys = collectLargeValueKeys(value)
   const id = `lv_${generateShortId(12)}`
   let key = await persistValue(id, json, context)

apps/sim/lib/logs/execution/trace-store-storage.test.ts

@@ -0,0 +1,134 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { clearLargeValueCacheForTests } from '@/lib/execution/payloads/cache'
+import {
+  MAX_DURABLE_LARGE_VALUE_BYTES,
+  MAX_TRACE_ARCHIVE_BYTES,
+} from '@/lib/execution/payloads/limits'
+import { storeLargeValue } from '@/lib/execution/payloads/store'
+import { EXECUTION_RESOURCE_LIMIT_CODE } from '@/lib/execution/resource-errors'
+import {
+  externalizeExecutionData,
+  materializeExecutionData,
+  TRACE_STORE_REF_KEY,
+} from '@/lib/logs/execution/trace-store'
+
+const { mockUploadFile, mockDownloadFile, mockRegisterOwner, mockAddReference } = vi.hoisted(
+  () => ({
+    mockUploadFile: vi.fn(),
+    mockDownloadFile: vi.fn(),
+    mockRegisterOwner: vi.fn(),
+    mockAddReference: vi.fn(),
+  })
+)
+
+/** Scale the two caps down to exercise real serialization and storage reads with small fixtures. */
+vi.mock('@/lib/execution/payloads/limits', async (importOriginal) => ({
+  ...(await importOriginal<typeof import('@/lib/execution/payloads/limits')>()),
+  MAX_DURABLE_LARGE_VALUE_BYTES: 1024,
+  MAX_TRACE_ARCHIVE_BYTES: 4096,
+}))
+
+vi.mock('@/lib/uploads', () => ({
+  StorageService: { uploadFile: mockUploadFile, downloadFile: mockDownloadFile },
+}))
+
+vi.mock('@/lib/execution/payloads/large-value-metadata', () => ({
+  registerLargeValueOwner: mockRegisterOwner,
+  addLargeValueReference: mockAddReference,
+}))
+
+const CONTEXT = {
+  workspaceId: 'workspace-1',
+  workflowId: 'workflow-1',
+  executionId: 'execution-1',
+  userId: 'user-1',
+}
+
+beforeEach(() => {
+  vi.clearAllMocks()
+  clearLargeValueCacheForTests()
+  mockRegisterOwner.mockResolvedValue(true)
+  mockUploadFile.mockImplementation(async ({ customKey, file }) => {
+    mockDownloadFile.mockResolvedValue(file)
+    return { key: customKey }
+  })
+})
+
+describe('trace archive storage round trip', () => {
+  it('uploads an archive above the ordinary value cap and reads it back after a cache miss', async () => {
+    const data = {
+      traceSpans: [{ id: 'span-1', output: 'é'.repeat(MAX_DURABLE_LARGE_VALUE_BYTES) }],
+      traceSpanCount: 1,
+      hasTraceSpans: true,
+    }
+    const json = JSON.stringify(data)
+    const size = Buffer.byteLength(json, 'utf8')
+    expect(size).toBeGreaterThan(MAX_DURABLE_LARGE_VALUE_BYTES)
+    expect(size).toBeLessThan(MAX_TRACE_ARCHIVE_BYTES)
+
+    await expect(storeLargeValue(data, json, size, CONTEXT)).rejects.toMatchObject({
+      code: EXECUTION_RESOURCE_LIMIT_CODE,
+    })
+    expect(mockUploadFile).not.toHaveBeenCalled()
+
+    const slim = await externalizeExecutionData(data, CONTEXT, { throwOnError: true })
+    expect(slim).toEqual({
+      [TRACE_STORE_REF_KEY]: expect.objectContaining({ size, key: expect.any(String) }),
+      traceSpanCount: 1,
+      hasTraceSpans: true,
+    })
+    expect(mockRegisterOwner).toHaveBeenCalledWith(
+      expect.objectContaining({
+        workspaceId: CONTEXT.workspaceId,
+        workflowId: CONTEXT.workflowId,
+        executionId: CONTEXT.executionId,
+        size,
+      }),
+      []
+    )
+    clearLargeValueCacheForTests()
+
+    await expect(materializeExecutionData(slim, CONTEXT)).resolves.toEqual(data)
+    expect(mockDownloadFile).toHaveBeenCalledExactlyOnceWith({
+      key: expect.any(String),
+      context: 'execution',
+      maxBytes: size,
+    })
+    expect(mockAddReference).not.toHaveBeenCalled()
+  })
+
+  it('rejects an over-limit archive before uploading it', async () => {
+    await expect(
+      externalizeExecutionData(
+        { traceSpans: [{ output: 'x'.repeat(MAX_TRACE_ARCHIVE_BYTES) }] },
+        CONTEXT,
+        { throwOnError: true }
+      )
+    ).rejects.toMatchObject({ code: EXECUTION_RESOURCE_LIMIT_CODE })
+    expect(mockUploadFile).not.toHaveBeenCalled()
+    expect(mockRegisterOwner).not.toHaveBeenCalled()
+  })
+
+  it('bounds stored archive reads even if a reference declares a larger size', async () => {
+    const data = { traceSpans: [], hasTraceSpans: false }
+    const slim = await externalizeExecutionData(data, CONTEXT, { throwOnError: true })
+    clearLargeValueCacheForTests()
+
+    await expect(
+      materializeExecutionData(
+        {
+          ...slim,
+          [TRACE_STORE_REF_KEY]: {
+            ...(slim[TRACE_STORE_REF_KEY] as Record<string, unknown>),
+            size: MAX_TRACE_ARCHIVE_BYTES + 1,
+          },
+        },
+        CONTEXT
+      )
+    ).resolves.toEqual({ hasTraceSpans: false })
+    expect(mockDownloadFile).not.toHaveBeenCalled()
+  })
+})

apps/sim/lib/logs/execution/trace-store.test.ts

@@ -3,13 +3,17 @@
  */
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 
-const { decryptSecretMock, materializeLargeValueRefMock, storeLargeValueMock, mockLogger } =
-  vi.hoisted(() => ({
-    decryptSecretMock: vi.fn(),
-    materializeLargeValueRefMock: vi.fn(),
-    storeLargeValueMock: vi.fn(),
-    mockLogger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
-  }))
+const {
+  decryptSecretMock,
+  materializeLargeValueRefMock,
+  storeExecutionTraceArchiveMock,
+  mockLogger,
+} = vi.hoisted(() => ({
+  decryptSecretMock: vi.fn(),
+  materializeLargeValueRefMock: vi.fn(),
+  storeExecutionTraceArchiveMock: vi.fn(),
+  mockLogger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
+}))
 
 vi.mock('@sim/logger', () => ({
   createLogger: () => mockLogger,
@@ -21,7 +25,7 @@ vi.mock('@/lib/core/security/encryption', () => ({
 
 vi.mock('@/lib/execution/payloads/store', () => ({
   materializeLargeValueRef: materializeLargeValueRefMock,
-  storeLargeValue: storeLargeValueMock,
+  storeExecutionTraceArchive: storeExecutionTraceArchiveMock,
 }))
 
 import {
@@ -54,7 +58,7 @@ describe('execution data storage', () => {
   it('propagates the original storage failure for strict backfills', async () => {
     const cause = new Error('column "size_bytes" does not exist')
     const error = new Error('Failed query', { cause })
-    storeLargeValueMock.mockRejectedValueOnce(error)
+    storeExecutionTraceArchiveMock.mockRejectedValueOnce(error)
 
     await expect(
       externalizeExecutionData({ traceSpans: [] }, CONTEXT, { throwOnError: true })
@@ -70,12 +74,12 @@ describe('execution data storage', () => {
         { throwOnError: true }
       )
     ).rejects.toThrow('Trace storage requires workspaceId, workflowId, and userId')
-    expect(storeLargeValueMock).not.toHaveBeenCalled()
+    expect(storeExecutionTraceArchiveMock).not.toHaveBeenCalled()
   })
 
   it('preserves inline completion data and logs the underlying database error', async () => {
     const data = { traceSpans: [] }
-    storeLargeValueMock.mockRejectedValueOnce(
+    storeExecutionTraceArchiveMock.mockRejectedValueOnce(
       new Error('Failed query\nparams: private-payload', {
         cause: new Error('permission denied for table workspace_files'),
       })
@@ -100,7 +104,7 @@ describe('execution data storage', () => {
       executionId: 'execution-1',
       preview: { unsafe: 'must-not-remain-inline' },
     } as const
-    storeLargeValueMock.mockResolvedValue(ref)
+    storeExecutionTraceArchiveMock.mockResolvedValue(ref)
     materializeLargeValueRefMock.mockRejectedValue(new Error('object unavailable'))
 
     const slim = await externalizeExecutionData(

apps/sim/lib/logs/execution/trace-store.ts

@@ -2,7 +2,11 @@ import { createLogger } from '@sim/logger'
 import { describeError, toError } from '@sim/utils/errors'
 import { isRecordLike, omit } from '@sim/utils/object'
 import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref'
-import { materializeLargeValueRef, storeLargeValue } from '@/lib/execution/payloads/store'
+import { MAX_TRACE_ARCHIVE_BYTES } from '@/lib/execution/payloads/limits'
+import {
+  materializeLargeValueRef,
+  storeExecutionTraceArchive,
+} from '@/lib/execution/payloads/store'
 import { FunctionalOutputsUnavailableError } from '@/lib/logs/execution/functional-outputs'
 import { projectTraceSpansForSecrets } from '@/lib/logs/execution/trace-secret-projection'
 import type { TraceSpan } from '@/lib/logs/types'
@@ -254,15 +258,12 @@ export async function externalizeExecutionData(
     const json = JSON.stringify(executionData)
     const size = Buffer.byteLength(json, 'utf8')
 
-    // storeLargeValue persists to the execution bucket with a conforming key and
-    // registers owner + dependency closure (trace -> nested span large values),
-    // so GC keeps nested children alive while this run's log row exists.
-    const ref = await storeLargeValue(executionData, json, size, {
+    /** Register the archive owner and dependencies so nested span values survive with the log. */
+    const ref = await storeExecutionTraceArchive(executionData, json, size, {
       workspaceId,
       workflowId,
       executionId,
       userId,
-      requireDurable: true,
     })
 
     const { preview: _preview, ...slimRef } = ref
@@ -316,7 +317,7 @@ export async function materializeExecutionData(
       workspaceId: context.workspaceId,
       workflowId,
       executionId: context.executionId,
-      maxBytes: ref.size,
+      maxBytes: Math.min(ref.size, MAX_TRACE_ARCHIVE_BYTES),
       // Read-only: the value is already referenced by its own execution; don't
       // re-register (or fail) on every view/export.
       trackReference: false,