fix(tables): route large CSV imports to the background job instead of 413

apps/sim/app/workspace/[workspaceId]/tables/components/import-csv-dialog/import-csv-dialog.tsx

@@ -23,14 +23,26 @@ import {
  TableRow,
  toast,
 } from '@/components/emcn'
+import { CSV_ASYNC_IMPORT_THRESHOLD_BYTES } from '@/lib/table/constants'
 import { buildAutoMapping, parseCsvBuffer } from '@/lib/table/import'
 import type { TableDefinition } from '@/lib/table/types'
-import { type CsvImportMode, useImportCsvIntoTable } from '@/hooks/queries/tables'
+import {
+ type CsvImportMode,
+ cancelTableImport,
+ useImportCsvIntoTable,
+ useImportCsvIntoTableAsync,
+} from '@/hooks/queries/tables'
+import { useImportTrayStore } from '@/stores/table/import-tray/store'
 
 const logger = createLogger('ImportCsvDialog')
 
 const MAX_SAMPLE_ROWS = 5
 const MAX_EXAMPLES_IN_ERROR = 3
+/**
+ * Bytes read for the preview/mapping. We never parse the whole file client-side — the importer
+ * streams it server-side and the DB row-count trigger enforces the row limit.
+ */
+const CSV_PREVIEW_BYTES = 512 * 1024
 /**
  * Sentinel value for the "Do not import" option in the mapping combobox. The
  * whitespace is intentional: valid column names must match `NAME_PATTERN`
@@ -92,7 +104,18 @@ interface ParsedCsv {
  file: File
  headers: string[]
  sampleRows: Record<string, unknown>[]
- totalRows: number
+}
+
+/** Parses the head of a CSV/TSV for the mapping + sample, dropping any truncated final line. */
+async function parseCsvPreview(file: File, delimiter: ',' | '\t') {
+ const sliced = file.size > CSV_PREVIEW_BYTES
+ const blob = sliced ? file.slice(0, CSV_PREVIEW_BYTES) : file
+ let bytes = new Uint8Array(await blob.arrayBuffer())
+ if (sliced) {
+ const lastNewline = bytes.lastIndexOf(0x0a)
+ if (lastNewline > 0) bytes = bytes.subarray(0, lastNewline + 1)
+ }
+ return parseCsvBuffer(bytes, delimiter)
 }
 
 export function ImportCsvDialog({
@@ -110,6 +133,7 @@ export function ImportCsvDialog({
  const [createHeaders, setCreateHeaders] = useState<Set<string>>(new Set())
  const [mode, setMode] = useState<CsvImportMode>('append')
  const importMutation = useImportCsvIntoTable()
+ const importAsyncMutation = useImportCsvIntoTableAsync()
 
  function resetState() {
  setParsed(null)
@@ -155,15 +179,13 @@ export function ImportCsvDialog({
  setParsing(true)
  setParseError(null)
  try {
- const arrayBuffer = await file.arrayBuffer()
- const delimiter = ext === 'tsv' ? '\t' : ','
- const { headers, rows } = await parseCsvBuffer(new Uint8Array(arrayBuffer), delimiter)
+ const delimiter: ',' | '\t' = ext === 'tsv' ? '\t' : ','
+ const { headers, rows } = await parseCsvPreview(file, delimiter)
  const autoMapping = buildAutoMapping(headers, table.schema)
  setParsed({
  file,
  headers,
  sampleRows: rows.slice(0, MAX_SAMPLE_ROWS),
- totalRows: rows.length,
  })
  setMapping(autoMapping)
  } catch (err) {
@@ -256,36 +278,70 @@ export function ImportCsvDialog({
  }
  }, [mapping, parsed?.headers, table.schema.columns, createHeaders])
 
- const appendCapacityDeficit =
- parsed && mode === 'append' && table.rowCount + parsed.totalRows > table.maxRows
- ? table.rowCount + parsed.totalRows - table.maxRows
- : 0
-
- const replaceCapacityDeficit =
- parsed && mode === 'replace' && parsed.totalRows > table.maxRows
- ? parsed.totalRows - table.maxRows
- : 0
-
  const canSubmit =
  parsed !== null &&
  !importMutation.isPending &&
+ !importAsyncMutation.isPending &&
  missingRequired.length === 0 &&
  duplicateTargets.length === 0 &&
- mappedCount + createCount > 0 &&
- appendCapacityDeficit === 0 &&
- replaceCapacityDeficit === 0
+ mappedCount + createCount > 0
 
  async function handleSubmit() {
  if (!parsed || !canSubmit) return
  setSubmitError(null)
+ const createColumns = createHeaders.size > 0 ? [...createHeaders] : undefined
+
+ // Large files can't be POSTed through the server (request-body cap) — upload them
+ // straight to storage and import in the background instead. Seed the header tray and
+ // close the dialog immediately so the indicator is visible during the upload, then run
+ // the upload + kickoff in the background (don't block the dialog on it).
+ if (parsed.file.size >= CSV_ASYNC_IMPORT_THRESHOLD_BYTES) {
+ useImportTrayStore.getState().startUpload({
+ uploadId: table.id,
+ workspaceId,
+ title: parsed.file.name,
+ })
+ onOpenChange(false)
+ toast.success(`Importing "${parsed.file.name}" into "${table.name}" in the background`)
+ importAsyncMutation.mutate(
+ {
+ workspaceId,
+ tableId: table.id,
+ file: parsed.file,
+ mode,
+ mapping,
+ createColumns,
+ onProgress: (percent) => {
+ useImportTrayStore.getState().setUploadPercent(table.id, percent)
+ },
+ },
+ {
+ onSuccess: (data) => {
+ useImportTrayStore.getState().endUpload(table.id)
+ // The server row drives the tray once the list refetches. If canceled mid-upload, flag
+ // the id so it's not shown and cancel the worker server-side.
+ if (useImportTrayStore.getState().consumeCanceled(table.id) && data?.importId) {
+ useImportTrayStore.getState().cancel(table.id)
+ void cancelTableImport(workspaceId, table.id, data.importId).catch(() => {})
+ }
+ },
+ onError: () => {
+ // The hook's onError surfaces the toast; just clear the tray indicator here.
+ useImportTrayStore.getState().endUpload(table.id)
+ },
+ }
+ )
+ return
+ }
+
  try {
  const result = await importMutation.mutateAsync({
  workspaceId,
  tableId: table.id,
  file: parsed.file,
  mode,
  mapping,
- createColumns: createHeaders.size > 0 ? [...createHeaders] : undefined,
+ createColumns,
  })
  const data = result.data
  if (mode === 'append') {
@@ -307,11 +363,7 @@ export function ImportCsvDialog({
  }
  }
 
- const hasWarning =
- missingRequired.length > 0 ||
- duplicateTargets.length > 0 ||
- appendCapacityDeficit > 0 ||
- replaceCapacityDeficit > 0
+ const hasWarning = missingRequired.length > 0 || duplicateTargets.length > 0
 
  return (
  <ChipModal
@@ -344,7 +396,7 @@ export function ImportCsvDialog({
  {parsed.file.name}
  </span>
  <span className='text-[var(--text-tertiary)] text-xs'>
- {parsed.totalRows.toLocaleString()} rows · {parsed.headers.length} columns
+ {parsed.headers.length} columns
  </span>
  </div>
  <Button variant='ghost' size='sm' onClick={resetState}>
@@ -442,20 +494,6 @@ export function ImportCsvDialog({
  Multiple CSV columns target: {duplicateTargets.join(', ')} (pick one)
  </p>
  )}
- {appendCapacityDeficit > 0 && (
- <p className='text-[var(--text-error)] text-caption leading-tight'>
- Append would exceed the row limit ({table.maxRows.toLocaleString()}) by{' '}
- {appendCapacityDeficit.toLocaleString()} row(s). Remove rows or switch to
- Replace.
- </p>
- )}
- {replaceCapacityDeficit > 0 && (
- <p className='text-[var(--text-error)] text-caption leading-tight'>
- CSV has {parsed.totalRows.toLocaleString()} rows, which exceeds the table limit
- of {table.maxRows.toLocaleString()} by {replaceCapacityDeficit.toLocaleString()}
- .
- </p>
- )}
  </div>
  )}
 

apps/sim/app/workspace/[workspaceId]/tables/tables.tsx

@@ -2,12 +2,13 @@
 
 import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
 import { createLogger } from '@sim/logger'
+import { generateId } from '@sim/utils/id'
 import { useParams, useRouter } from 'next/navigation'
 import type { ComboboxOption } from '@/components/emcn'
 import { ChipCombobox, ChipConfirmModal, toast, Upload } from '@/components/emcn'
 import { Columns3, Rows3, Table as TableIcon } from '@/components/emcn/icons'
 import type { TableDefinition } from '@/lib/table'
-import { generateUniqueTableName } from '@/lib/table/constants'
+import { CSV_ASYNC_IMPORT_THRESHOLD_BYTES, generateUniqueTableName } from '@/lib/table/constants'
 import type {
  FilterTag,
  ResourceColumn,
@@ -24,14 +25,17 @@ import {
 import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider'
 import {
  ImportCsvDialog,
+ ImportProgressMenu,
  TablesListContextMenu,
 } from '@/app/workspace/[workspaceId]/tables/components'
 import { TableContextMenu } from '@/app/workspace/[workspaceId]/tables/components/table-context-menu'
 import { useContextMenu } from '@/app/workspace/[workspaceId]/w/components/sidebar/hooks'
 import {
+ cancelTableImport,
  downloadTableExport,
  useCreateTable,
  useDeleteTable,
+ useImportCsvAsync,
  useRenameTable,
  useTablesList,
  useUploadCsvToTable,
@@ -40,6 +44,7 @@ import { useWorkspaceMembersQuery } from '@/hooks/queries/workspace'
 import { useDebounce } from '@/hooks/use-debounce'
 import { useInlineRename } from '@/hooks/use-inline-rename'
 import { usePermissionConfig } from '@/hooks/use-permission-config'
+import { useImportTrayStore } from '@/stores/table/import-tray/store'
 
 const logger = createLogger('Tables')
 
@@ -76,6 +81,7 @@ export function Tables() {
  const renameTable = useRenameTable(workspaceId)
  const createTable = useCreateTable(workspaceId)
  const uploadCsv = useUploadCsvToTable()
+ const importCsvAsync = useImportCsvAsync()
 
  const tableRename = useInlineRename({
  onSave: (tableId, name) => renameTable.mutate({ tableId, name }),
@@ -407,37 +413,80 @@ export function Tables() {
  const list = e.target.files
  if (!list || list.length === 0 || !workspaceId) return
 
- try {
- setUploading(true)
+ const csvFiles = Array.from(list).filter((f) => {
+ const ext = f.name.split('.').pop()?.toLowerCase()
+ return ext === 'csv' || ext === 'tsv'
+ })
+
+ if (csvFiles.length === 0) {
+ toast.error('No CSV or TSV files selected')
+ if (csvInputRef.current) csvInputRef.current.value = ''
+ return
+ }
 
- const csvFiles = Array.from(list).filter((f) => {
- const ext = f.name.split('.').pop()?.toLowerCase()
- return ext === 'csv' || ext === 'tsv'
- })
+ // Large files can't be POSTed through the server (request-body cap) — upload them
+ // straight to storage and import in the background. These are tracked by the import
+ // tray, never the header upload button, so don't touch uploading/uploadProgress here.
+ const asyncFiles = csvFiles.filter((f) => f.size >= CSV_ASYNC_IMPORT_THRESHOLD_BYTES)
+ const syncFiles = csvFiles.filter((f) => f.size < CSV_ASYNC_IMPORT_THRESHOLD_BYTES)
 
- if (csvFiles.length === 0) {
- toast.error('No CSV or TSV files selected')
- return
+ try {
+ for (const file of asyncFiles) {
+ // Show the indicator immediately under a temporary id (the real table id doesn't
+ // exist until kickoff returns), then let the tray track it. Don't redirect — the
+ // table is still empty/importing, so stay on the list.
+ const pendingId = `pending_${generateId()}`
+ useImportTrayStore
+ .getState()
+ .startUpload({ uploadId: pendingId, workspaceId, title: file.name })
+ toast.success(`Importing "${file.name}" in the background`)
+ try {
+ const result = await importCsvAsync.mutateAsync({
+ workspaceId,
+ file,
+ onProgress: (percent) => {
+ useImportTrayStore.getState().setUploadPercent(pendingId, percent)
+ },
+ })
+ useImportTrayStore.getState().endUpload(pendingId)
+ // The server row drives the tray once the list refetches (mutation invalidates it).
+ // If canceled mid-upload, flag the real id so it's not shown and cancel server-side.
+ if (
+ result?.tableId &&
+ result.importId &&
+ useImportTrayStore.getState().consumeCanceled(pendingId)
+ ) {
+ useImportTrayStore.getState().cancel(result.tableId)
+ void cancelTableImport(workspaceId, result.tableId, result.importId).catch(() => {})
+ }
+ } catch {
+ // The hook's onError surfaces the toast; just clear the tray indicator here.
+ useImportTrayStore.getState().endUpload(pendingId)
+ }
  }
 
- setUploadProgress({ completed: 0, total: csvFiles.length })
+ if (syncFiles.length === 0) return
+
+ setUploading(true)
+ setUploadProgress({ completed: 0, total: syncFiles.length })
  const failed: string[] = []
 
- for (let i = 0; i < csvFiles.length; i++) {
+ for (let i = 0; i < syncFiles.length; i++) {
+ const file = syncFiles[i]
  try {
- const result = await uploadCsv.mutateAsync({ workspaceId, file: csvFiles[i] })
+ const result = await uploadCsv.mutateAsync({ workspaceId, file })
 
- if (csvFiles.length === 1) {
+ if (syncFiles.length === 1 && asyncFiles.length === 0) {
  const tableId = result?.data?.table?.id
  if (tableId) {
  router.push(`/workspace/${workspaceId}/tables/${tableId}`)
  }
  }
  } catch (err) {
- failed.push(csvFiles[i].name)
+ failed.push(file.name)
  logger.error('Error uploading CSV:', err)
  } finally {
- setUploadProgress({ completed: i + 1, total: csvFiles.length })
+ setUploadProgress({ completed: i + 1, total: syncFiles.length })
  }
  }
 
@@ -459,7 +508,8 @@ export function Tables() {
  }
  }
  },
- [workspaceId, router, uploadCsv]
+ // eslint-disable-next-line react-hooks/exhaustive-deps -- mutation objects are unstable; mutateAsync is stable in v5
+ [workspaceId, router]
  )
 
  const handleListUploadCsv = useCallback(() => {
@@ -508,6 +558,7 @@ export function Tables() {
  sort={sortConfig}
  filter={filterContent}
  filterTags={filterTags}
+ leadingActions={<ImportProgressMenu workspaceId={workspaceId} />}
  headerActions={[
  {
  label: uploadButtonLabel,