fix(kb): disable connectors after repeated sync failures

apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts

@@ -222,6 +222,13 @@ export async function PATCH(request: NextRequest, { params }: RouteParams) {
  }
  if (parsed.data.status !== undefined) {
  updates.status = parsed.data.status
+ if (parsed.data.status === 'active') {
+ updates.consecutiveFailures = 0
+ updates.lastSyncError = null
+ if (updates.nextSyncAt === undefined) {
+ updates.nextSyncAt = new Date()
+ }
+ }
  }
 
  await db

apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx

@@ -5,6 +5,7 @@ import { createLogger } from '@sim/logger'
 import { format, formatDistanceToNow, isPast } from 'date-fns'
 import {
  AlertCircle,
+ AlertTriangle,
  CheckCircle2,
  ChevronDown,
  Loader2,
@@ -66,6 +67,7 @@ const STATUS_CONFIG = {
  syncing: { label: 'Syncing', variant: 'amber' as const },
  error: { label: 'Error', variant: 'red' as const },
  paused: { label: 'Paused', variant: 'gray' as const },
+ disabled: { label: 'Disabled', variant: 'amber' as const },
 } as const
 
 export function ConnectorsSection({
@@ -159,7 +161,10 @@ export function ConnectorsSection({
  knowledgeBaseId,
  connectorId: connector.id,
  updates: {
- status: connector.status === 'paused' ? 'active' : 'paused',
+ status:
+ connector.status === 'paused' || connector.status === 'disabled'
+ ? 'active'
+ : 'paused',
  },
  },
  {
@@ -352,7 +357,12 @@ function ConnectorCard({
  <div className='rounded-lg border border-[var(--border-1)]'>
  <div className='flex items-center justify-between px-3 py-2.5'>
  <div className='flex items-center gap-2.5'>
- {Icon && <Icon className='h-5 w-5 flex-shrink-0' />}
+ <div className='relative flex-shrink-0'>
+ {Icon && <Icon className='h-5 w-5' />}
+ {connector.status === 'disabled' && (
+ <AlertTriangle className='-right-1 -top-1 absolute h-3 w-3 text-amber-500' />
+ )}
+ </div>
  <div className='flex flex-col gap-0.5'>
  <div className='flex items-center gap-2'>
  <span className='flex items-center gap-1.5 font-medium text-[var(--text-primary)] text-small'>
@@ -407,7 +417,12 @@ function ConnectorCard({
  variant='ghost'
  className='h-7 w-7 p-0'
  onClick={onSync}
- disabled={connector.status === 'syncing' || isSyncPending || syncCooldown}
+ disabled={
+ connector.status === 'syncing' ||
+ connector.status === 'disabled' ||
+ isSyncPending ||
+ syncCooldown
+ }
  >
  <RefreshCw
  className={cn(
@@ -441,15 +456,17 @@ function ConnectorCard({
  >
  {isUpdating ? (
  <Loader2 className='h-3.5 w-3.5 animate-spin' />
- ) : connector.status === 'paused' ? (
+ ) : connector.status === 'paused' || connector.status === 'disabled' ? (
  <Play className='h-3.5 w-3.5' />
  ) : (
  <Pause className='h-3.5 w-3.5' />
  )}
  </Button>
  </Tooltip.Trigger>
  <Tooltip.Content>
- {connector.status === 'paused' ? 'Resume' : 'Pause'}
+ {connector.status === 'paused' || connector.status === 'disabled'
+ ? 'Resume'
+ : 'Pause'}
  </Tooltip.Content>
  </Tooltip.Root>
 
@@ -481,7 +498,46 @@ function ConnectorCard({
  </div>
  </div>
 
- {missingScopes.length > 0 && (
+ {connector.status === 'disabled' && (
+ <div className='border-[var(--border-1)] border-t px-3 py-2'>
+ <div className='flex flex-col gap-1 rounded-sm border border-amber-200 bg-amber-50 px-2 py-1.5 dark:border-amber-900 dark:bg-amber-950'>
+ <div className='flex items-center gap-1.5 font-medium text-amber-800 text-caption dark:text-amber-200'>
+ <AlertTriangle className='h-3 w-3 flex-shrink-0' />
+ Connector disabled after repeated sync failures
+ </div>
+ <p className='text-amber-700 text-micro dark:text-amber-300'>
+ Syncing has been paused due to {connector.consecutiveFailures} consecutive failures.
+ {serviceId
+ ? ' Reconnect your account to resume syncing.'
+ : ' Use the resume button to re-enable syncing.'}
+ </p>
+ {canEdit && serviceId && providerId && (
+ <Button
+ variant='active'
+ onClick={() => {
+ if (connector.credentialId) {
+ writeOAuthReturnContext({
+ origin: 'kb-connectors',
+ knowledgeBaseId,
+ displayName: connectorDef?.name ?? connector.connectorType,
+ providerId: providerId!,
+ preCount: credentials?.length ?? 0,
+ workspaceId,
+ requestedAt: Date.now(),
+ })
+ }
+ setShowOAuthModal(true)
+ }}
+ className='w-full px-2 py-1 font-medium text-caption'
+ >
+ Reconnect
+ </Button>
+ )}
+ </div>
+ </div>
+ )}
+
+ {missingScopes.length > 0 && connector.status !== 'disabled' && (
  <div className='border-[var(--border-1)] border-t px-3 py-2'>
  <div className='flex flex-col gap-1 rounded-sm border bg-[var(--surface-2)] px-2 py-1.5'>
  <div className='flex items-center font-medium text-caption'>

apps/sim/hooks/queries/kb/connectors.ts

@@ -12,7 +12,7 @@ export interface ConnectorData {
  sourceConfig: Record<string, unknown>
  syncMode: string
  syncIntervalMinutes: number
- status: 'active' | 'paused' | 'syncing' | 'error'
+ status: 'active' | 'paused' | 'syncing' | 'error' | 'disabled'
  lastSyncAt: string | null
  lastSyncError: string | null
  lastSyncDocCount: number | null

apps/sim/lib/knowledge/connectors/sync-engine.ts

@@ -46,6 +46,7 @@ const MAX_PAGES = 500
 const MAX_SAFE_TITLE_LENGTH = 200
 const STALE_PROCESSING_MINUTES = 45
 const RETRY_WINDOW_DAYS = 7
+const MAX_CONSECUTIVE_FAILURES = 10
 
 /** Sanitizes a document title for use in S3 storage keys. */
 function sanitizeStorageTitle(title: string): string {
@@ -230,7 +231,7 @@ async function resolveAccessToken(
  connector: { credentialId: string | null; encryptedApiKey: string | null },
  connectorConfig: { auth: ConnectorAuthConfig },
  userId: string
-): Promise<string | null> {
+): Promise<string> {
  if (connectorConfig.auth.mode === 'apiKey') {
  if (!connector.encryptedApiKey) {
  throw new Error('API key connector is missing encrypted API key')
@@ -243,11 +244,22 @@ async function resolveAccessToken(
  throw new Error('OAuth connector is missing credential ID')
  }
 
- return refreshAccessTokenIfNeeded(
- connector.credentialId,
- userId,
- `sync-${connector.credentialId}`
- )
+ const requestId = `sync-${connector.credentialId}`
+ const token = await refreshAccessTokenIfNeeded(connector.credentialId, userId, requestId)
+
+ if (!token) {
+ logger.error(`[${requestId}] refreshAccessTokenIfNeeded returned null`, {
+ credentialId: connector.credentialId,
+ userId,
+ authMode: connectorConfig.auth.mode,
+ authProvider: connectorConfig.auth.provider,
+ })
+ throw new Error(
+ `Failed to obtain access token for credential ${connector.credentialId} (provider: ${connectorConfig.auth.provider})`
+ )
+ }
+
+ return token
 }
 
 /**
@@ -305,12 +317,6 @@ export async function executeSync(
  const userId = kbRows[0].userId
  const sourceConfig = connector.sourceConfig as Record<string, unknown>
 
- let accessToken = await resolveAccessToken(connector, connectorConfig, userId)
-
- if (!accessToken) {
- throw new Error('Failed to obtain access token')
- }
-
  const lockResult = await db
  .update(knowledgeConnector)
  .set({ status: 'syncing', updatedAt: new Date() })
@@ -341,6 +347,8 @@ export async function executeSync(
  let syncExitedCleanly = false
 
  try {
+ let accessToken = await resolveAccessToken(connector, connectorConfig, userId)
+
  const externalDocs: ExternalDocument[] = []
  let cursor: string | undefined
  let hasMore = true
@@ -357,8 +365,7 @@ export async function executeSync(
 
  for (let pageNum = 0; hasMore && pageNum < MAX_PAGES; pageNum++) {
  if (pageNum > 0 && connectorConfig.auth.mode === 'oauth') {
- const refreshed = await resolveAccessToken(connector, connectorConfig, userId)
- if (refreshed) accessToken = refreshed
+ accessToken = await resolveAccessToken(connector, connectorConfig, userId)
  }
 
  const page = await connectorConfig.listDocuments(
@@ -496,8 +503,7 @@ export async function executeSync(
 
  if (deferredOps.length > 0) {
  if (connectorConfig.auth.mode === 'oauth') {
- const refreshed = await resolveAccessToken(connector, connectorConfig, userId)
- if (refreshed) accessToken = refreshed
+ accessToken = await resolveAccessToken(connector, connectorConfig, userId)
  }
 
  const hydrated = await Promise.allSettled(
@@ -789,15 +795,25 @@ export async function executeSync(
 
  const now = new Date()
  const failures = (connector.consecutiveFailures ?? 0) + 1
+ const disabled = failures >= MAX_CONSECUTIVE_FAILURES
  const backoffMinutes = Math.min(failures * 30, 1440)
- const nextSync = new Date(now.getTime() + backoffMinutes * 60 * 1000)
+ const nextSync = disabled ? null : new Date(now.getTime() + backoffMinutes * 60 * 1000)
+
+ if (disabled) {
+ logger.warn('Connector disabled after repeated failures', {
+ connectorId,
+ consecutiveFailures: failures,
+ })
+ }
 
  await db
  .update(knowledgeConnector)
  .set({
- status: 'error',
+ status: disabled ? 'disabled' : 'error',
  lastSyncAt: now,
- lastSyncError: errorMessage,
+ lastSyncError: disabled
+ ? 'Connector disabled after repeated sync failures. Please reconnect.'
+ : errorMessage,
  nextSyncAt: nextSync,
  consecutiveFailures: failures,
  updatedAt: now,