fix(brightdata): fix async Discover API, echo-back fields, and registry ordering

apps/docs/content/docs/en/tools/agiloft.mdx

@@ -7,7 +7,7 @@ import { BlockInfoCard } from "@/components/ui/block-info-card"
 
 <BlockInfoCard 
  type="agiloft"
- color="#263A5C"
+ color="#FFFFFF"
 />
 
 {/* MANUAL-CONTENT-START:intro */}

apps/sim/blocks/registry.ts

@@ -245,9 +245,9 @@ export const registry: Record<string, BlockConfig> = {
  ashby: AshbyBlock,
  athena: AthenaBlock,
  attio: AttioBlock,
+ box: BoxBlock,
  brandfetch: BrandfetchBlock,
  brightdata: BrightDataBlock,
- box: BoxBlock,
  browser_use: BrowserUseBlock,
  calcom: CalComBlock,
  calendly: CalendlyBlock,

apps/sim/tools/brightdata/cancel_snapshot.ts

@@ -38,17 +38,17 @@ export const brightDataCancelSnapshotTool: ToolConfig<
  }),
  },
 
- transformResponse: async (response: Response) => {
+ transformResponse: async (response: Response, params) => {
  if (!response.ok) {
  const errorText = await response.text()
  throw new Error(errorText || `Cancel snapshot failed with status ${response.status}`)
  }
 
-const data = (await response.json().catch(() => null)) as Record<string, unknown> | null
+ await response.json().catch(() => null)
  return {
  success: true,
  output: {
- snapshotId: (data?.snapshot_id as string) ?? null,
+ snapshotId: params?.snapshotId ?? null,
  cancelled: true,
  },
  }

apps/sim/tools/brightdata/discover.ts

@@ -1,6 +1,13 @@
+import { createLogger } from '@sim/logger'
+import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits'
 import type { BrightDataDiscoverParams, BrightDataDiscoverResponse } from '@/tools/brightdata/types'
 import type { ToolConfig } from '@/tools/types'
 
+const logger = createLogger('tools:brightdata:discover')
+
+const POLL_INTERVAL_MS = 3000
+const MAX_POLL_TIME_MS = DEFAULT_EXECUTION_TIMEOUT_MS
+
 export const brightDataDiscoverTool: ToolConfig<
  BrightDataDiscoverParams,
  BrightDataDiscoverResponse
@@ -84,45 +91,120 @@ export const brightDataDiscoverTool: ToolConfig<
  },
  },
 
- transformResponse: async (response: Response) => {
+ transformResponse: async (response: Response, params) => {
  if (!response.ok) {
  const errorText = await response.text()
  throw new Error(errorText || `Discover request failed with status ${response.status}`)
  }
 
  const data = await response.json()
 
- let results: Array<{
- url: string | null
- title: string | null
- description: string | null
- relevanceScore: number | null
- content: string | null
- }> = []
-
- const items = Array.isArray(data) ? data : (data?.results ?? data?.data ?? [])
-
- if (Array.isArray(items)) {
- results = items.map((item: Record<string, unknown>) => ({
- url: (item.link as string) ?? (item.url as string) ?? null,
- title: (item.title as string) ?? null,
- description: (item.description as string) ?? (item.snippet as string) ?? null,
- relevanceScore: (item.relevance_score as number) ?? null,
- content:
- (item.content as string) ?? (item.text as string) ?? (item.markdown as string) ?? null,
- }))
- }
-
  return {
  success: true,
  output: {
- results,
- query: null,
- totalResults: results.length,
+ results: [],
+ query: params?.query ?? null,
+ totalResults: 0,
+ taskId: data.task_id ?? null,
  },
  }
  },
 
+ postProcess: async (result, params) => {
+ if (!result.success) return result
+
+ const taskId = result.output.taskId
+ if (!taskId) {
+ return {
+ ...result,
+ success: false,
+ error: 'Discover API did not return a task_id. Cannot poll for results.',
+ }
+ }
+
+ logger.info(`Bright Data Discover task ${taskId} created, polling for results...`)
+
+ let elapsedTime = 0
+
+ while (elapsedTime < MAX_POLL_TIME_MS) {
+ try {
+ const pollResponse = await fetch(
+ `https://api.brightdata.com/discover?task_id=${encodeURIComponent(taskId)}`,
+ {
+ method: 'GET',
+ headers: {
+ Authorization: `Bearer ${params.apiKey}`,
+ },
+ }
+ )
+
+ if (!pollResponse.ok) {
+ return {
+ ...result,
+ success: false,
+ error: `Failed to poll discover results: ${pollResponse.statusText}`,
+ }
+ }
+
+ const data = await pollResponse.json()
+ logger.info(`Bright Data Discover task ${taskId} status: ${data.status}`)
+
+ if (data.status === 'done') {
+ const items = Array.isArray(data.results) ? data.results : []
+
+ const results = items.map((item: Record<string, unknown>) => ({
+ url: (item.link as string) ?? (item.url as string) ?? null,
+ title: (item.title as string) ?? null,
+ description: (item.description as string) ?? (item.snippet as string) ?? null,
+ relevanceScore: (item.relevance_score as number) ?? null,
+ content: (item.content as string) ?? null,
+ }))
+
+ return {
+ success: true,
+ output: {
+ results,
+ query: params.query ?? null,
+ totalResults: results.length,
+ },
+ }
+ }
+
+ if (data.status === 'failed' || data.status === 'error') {
+ return {
+ ...result,
+ success: false,
+ error: `Discover task failed: ${data.error ?? 'Unknown error'}`,
+ }
+ }
+
+ await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS))
+ elapsedTime += POLL_INTERVAL_MS
+ } catch (error) {
+ logger.error('Error polling for discover task:', {
+ message: error instanceof Error ? error.message : String(error),
+ taskId,
+ })
+
+ return {
+ ...result,
+ success: false,
+ error: `Error polling for discover task: ${error instanceof Error ? error.message : String(error)}`,
+ }
+ }
+ }
+
+ logger.warn(
+ `Discover task ${taskId} did not complete within the maximum polling time (${MAX_POLL_TIME_MS / 1000}s)`
+ )
+
+ return {
+ ...result,
+ success: false,
+ error: `Discover task ${taskId} timed out after ${MAX_POLL_TIME_MS / 1000}s. Check status manually.`,
+ }
+ },
+
  outputs: {
  results: {
  type: 'array',

apps/sim/tools/brightdata/download_snapshot.ts

@@ -56,7 +56,7 @@ export const brightDataDownloadSnapshotTool: ToolConfig<
  }),
  },
 
- transformResponse: async (response: Response) => {
+ transformResponse: async (response: Response, params) => {
  if (response.status === 409) {
  throw new Error(
  'Snapshot is not ready for download. Check the snapshot status first and wait until it is "ready".'
@@ -89,7 +89,7 @@ export const brightDataDownloadSnapshotTool: ToolConfig<
  output: {
  data,
  format: contentType,
- snapshotId: (data[0]?.snapshot_id as string) ?? null,
+ snapshotId: params?.snapshotId ?? null,
  },
  }
  },

apps/sim/tools/brightdata/scrape_url.ts

@@ -66,7 +66,7 @@ export const brightDataScrapeUrlTool: ToolConfig<
  },
  },
 
- transformResponse: async (response: Response) => {
+ transformResponse: async (response: Response, params) => {
  const contentType = response.headers.get('content-type') || ''
 
  if (!response.ok) {
@@ -86,7 +86,7 @@ export const brightDataScrapeUrlTool: ToolConfig<
  success: true,
  output: {
  content,
- url: null,
+ url: params?.url ?? null,
  statusCode: response.status,
  },
  }

apps/sim/tools/brightdata/serp_search.ts

@@ -129,7 +129,7 @@ export const brightDataSerpSearchTool: ToolConfig<
  },
  },
 
- transformResponse: async (response: Response) => {
+ transformResponse: async (response: Response, params) => {
  if (!response.ok) {
  const errorText = await response.text()
  throw new Error(errorText || `SERP request failed with status ${response.status}`)
@@ -178,9 +178,14 @@ export const brightDataSerpSearchTool: ToolConfig<
  success: true,
  output: {
  results,
- query: ((data?.general as Record<string, unknown> | undefined)?.query as string) ?? null,
+ query:
+ ((data?.general as Record<string, unknown> | undefined)?.query as string) ??
+ params?.query ??
+ null,
  searchEngine:
- ((data?.general as Record<string, unknown> | undefined)?.search_engine as string) ?? null,
+ ((data?.general as Record<string, unknown> | undefined)?.search_engine as string) ??
+ params?.searchEngine ??
+ null,
  },
  }
  },

apps/sim/tools/brightdata/types.ts

@@ -131,6 +131,7 @@ export interface BrightDataDiscoverResponse extends ToolResponse {
  }>
  query: string | null
  totalResults: number
+ taskId?: string | null
  }
 }