From 2ab3bbdc60115d32df6c81af643f449ba7582b15 Mon Sep 17 00:00:00 2001 From: meirk-brd Date: Sun, 5 Apr 2026 11:13:31 +0300 Subject: [PATCH] feat: add discover command --- README.md | 56 +++++ package.json | 2 +- src/__tests__/commands/discover.test.ts | 260 ++++++++++++++++++++++++ src/commands/discover.ts | 188 +++++++++++++++++ src/index.ts | 2 + src/types/discover.ts | 60 ++++++ 6 files changed, 567 insertions(+), 1 deletion(-) create mode 100644 src/__tests__/commands/discover.test.ts create mode 100644 src/commands/discover.ts create mode 100644 src/types/discover.ts diff --git a/README.md b/README.md index f1438bf..2228896 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ |---|---| | `brightdata scrape` | Scrape any URL — bypasses CAPTCHAs, JS rendering, anti-bot protections | | `brightdata search` | Google / Bing / Yandex search with structured JSON output | +| `brightdata discover` | AI-powered web discovery - find and rank results by intent with optional full-page content | | `brightdata pipelines` | Extract structured data from 40+ platforms (Amazon, LinkedIn, TikTok…) | | `brightdata browser` | Control a real browser via Bright Data's Scraping Browser — navigate, snapshot, click, type, and more | | `brightdata zones` | List and inspect your Bright Data proxy zones | @@ -44,6 +45,7 @@ - [init](#init) - [scrape](#scrape) - [search](#search) + - [discover](#discover) - [pipelines](#pipelines) - [browser](#browser) - [status](#status) @@ -246,6 +248,60 @@ brightdata search "bright data pricing" --engine bing --- +### `discover` + +AI-powered web discovery. Submit a query with optional intent, and Bright Data finds, ranks, and optionally extracts full-page content for each result. + +```bash +brightdata discover [options] +``` + +| Flag | Description | +|---|---| +| `--intent ` | AI intent to evaluate and rank result relevance | +| `--country ` | ISO country code (default: `US`) | +| `--city ` | City for localized results (e.g. `"New York"`) | +| `--language ` | Language code (default: `en`) | +| `--num-results ` | Number of results to return | +| `--filter-keywords ` | Comma-separated keywords that must appear in results | +| `--include-content` | Include full page content in each result | +| `--no-remove-duplicates` | Keep duplicate results | +| `--start-date ` | Only content updated from date (`YYYY-MM-DD`) | +| `--end-date ` | Only content updated until date (`YYYY-MM-DD`) | +| `--timeout ` | Polling timeout (default: `600`) | +| `-o, --output ` | Write output to file | +| `--json` / `--pretty` | JSON output (raw / indented) | +| `-k, --api-key ` | Override API key | + +**Examples** + +```bash +# Basic discovery — table output +brightdata discover "AI trends" + +# With AI intent for relevance ranking +brightdata discover "AI trends" \ + --intent "Prioritize institutional reports for VC research" + +# Include full page content as markdown +brightdata discover "AI trends" --include-content --num-results 5 + +# Geo-targeted with date range +brightdata discover "best restaurants" --country US --city "New York" \ + --start-date 2025-01-01 --end-date 2025-12-31 + +# Filter results by keywords +brightdata discover "generative AI SaaS" --filter-keywords "revenue,SaaS" + +# JSON output to file +brightdata discover "AI trends" --num-results 10 --pretty -o results.json + +# Pipe-friendly — redirected stdout outputs JSON automatically +brightdata discover "AI trends" --include-content --num-results 3 > results.json +``` + +--- + ### `pipelines` Extract structured data from 40+ platforms using Bright Data's Web Scraper API. Triggers an async collection job, polls until ready, and returns results. diff --git a/package.json b/package.json index 27c252f..b0a30a4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@brightdata/cli", - "version": "0.1.6", + "version": "0.1.7", "description": "Command-line interface for Bright Data. Scrape, search, extract structured data, and automate browsers directly from your terminal.", "main": "dist/index.js", "bin": { diff --git a/src/__tests__/commands/discover.test.ts b/src/__tests__/commands/discover.test.ts new file mode 100644 index 0000000..019f1e0 --- /dev/null +++ b/src/__tests__/commands/discover.test.ts @@ -0,0 +1,260 @@ +import {describe, it, expect, beforeEach, vi} from 'vitest'; + +const mocks = vi.hoisted(()=>({ + post: vi.fn(), + get: vi.fn(), + ensure_authenticated: vi.fn(), + stop: vi.fn(), + start: vi.fn(), + print: vi.fn(), + print_table: vi.fn(), + fail: vi.fn((msg: string)=>{ throw new Error(`fail:${msg}`); }), + dim: vi.fn((msg: string)=>msg), + parse_timeout: vi.fn(), + poll_until: vi.fn(), +})); + +vi.mock('../../utils/client', ()=>({ + post: mocks.post, + get: mocks.get, +})); + +vi.mock('../../utils/auth', ()=>({ + ensure_authenticated: mocks.ensure_authenticated, +})); + +vi.mock('../../utils/spinner', ()=>({ + start: mocks.start, +})); + +vi.mock('../../utils/output', ()=>({ + print: mocks.print, + print_table: mocks.print_table, + fail: mocks.fail, + dim: mocks.dim, +})); + +vi.mock('../../utils/polling', ()=>({ + parse_timeout: mocks.parse_timeout, + poll_until: mocks.poll_until, +})); + +import { + handle_discover, + build_request, + extract_status, + format_markdown, + print_discover_table, +} from '../../commands/discover'; + +describe('commands/discover', ()=>{ + beforeEach(()=>{ + vi.clearAllMocks(); + mocks.ensure_authenticated.mockReturnValue('api_key'); + mocks.parse_timeout.mockReturnValue(600); + mocks.start.mockReturnValue({stop: mocks.stop}); + }); + + describe('build_request', ()=>{ + it('builds minimal request with only query', ()=>{ + const req = build_request('AI trends', {}); + expect(req).toEqual({query: 'AI trends'}); + }); + + it('includes all optional params', ()=>{ + const req = build_request('AI trends', { + intent: 'find research papers', + city: 'New York', + country: 'US', + language: 'en', + numResults: '10', + filterKeywords: 'AI, machine learning', + includeContent: true, + startDate: '2025-01-01', + endDate: '2025-12-31', + }); + expect(req).toEqual({ + query: 'AI trends', + intent: 'find research papers', + city: 'New York', + country: 'US', + language: 'en', + num_results: 10, + filter_keywords: ['AI', 'machine learning'], + include_content: true, + start_date: '2025-01-01', + end_date: '2025-12-31', + }); + }); + + it('parses comma-separated filter keywords with whitespace', ()=>{ + const req = build_request('q', {filterKeywords: ' a , b , c '}); + expect(req.filter_keywords).toEqual(['a', 'b', 'c']); + }); + + it('does not set format by default (API returns JSON)', ()=>{ + const req = build_request('test', {}); + expect(req.format).toBeUndefined(); + }); + + it('does not set format when include-content is used', ()=>{ + const req = build_request('test', {includeContent: true}); + expect(req.format).toBeUndefined(); + expect(req.include_content).toBe(true); + }); + }); + + describe('extract_status', ()=>{ + it('returns status from valid response', ()=>{ + expect(extract_status({status: 'processing'})).toBe('processing'); + expect(extract_status({status: 'done'})).toBe('done'); + }); + + it('returns undefined for invalid input', ()=>{ + expect(extract_status(null as never)).toBeUndefined(); + expect(extract_status(undefined as never)).toBeUndefined(); + }); + }); + + describe('format_markdown', ()=>{ + it('formats results as markdown', ()=>{ + const md = format_markdown([ + { + link: 'https://example.com', + title: 'Example', + description: 'A description', + relevance_score: 0.95, + }, + ], 'test query'); + expect(md).toContain('# Discover results for "test query"'); + expect(md).toContain('**1. [Example](https://example.com)** (95.0%)'); + expect(md).toContain('A description'); + }); + + it('includes content when present', ()=>{ + const md = format_markdown([ + { + link: 'https://example.com', + title: 'Example', + description: 'Desc', + relevance_score: 0.5, + content: '# Page content here', + }, + ], 'q'); + expect(md).toContain('# Page content here'); + }); + }); + + describe('print_discover_table', ()=>{ + it('calls print_table with formatted rows', ()=>{ + const results = [ + { + link: 'https://example.com', + title: 'Example Title', + description: 'Desc', + relevance_score: 0.98184747, + }, + ]; + print_discover_table(results); + expect(mocks.print_table).toHaveBeenCalledWith( + [{ + '#': '1', + title: 'Example Title', + score: '98.2%', + url: 'https://example.com', + }], + ['#', 'title', 'score', 'url'] + ); + }); + + it('prints dim message when no results', ()=>{ + const log = vi.spyOn(console, 'log').mockImplementation(()=>{}); + print_discover_table([]); + expect(log).toHaveBeenCalled(); + expect(mocks.print_table).not.toHaveBeenCalled(); + log.mockRestore(); + }); + }); + + describe('handle_discover', ()=>{ + it('triggers and polls then prints table', async()=>{ + mocks.post.mockResolvedValue({status: 'ok', task_id: 'abc123'}); + mocks.poll_until.mockResolvedValue({ + result: { + status: 'done', + duration_seconds: 5, + results: [ + { + link: 'https://example.com', + title: 'Result', + description: 'Desc', + relevance_score: 0.9, + }, + ], + }, + attempts: 3, + }); + await handle_discover('AI trends', {}); + expect(mocks.post).toHaveBeenCalledWith( + 'api_key', + '/discover', + {query: 'AI trends'}, + {timing: undefined} + ); + expect(mocks.poll_until).toHaveBeenCalledTimes(1); + expect(mocks.print_table).toHaveBeenCalledTimes(1); + }); + + it('prints json when --json is set', async()=>{ + const response = { + status: 'done', + duration_seconds: 2, + results: [{ + link: 'https://example.com', + title: 'R', + description: 'D', + relevance_score: 0.8, + }], + }; + mocks.post.mockResolvedValue({status: 'ok', task_id: 't1'}); + mocks.poll_until.mockResolvedValue({result: response, attempts: 1}); + await handle_discover('q', {json: true}); + expect(mocks.print).toHaveBeenCalledWith( + response, + {json: true, pretty: undefined, output: undefined} + ); + expect(mocks.print_table).not.toHaveBeenCalled(); + }); + + it('prints raw JSON when --output is set', async()=>{ + const response = { + status: 'done', + results: [{ + link: 'https://example.com', + title: 'R', + description: 'D', + relevance_score: 0.7, + }], + }; + mocks.post.mockResolvedValue({status: 'ok', task_id: 't2'}); + mocks.poll_until.mockResolvedValue({result: response, attempts: 1}); + await handle_discover('q', {output: 'out.json'}); + expect(mocks.print).toHaveBeenCalledWith( + response, + {json: undefined, pretty: undefined, output: 'out.json'} + ); + }); + + it('fails when trigger returns no task_id', async()=>{ + mocks.post.mockResolvedValue({status: 'ok'}); + const exit = vi.spyOn(process, 'exit') + .mockImplementation(()=>undefined as never); + const error = vi.spyOn(console, 'error') + .mockImplementation(()=>{}); + await handle_discover('q', {}); + expect(mocks.fail).toHaveBeenCalled(); + exit.mockRestore(); + error.mockRestore(); + }); + }); +}); diff --git a/src/commands/discover.ts b/src/commands/discover.ts new file mode 100644 index 0000000..f8368ce --- /dev/null +++ b/src/commands/discover.ts @@ -0,0 +1,188 @@ +import {Command} from 'commander'; +import {post, get} from '../utils/client'; +import {ensure_authenticated} from '../utils/auth'; +import {start as start_spinner} from '../utils/spinner'; +import {parse_timeout, poll_until} from '../utils/polling'; +import {print, print_table, dim, fail, is_tty} from '../utils/output'; +import type { + Discover_request, + Discover_trigger_response, + Discover_result, + Discover_poll_response, + Discover_opts, +} from '../types/discover'; + +const ENDPOINT = '/discover'; +const RUNNING_STATUSES = ['processing']; + +const build_request = (query: string, opts: Discover_opts): Discover_request=>{ + const req: Discover_request = {query}; + if (opts.intent) + req.intent = opts.intent; + if (opts.city) + req.city = opts.city; + if (opts.country) + req.country = opts.country; + if (opts.language) + req.language = opts.language; + if (opts.numResults) + req.num_results = +opts.numResults; + if (opts.filterKeywords) + req.filter_keywords = opts.filterKeywords.split(',').map(k=>k.trim()); + if (opts.includeContent) + { + req.include_content = true; + } + if (opts.removeDuplicates === false) + req.remove_duplicates = false; + if (opts.startDate) + req.start_date = opts.startDate; + if (opts.endDate) + req.end_date = opts.endDate; + return req; +}; + +const format_markdown = ( + results: Discover_result[], + query: string +): string=>{ + const lines: string[] = []; + lines.push(`# Discover results for "${query}"`); + lines.push(`_${results.length} results_`); + lines.push(''); + for (let i=0; i`); + if (r.content) + { + lines.push(''); + lines.push(r.content); + } + lines.push(''); + } + return lines.join('\n'); +}; + +const print_discover_table = (results: Discover_result[])=>{ + if (!results.length) + { + console.log(dim('No results found.')); + return; + } + const rows = results.map((r, i)=>({ + '#': String(i+1), + title: (r.title ?? '').slice(0, 50), + score: (r.relevance_score * 100).toFixed(1)+'%', + url: (r.link ?? '').slice(0, 60), + })); + print_table(rows, ['#', 'title', 'score', 'url']); +}; + +const extract_status = (result: Discover_poll_response): string|undefined=>{ + if (!result || typeof result != 'object') + return undefined; + return result.status; +}; + +const handle_discover = async(query: string, opts: Discover_opts)=>{ + const api_key = ensure_authenticated(opts.apiKey); + let timeout = 600; + try { + timeout = parse_timeout(opts.timeout); + } catch(e) { + fail((e as Error).message); + return; + } + const body = build_request(query, opts); + const spinner = start_spinner(`Discovering results for "${query}"...`); + try { + const trigger = await post( + api_key, + ENDPOINT, + body, + {timing: opts.timing} + ); + const task_id = trigger.task_id; + if (!task_id) + { + spinner.stop(); + fail('Failed to trigger discover (missing task_id).'); + return; + } + spinner.stop(); + console.error(dim(`Task submitted: ${task_id}`)); + const poll_spinner = start_spinner('Waiting for results...'); + const poll_result = await poll_until({ + timeout_seconds: timeout, + fetch_once: ()=>get( + api_key, + `${ENDPOINT}?task_id=${task_id}`, + {timing: opts.timing} + ), + get_status: extract_status, + running_statuses: RUNNING_STATUSES, + timeout_label: 'discover results', + on_running: ({attempt, timeout_seconds, status})=>{ + console.error(dim( + `Status: ${status} — polling ` + +`(attempt ${attempt}/${timeout_seconds})` + )); + }, + }); + poll_spinner.stop(); + const response = poll_result.result; + const results = response.results ?? []; + if (response.duration_seconds != null) + { + console.error(dim( + `Done in ${response.duration_seconds}s ` + +`(${poll_result.attempts} poll attempts)` + )); + } + const print_opts = {json: opts.json, pretty: opts.pretty, + output: opts.output}; + if (opts.json || opts.pretty || opts.output || !is_tty) + { + print(response, print_opts); + return; + } + print_discover_table(results); + } catch(e) { + spinner.stop(); + console.error((e as Error).message); + process.exit(1); + } +}; + +const discover_command = new Command('discover') + .description('Search and rank web results using AI-driven intent') + .argument('', 'Search query') + .option('--intent ', + 'AI intent to evaluate and rank result relevance') + .option('--country ', + 'ISO country code for localized results (default: US)') + .option('--city ', 'City for localized results (e.g. "New York")') + .option('--language ', 'Language code (default: en)') + .option('--num-results ', 'Number of results to return') + .option('--filter-keywords ', + 'Comma-separated keywords that must appear in results') + .option('--include-content', 'Include page content in markdown format') + .option('--no-remove-duplicates', 'Keep duplicate results') + .option('--start-date ', 'Only content updated from date (YYYY-MM-DD)') + .option('--end-date ', 'Only content updated until date (YYYY-MM-DD)') + .option('--timeout ', + 'Polling timeout in seconds (default: 600)') + .option('-o, --output ', 'Write output to file') + .option('--json', 'Force JSON output') + .option('--pretty', 'Pretty-print JSON output') + .option('--timing', 'Show request timing') + .option('-k, --api-key ', 'Override API key') + .action(handle_discover); + +export {discover_command, handle_discover, build_request, extract_status, + format_markdown, print_discover_table}; diff --git a/src/index.ts b/src/index.ts index 5ef0183..11d5db3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -15,6 +15,7 @@ import {skill_command} from './commands/skill'; import {budget_command} from './commands/budget'; import {browser_command} from './commands/browser'; import {add_mcp_command} from './commands/add-mcp'; +import {discover_command} from './commands/discover'; import packageJson from '../package.json'; const build_program = ()=>{ @@ -46,6 +47,7 @@ const build_program = ()=>{ program.addCommand(skill_command); program.addCommand(budget_command); program.addCommand(browser_command); + program.addCommand(discover_command); program.addCommand(add_command); return program; diff --git a/src/types/discover.ts b/src/types/discover.ts new file mode 100644 index 0000000..13e8df7 --- /dev/null +++ b/src/types/discover.ts @@ -0,0 +1,60 @@ +type Discover_request = { + query: string; + intent?: string; + city?: string; + country?: string; + filter_keywords?: string[]; + format?: 'json'|'md'; + include_content?: boolean; + language?: string; + num_results?: number; + remove_duplicates?: boolean; + start_date?: string; + end_date?: string; +}; + +type Discover_trigger_response = { + status: string; + task_id: string; +}; + +type Discover_result = { + link: string; + title: string; + description: string; + relevance_score: number; + content?: string|null; +}; + +type Discover_poll_response = { + status: string; + duration_seconds?: number; + results?: Discover_result[]; +}; + +type Discover_opts = { + intent?: string; + city?: string; + country?: string; + language?: string; + numResults?: string; + filterKeywords?: string; + includeContent?: boolean; + removeDuplicates?: boolean; + startDate?: string; + endDate?: string; + timeout?: string; + output?: string; + json?: boolean; + pretty?: boolean; + timing?: boolean; + apiKey?: string; +}; + +export type { + Discover_request, + Discover_trigger_response, + Discover_result, + Discover_poll_response, + Discover_opts, +};