diff --git a/examples/browser_use_integration.py b/examples/browser_use_integration.py new file mode 100644 index 0000000..d24468f --- /dev/null +++ b/examples/browser_use_integration.py @@ -0,0 +1,206 @@ +""" +Example: Using Sentience with browser-use for element grounding. + +This example demonstrates how to integrate Sentience's semantic element +detection with browser-use, enabling accurate click/type/scroll operations +using Sentience's snapshot-based grounding instead of coordinate estimation. + +Requirements: + pip install browser-use sentienceapi + +Usage: + python examples/browser_use_integration.py +""" + +import asyncio + +# Sentience imports +from sentience import find, get_extension_dir, query +from sentience.backends import ( + BrowserUseAdapter, + CachedSnapshot, + ExtensionNotLoadedError, + click, + scroll, + snapshot, + type_text, +) + +# browser-use imports (install via: pip install browser-use) +# from browser_use import BrowserSession, BrowserProfile + + +async def main() -> None: + """ + Demo: Search on Google using Sentience grounding with browser-use. + + This example shows the full workflow: + 1. Launch browser-use with Sentience extension loaded + 2. Create a Sentience backend adapter + 3. Take snapshots and interact with elements using semantic queries + """ + + # ========================================================================= + # STEP 1: Setup browser-use with Sentience extension + # ========================================================================= + # + # The Sentience extension must be loaded for element grounding to work. + # Use get_extension_dir() to get the path to the bundled extension. + # + # Uncomment the following when running with browser-use installed: + + # extension_path = get_extension_dir() + # print(f"Loading Sentience extension from: {extension_path}") + # + # profile = BrowserProfile( + # args=[ + # f"--load-extension={extension_path}", + # "--disable-extensions-except=" + extension_path, + # ], + # ) + # session = BrowserSession(browser_profile=profile) + # await session.start() + + # ========================================================================= + # STEP 2: Create Sentience backend adapter + # ========================================================================= + # + # The adapter bridges browser-use's CDP client to Sentience's backend protocol. + # + # adapter = BrowserUseAdapter(session) + # backend = await adapter.create_backend() + + # ========================================================================= + # STEP 3: Navigate and take snapshots + # ========================================================================= + # + # await session.navigate("https://www.google.com") + # + # # Take a snapshot - this uses the Sentience extension's element detection + # try: + # snap = await snapshot(backend) + # print(f"Found {len(snap.elements)} elements") + # except ExtensionNotLoadedError as e: + # print(f"Extension not loaded: {e}") + # print("Make sure the browser was launched with --load-extension flag") + # return + + # ========================================================================= + # STEP 4: Find and interact with elements using semantic queries + # ========================================================================= + # + # Sentience provides powerful element selectors: + # - Role-based: 'role=textbox', 'role=button' + # - Name-based: 'role=button[name="Submit"]' + # - Text-based: 'text=Search' + # + # # Find the search input + # search_input = find(snap, 'role=textbox[name*="Search"]') + # if search_input: + # # Click on the search input (uses center of bounding box) + # await click(backend, search_input.bbox) + # + # # Type search query + # await type_text(backend, "Sentience AI browser automation") + # print("Typed search query") + + # ========================================================================= + # STEP 5: Using cached snapshots for efficiency + # ========================================================================= + # + # Taking snapshots has overhead. Use CachedSnapshot to reuse recent snapshots: + # + # cache = CachedSnapshot(backend, max_age_ms=2000) + # + # # First call takes fresh snapshot + # snap1 = await cache.get() + # + # # Second call returns cached version if less than 2 seconds old + # snap2 = await cache.get() + # + # # After actions that modify DOM, invalidate the cache + # await click(backend, some_element.bbox) + # cache.invalidate() # Next get() will take fresh snapshot + + # ========================================================================= + # STEP 6: Scrolling to elements + # ========================================================================= + # + # # Scroll down by 500 pixels + # await scroll(backend, delta_y=500) + # + # # Scroll at a specific position (useful for scrollable containers) + # await scroll(backend, delta_y=300, target=(400, 500)) + + # ========================================================================= + # STEP 7: Advanced element queries + # ========================================================================= + # + # # Find all buttons + # buttons = query(snap, 'role=button') + # print(f"Found {len(buttons)} buttons") + # + # # Find by partial text match + # links = query(snap, 'role=link[name*="Learn"]') + # + # # Find by exact text + # submit_btn = find(snap, 'role=button[name="Submit"]') + + # ========================================================================= + # STEP 8: Error handling + # ========================================================================= + # + # Sentience provides specific exceptions for common errors: + # + # from sentience.backends import ( + # ExtensionNotLoadedError, # Extension not loaded in browser + # SnapshotError, # Snapshot failed + # ActionError, # Click/type/scroll failed + # ) + # + # try: + # snap = await snapshot(backend) + # except ExtensionNotLoadedError as e: + # # The error message includes fix suggestions + # print(f"Fix: {e}") + + # ========================================================================= + # CLEANUP + # ========================================================================= + # + # await session.stop() + + print("=" * 60) + print("browser-use + Sentience Integration Example") + print("=" * 60) + print() + print("This example demonstrates the integration pattern.") + print("To run with a real browser, uncomment the code sections above") + print("and install browser-use: pip install browser-use") + print() + print("Key imports:") + print(" from sentience import get_extension_dir, find, query") + print(" from sentience.backends import (") + print(" BrowserUseAdapter, snapshot, click, type_text, scroll") + print(" )") + print() + print("Extension path:", get_extension_dir()) + + +async def full_example() -> None: + """ + Complete working example - requires browser-use installed. + + This is the uncommented version for users who have browser-use installed. + """ + # Import browser-use (uncomment when installed) + # from browser_use import BrowserSession, BrowserProfile + + print("To run the full example:") + print("1. Install browser-use: pip install browser-use") + print("2. Uncomment the imports in this function") + print("3. Run: python examples/browser_use_integration.py") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/sentience/__init__.py b/sentience/__init__.py index ecb4711..91ebe36 100644 --- a/sentience/__init__.py +++ b/sentience/__init__.py @@ -118,7 +118,7 @@ from .visual_agent import SentienceVisualAgent, SentienceVisualAgentAsync from .wait import wait_for -__version__ = "0.92.3" +__version__ = "0.93.0" __all__ = [ # Extension helpers (for browser-use integration) diff --git a/sentience/backends/__init__.py b/sentience/backends/__init__.py index 0c7d7f3..97601c6 100644 --- a/sentience/backends/__init__.py +++ b/sentience/backends/__init__.py @@ -5,13 +5,28 @@ Sentience actions (click, type, scroll) to work with different browser automation frameworks. -Supported backends: -- PlaywrightBackend: Default backend using Playwright (existing SentienceBrowser) -- CDPBackendV0: CDP-based backend for browser-use integration +Supported Backends +------------------ + +**PlaywrightBackend** + Wraps Playwright Page objects. Use this when integrating with existing + SentienceBrowser or Playwright-based code. + +**CDPBackendV0** + Low-level CDP (Chrome DevTools Protocol) backend. Use this when you have + direct access to a CDP client and session. + +**BrowserUseAdapter** + High-level adapter for browser-use framework. Automatically creates a + CDPBackendV0 from a BrowserSession. + +Quick Start with browser-use +---------------------------- + +.. code-block:: python -For browser-use integration: from browser_use import BrowserSession, BrowserProfile - from sentience import get_extension_dir + from sentience import get_extension_dir, find from sentience.backends import BrowserUseAdapter, snapshot, click, type_text # Setup browser-use with Sentience extension @@ -23,15 +38,63 @@ adapter = BrowserUseAdapter(session) backend = await adapter.create_backend() - # Take snapshot and interact + # Take snapshot and interact with elements snap = await snapshot(backend) - element = find(snap, 'role=button[name="Submit"]') + search_box = find(snap, 'role=textbox[name*="Search"]') + await click(backend, search_box.bbox) + await type_text(backend, "Sentience AI") + +Snapshot Caching +---------------- + +Use CachedSnapshot to reduce redundant snapshot calls in action loops: + +.. code-block:: python + + from sentience.backends import CachedSnapshot + + cache = CachedSnapshot(backend, max_age_ms=2000) + + snap1 = await cache.get() # Takes fresh snapshot + snap2 = await cache.get() # Returns cached if < 2s old + await click(backend, element.bbox) + cache.invalidate() # Force refresh on next get() + +Error Handling +-------------- + +The module provides specific exceptions for common failure modes: + +- ``ExtensionNotLoadedError``: Extension not loaded in browser launch args +- ``SnapshotError``: window.sentience.snapshot() failed +- ``ActionError``: Click/type/scroll operation failed + +All exceptions inherit from ``SentienceBackendError`` and include helpful +fix suggestions in their error messages. + +.. code-block:: python + + from sentience.backends import ExtensionNotLoadedError, snapshot + + try: + snap = await snapshot(backend) + except ExtensionNotLoadedError as e: + print(f"Fix suggestion: {e}") """ from .actions import click, scroll, scroll_to_element, type_text, wait_for_stable from .browser_use_adapter import BrowserUseAdapter, BrowserUseCDPTransport from .cdp_backend import CDPBackendV0, CDPTransport +from .exceptions import ( + ActionError, + BackendEvalError, + ExtensionDiagnostics, + ExtensionInjectionError, + ExtensionNotLoadedError, + SentienceBackendError, + SnapshotError, +) from .playwright_backend import PlaywrightBackend from .protocol_v0 import BrowserBackendV0, LayoutMetrics, ViewportInfo from .snapshot import CachedSnapshot, snapshot @@ -58,4 +121,12 @@ "scroll", "scroll_to_element", "wait_for_stable", + # Exceptions + "SentienceBackendError", + "ExtensionNotLoadedError", + "ExtensionInjectionError", + "ExtensionDiagnostics", + "BackendEvalError", + "SnapshotError", + "ActionError", ] diff --git a/sentience/backends/actions.py b/sentience/backends/actions.py index c987d64..67ec479 100644 --- a/sentience/backends/actions.py +++ b/sentience/backends/actions.py @@ -226,7 +226,8 @@ async def scroll_to_element( start_time = time.time() try: - scrolled = await backend.eval(f""" + scrolled = await backend.eval( + f""" (() => {{ const el = window.sentience_registry && window.sentience_registry[{element_id}]; if (el && el.scrollIntoView) {{ @@ -239,7 +240,8 @@ async def scroll_to_element( }} return false; }})() - """) + """ + ) # Wait for scroll animation wait_time = 0.3 if behavior == "smooth" else 0.05 diff --git a/sentience/backends/exceptions.py b/sentience/backends/exceptions.py new file mode 100644 index 0000000..a1d176c --- /dev/null +++ b/sentience/backends/exceptions.py @@ -0,0 +1,211 @@ +""" +Custom exceptions for Sentience backends. + +These exceptions provide clear, actionable error messages when things go wrong +during browser-use integration or backend operations. +""" + +from dataclasses import dataclass +from typing import Any + + +class SentienceBackendError(Exception): + """Base exception for all Sentience backend errors.""" + + pass + + +@dataclass +class ExtensionDiagnostics: + """Diagnostics collected when extension loading fails.""" + + sentience_defined: bool = False + sentience_snapshot: bool = False + url: str = "" + error: str | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "ExtensionDiagnostics": + """Create from diagnostic dict returned by browser eval.""" + return cls( + sentience_defined=data.get("sentience_defined", False), + sentience_snapshot=data.get("sentience_snapshot", False), + url=data.get("url", ""), + error=data.get("error"), + ) + + def to_dict(self) -> dict[str, Any]: + """Convert to dict for serialization.""" + return { + "sentience_defined": self.sentience_defined, + "sentience_snapshot": self.sentience_snapshot, + "url": self.url, + "error": self.error, + } + + +class ExtensionNotLoadedError(SentienceBackendError): + """ + Raised when the Sentience extension is not loaded in the browser. + + This typically means: + 1. Browser was launched without --load-extension flag + 2. Extension path is incorrect + 3. Extension failed to initialize + + Example fix for browser-use: + from sentience import get_extension_dir + from browser_use import BrowserSession, BrowserProfile + + profile = BrowserProfile( + args=[f"--load-extension={get_extension_dir()}"], + ) + session = BrowserSession(browser_profile=profile) + """ + + def __init__( + self, + message: str, + timeout_ms: int | None = None, + diagnostics: ExtensionDiagnostics | None = None, + ) -> None: + self.timeout_ms = timeout_ms + self.diagnostics = diagnostics + super().__init__(message) + + @classmethod + def from_timeout( + cls, + timeout_ms: int, + diagnostics: ExtensionDiagnostics | None = None, + ) -> "ExtensionNotLoadedError": + """Create error from timeout during extension wait.""" + diag_info = "" + if diagnostics: + if diagnostics.error: + diag_info = f"\n Error: {diagnostics.error}" + else: + diag_info = ( + f"\n window.sentience defined: {diagnostics.sentience_defined}" + f"\n window.sentience.snapshot available: {diagnostics.sentience_snapshot}" + f"\n Page URL: {diagnostics.url}" + ) + + message = ( + f"Sentience extension not loaded after {timeout_ms}ms.{diag_info}\n\n" + "To fix this, ensure the extension is loaded when launching the browser:\n\n" + " from sentience import get_extension_dir\n" + " from browser_use import BrowserSession, BrowserProfile\n\n" + " profile = BrowserProfile(\n" + f' args=[f"--load-extension={{get_extension_dir()}}"],\n' + " )\n" + " session = BrowserSession(browser_profile=profile)\n" + ) + return cls(message, timeout_ms=timeout_ms, diagnostics=diagnostics) + + +class ExtensionInjectionError(SentienceBackendError): + """ + Raised when window.sentience API is not available on the page. + + This can happen when: + 1. Page loaded before extension could inject + 2. Page has Content Security Policy blocking extension + 3. Extension crashed or was disabled + + Call snapshot() with a longer timeout or wait for page load. + """ + + def __init__( + self, + message: str, + url: str | None = None, + ) -> None: + self.url = url + super().__init__(message) + + @classmethod + def from_page(cls, url: str) -> "ExtensionInjectionError": + """Create error for a specific page.""" + message = ( + f"window.sentience API not available on page: {url}\n\n" + "Possible causes:\n" + " 1. Page loaded before extension could inject (try increasing timeout)\n" + " 2. Page has Content Security Policy blocking the extension\n" + " 3. Extension was disabled or crashed\n\n" + "Try:\n" + " snap = await snapshot(backend, options=SnapshotOptions(timeout_ms=10000))" + ) + return cls(message, url=url) + + +class BackendEvalError(SentienceBackendError): + """ + Raised when JavaScript evaluation fails in the browser. + + This wraps underlying CDP or Playwright errors with context. + """ + + def __init__( + self, + message: str, + expression: str | None = None, + original_error: Exception | None = None, + ) -> None: + self.expression = expression + self.original_error = original_error + super().__init__(message) + + +class SnapshotError(SentienceBackendError): + """ + Raised when taking a snapshot fails. + + This can happen when: + 1. Extension returned null or invalid data + 2. Page is in an invalid state + 3. Extension threw an error + """ + + def __init__( + self, + message: str, + url: str | None = None, + raw_result: Any = None, + ) -> None: + self.url = url + self.raw_result = raw_result + super().__init__(message) + + @classmethod + def from_null_result(cls, url: str | None = None) -> "SnapshotError": + """Create error for null snapshot result.""" + message = ( + "window.sentience.snapshot() returned null.\n\n" + "Possible causes:\n" + " 1. Extension is not properly initialized\n" + " 2. Page DOM is in an invalid state\n" + " 3. Extension encountered an internal error\n\n" + "Try refreshing the page and taking a new snapshot." + ) + if url: + message = f"{message}\n Page URL: {url}" + return cls(message, url=url, raw_result=None) + + +class ActionError(SentienceBackendError): + """ + Raised when a browser action (click, type, scroll) fails. + """ + + def __init__( + self, + action: str, + message: str, + coordinates: tuple[float, float] | None = None, + original_error: Exception | None = None, + ) -> None: + self.action = action + self.coordinates = coordinates + self.original_error = original_error + super().__init__(f"{action} failed: {message}") diff --git a/sentience/backends/playwright_backend.py b/sentience/backends/playwright_backend.py index f5ea8df..719561a 100644 --- a/sentience/backends/playwright_backend.py +++ b/sentience/backends/playwright_backend.py @@ -57,7 +57,8 @@ def page(self) -> "AsyncPage": async def refresh_page_info(self) -> ViewportInfo: """Cache viewport + scroll offsets; cheap & safe to call often.""" - result = await self._page.evaluate(""" + result = await self._page.evaluate( + """ (() => ({ width: window.innerWidth, height: window.innerHeight, @@ -66,7 +67,8 @@ async def refresh_page_info(self) -> ViewportInfo: content_width: document.documentElement.scrollWidth, content_height: document.documentElement.scrollHeight }))() - """) + """ + ) self._cached_viewport = ViewportInfo( width=result.get("width", 0), @@ -96,7 +98,8 @@ async def get_layout_metrics(self) -> LayoutMetrics: """Get page layout metrics.""" # Playwright doesn't expose CDP directly in the same way, # so we approximate using JavaScript - result = await self._page.evaluate(""" + result = await self._page.evaluate( + """ (() => ({ viewport_x: window.scrollX, viewport_y: window.scrollY, @@ -106,7 +109,8 @@ async def get_layout_metrics(self) -> LayoutMetrics: content_height: document.documentElement.scrollHeight, device_scale_factor: window.devicePixelRatio || 1 }))() - """) + """ + ) return LayoutMetrics( viewport_x=result.get("viewport_x", 0), @@ -172,8 +176,7 @@ async def wait_ready_state( elapsed = time.monotonic() - start if elapsed >= timeout_sec: raise TimeoutError( - f"Timed out waiting for document.readyState='{state}' " - f"after {timeout_ms}ms" + f"Timed out waiting for document.readyState='{state}' " f"after {timeout_ms}ms" ) current_state = await self._page.evaluate("document.readyState") diff --git a/sentience/backends/snapshot.py b/sentience/backends/snapshot.py index 6f11dd9..2a1ff7d 100644 --- a/sentience/backends/snapshot.py +++ b/sentience/backends/snapshot.py @@ -25,6 +25,12 @@ from typing import TYPE_CHECKING, Any from ..models import Snapshot, SnapshotOptions +from ..snapshot import ( + _build_snapshot_payload, + _merge_api_result_with_local, + _post_snapshot_to_gateway_async, +) +from .exceptions import ExtensionDiagnostics, ExtensionNotLoadedError, SnapshotError if TYPE_CHECKING: from .protocol_v0 import BrowserBackendV0 @@ -144,8 +150,9 @@ async def snapshot( """ Take a Sentience snapshot using the backend protocol. - This function calls window.sentience.snapshot() via the backend's eval(), - enabling snapshot collection with any BrowserBackendV0 implementation. + This function respects the `use_api` option and can call either: + - Server-side API (Pro/Enterprise tier) when `use_api=True` and API key is provided + - Local extension (Free tier) when `use_api=False` or no API key Requires: - Sentience extension loaded in browser (via --load-extension) @@ -153,64 +160,50 @@ async def snapshot( Args: backend: BrowserBackendV0 implementation (CDPBackendV0, PlaywrightBackend, etc.) - options: Snapshot options (limit, filter, screenshot, etc.) + options: Snapshot options (limit, filter, screenshot, use_api, sentience_api_key, etc.) Returns: Snapshot with elements, viewport, and optional screenshot Example: from sentience.backends import BrowserUseAdapter - from sentience.backends.snapshot import snapshot_from_backend + from sentience.backends.snapshot import snapshot + from sentience.models import SnapshotOptions adapter = BrowserUseAdapter(session) backend = await adapter.create_backend() - # Basic snapshot - snap = await snapshot_from_backend(backend) + # Basic snapshot (uses local extension) + snap = await snapshot(backend) - # With options - snap = await snapshot_from_backend(backend, SnapshotOptions( + # With server-side API (Pro/Enterprise tier) + snap = await snapshot(backend, SnapshotOptions( + use_api=True, + sentience_api_key="sk_pro_xxxxx", limit=100, screenshot=True )) + + # Force local extension (Free tier) + snap = await snapshot(backend, SnapshotOptions( + use_api=False + )) """ if options is None: options = SnapshotOptions() - # Wait for extension injection - await _wait_for_extension(backend, timeout_ms=5000) + # Determine if we should use server-side API + # Same logic as main snapshot() function in sentience/snapshot.py + should_use_api = ( + options.use_api if options.use_api is not None else (options.sentience_api_key is not None) + ) - # Build options dict for extension API - ext_options = _build_extension_options(options) - - # Call extension's snapshot function - result = await backend.eval(f""" - (() => {{ - const options = {_json_serialize(ext_options)}; - return window.sentience.snapshot(options); - }})() - """) - - if result is None: - raise RuntimeError( - "window.sentience.snapshot() returned null. " - "Is the Sentience extension loaded and injected?" - ) - - # Show overlay if requested - if options.show_overlay: - raw_elements = result.get("raw_elements", []) - if raw_elements: - await backend.eval(f""" - (() => {{ - if (window.sentience && window.sentience.showOverlay) {{ - window.sentience.showOverlay({_json_serialize(raw_elements)}, null); - }} - }})() - """) - - # Build and return Snapshot - return Snapshot(**result) + if should_use_api and options.sentience_api_key: + # Use server-side API (Pro/Enterprise tier) + return await _snapshot_via_api(backend, options) + else: + # Use local extension (Free tier) + return await _snapshot_via_extension(backend, options) async def _wait_for_extension( @@ -228,28 +221,45 @@ async def _wait_for_extension( RuntimeError: If extension not injected within timeout """ import asyncio + import logging + + logger = logging.getLogger("sentience.backends.snapshot") start = time.monotonic() timeout_sec = timeout_ms / 1000.0 + poll_count = 0 + + logger.debug(f"Waiting for extension injection (timeout={timeout_ms}ms)...") while True: elapsed = time.monotonic() - start + poll_count += 1 + + if poll_count % 10 == 0: # Log every 10 polls (~1 second) + logger.debug(f"Extension poll #{poll_count}, elapsed={elapsed*1000:.0f}ms") + if elapsed >= timeout_sec: # Gather diagnostics try: - diag = await backend.eval(""" + diag_dict = await backend.eval( + """ (() => ({ sentience_defined: typeof window.sentience !== 'undefined', sentience_snapshot: typeof window.sentience?.snapshot === 'function', - url: window.location.href + url: window.location.href, + extension_id: document.documentElement.dataset.sentienceExtensionId || null, + has_content_script: !!document.documentElement.dataset.sentienceExtensionId }))() - """) - except Exception: - diag = {"error": "Could not gather diagnostics"} - - raise RuntimeError( - f"Sentience extension failed to inject window.sentience API " - f"within {timeout_ms}ms. Diagnostics: {diag}" + """ + ) + diagnostics = ExtensionDiagnostics.from_dict(diag_dict) + logger.debug(f"Extension diagnostics: {diag_dict}") + except Exception as e: + diagnostics = ExtensionDiagnostics(error=f"Could not gather diagnostics: {e}") + + raise ExtensionNotLoadedError.from_timeout( + timeout_ms=timeout_ms, + diagnostics=diagnostics, ) # Check if extension is ready @@ -266,6 +276,124 @@ async def _wait_for_extension( await asyncio.sleep(0.1) +async def _snapshot_via_extension( + backend: "BrowserBackendV0", + options: SnapshotOptions, +) -> Snapshot: + """Take snapshot using local extension (Free tier)""" + # Wait for extension injection + await _wait_for_extension(backend, timeout_ms=5000) + + # Build options dict for extension API + ext_options = _build_extension_options(options) + + # Call extension's snapshot function + result = await backend.eval( + f""" + (() => {{ + const options = {_json_serialize(ext_options)}; + return window.sentience.snapshot(options); + }})() + """ + ) + + if result is None: + # Try to get URL for better error message + try: + url = await backend.eval("window.location.href") + except Exception: + url = None + raise SnapshotError.from_null_result(url=url) + + # Show overlay if requested + if options.show_overlay: + raw_elements = result.get("raw_elements", []) + if raw_elements: + await backend.eval( + f""" + (() => {{ + if (window.sentience && window.sentience.showOverlay) {{ + window.sentience.showOverlay({_json_serialize(raw_elements)}, null); + }} + }})() + """ + ) + + # Build and return Snapshot + return Snapshot(**result) + + +async def _snapshot_via_api( + backend: "BrowserBackendV0", + options: SnapshotOptions, +) -> Snapshot: + """Take snapshot using server-side API (Pro/Enterprise tier)""" + # Default API URL (same as main snapshot function) + api_url = "https://api.sentienceapi.com" + + # Wait for extension injection (needed even for API mode to collect raw data) + await _wait_for_extension(backend, timeout_ms=5000) + + # Step 1: Get raw data from local extension (always happens locally) + raw_options: dict[str, Any] = {} + if options.screenshot is not False: + raw_options["screenshot"] = options.screenshot + + # Call extension to get raw elements + raw_result = await backend.eval( + f""" + (() => {{ + const options = {_json_serialize(raw_options)}; + return window.sentience.snapshot(options); + }})() + """ + ) + + if raw_result is None: + try: + url = await backend.eval("window.location.href") + except Exception: + url = None + raise SnapshotError.from_null_result(url=url) + + # Step 2: Send to server for smart ranking/filtering + payload = _build_snapshot_payload(raw_result, options) + + try: + api_result = await _post_snapshot_to_gateway_async( + payload, options.sentience_api_key, api_url + ) + + # Merge API result with local data (screenshot, etc.) + snapshot_data = _merge_api_result_with_local(api_result, raw_result) + + # Show visual overlay if requested (use API-ranked elements) + if options.show_overlay: + elements = api_result.get("elements", []) + if elements: + await backend.eval( + f""" + (() => {{ + if (window.sentience && window.sentience.showOverlay) {{ + window.sentience.showOverlay({_json_serialize(elements)}, null); + }} + }})() + """ + ) + + return Snapshot(**snapshot_data) + except (RuntimeError, ValueError): + # Re-raise validation errors as-is + raise + except Exception as e: + # Fallback to local extension on API error + # This matches the behavior of the main snapshot function + raise RuntimeError( + f"Server-side snapshot API failed: {e}. " + "Try using use_api=False to use local extension instead." + ) from e + + def _build_extension_options(options: SnapshotOptions) -> dict[str, Any]: """Build options dict for extension API call.""" ext_options: dict[str, Any] = {} @@ -294,4 +422,5 @@ def _build_extension_options(options: SnapshotOptions) -> dict[str, Any]: def _json_serialize(obj: Any) -> str: """Serialize object to JSON string for embedding in JS.""" import json + return json.dumps(obj) diff --git a/sentience/extension/background.js b/sentience/extension/background.js index aff49b0..02c0408 100644 --- a/sentience/extension/background.js +++ b/sentience/extension/background.js @@ -1,4 +1,4 @@ -import init, { analyze_page_with_options, analyze_page, prune_for_api } from "../pkg/sentience_core.js"; +import init, { analyze_page_with_options, analyze_page, prune_for_api } from "./pkg/sentience_core.js"; let wasmReady = !1, wasmInitPromise = null; diff --git a/sentience/snapshot.py b/sentience/snapshot.py index ec17d5a..3366141 100644 --- a/sentience/snapshot.py +++ b/sentience/snapshot.py @@ -19,6 +19,122 @@ MAX_PAYLOAD_BYTES = 10 * 1024 * 1024 +def _build_snapshot_payload( + raw_result: dict[str, Any], + options: SnapshotOptions, +) -> dict[str, Any]: + """ + Build payload dict for gateway snapshot API. + + Shared helper used by both sync and async snapshot implementations. + """ + return { + "raw_elements": raw_result.get("raw_elements", []), + "url": raw_result.get("url", ""), + "viewport": raw_result.get("viewport"), + "goal": options.goal, + "options": { + "limit": options.limit, + "filter": options.filter.model_dump() if options.filter else None, + }, + } + + +def _validate_payload_size(payload_json: str) -> None: + """ + Validate payload size before sending to gateway. + + Raises ValueError if payload exceeds server limit. + """ + payload_size = len(payload_json.encode("utf-8")) + if payload_size > MAX_PAYLOAD_BYTES: + raise ValueError( + f"Payload size ({payload_size / 1024 / 1024:.2f}MB) exceeds server limit " + f"({MAX_PAYLOAD_BYTES / 1024 / 1024:.0f}MB). " + f"Try reducing the number of elements on the page or filtering elements." + ) + + +def _post_snapshot_to_gateway_sync( + payload: dict[str, Any], + api_key: str, + api_url: str = "https://api.sentienceapi.com", +) -> dict[str, Any]: + """ + Post snapshot payload to gateway (synchronous). + + Used by sync snapshot() function. + """ + payload_json = json.dumps(payload) + _validate_payload_size(payload_json) + + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + + response = requests.post( + f"{api_url}/v1/snapshot", + data=payload_json, + headers=headers, + timeout=30, + ) + response.raise_for_status() + return response.json() + + +async def _post_snapshot_to_gateway_async( + payload: dict[str, Any], + api_key: str, + api_url: str = "https://api.sentienceapi.com", +) -> dict[str, Any]: + """ + Post snapshot payload to gateway (asynchronous). + + Used by async backend snapshot() function. + """ + # Lazy import httpx - only needed for async API calls + import httpx + + payload_json = json.dumps(payload) + _validate_payload_size(payload_json) + + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + + async with httpx.AsyncClient(timeout=30.0) as client: + response = await client.post( + f"{api_url}/v1/snapshot", + content=payload_json, + headers=headers, + ) + response.raise_for_status() + return response.json() + + +def _merge_api_result_with_local( + api_result: dict[str, Any], + raw_result: dict[str, Any], +) -> dict[str, Any]: + """ + Merge API result with local data (screenshot, etc.). + + Shared helper used by both sync and async snapshot implementations. + """ + return { + "status": api_result.get("status", "success"), + "timestamp": api_result.get("timestamp"), + "url": api_result.get("url", raw_result.get("url", "")), + "viewport": api_result.get("viewport", raw_result.get("viewport")), + "elements": api_result.get("elements", []), + "screenshot": raw_result.get("screenshot"), # Keep local screenshot + "screenshot_format": raw_result.get("screenshot_format"), + "error": api_result.get("error"), + } + + def _save_trace_to_file(raw_elements: list[dict[str, Any]], trace_path: str | None = None) -> None: """ Save raw_elements to a JSON file for benchmarking/training @@ -181,54 +297,13 @@ def _snapshot_via_api( # Step 2: Send to server for smart ranking/filtering # Use raw_elements (raw data) instead of elements (processed data) # Server validates API key and applies proprietary ranking logic - payload = { - "raw_elements": raw_result.get("raw_elements", []), # Raw data needed for server processing - "url": raw_result.get("url", ""), - "viewport": raw_result.get("viewport"), - "goal": options.goal, # Optional goal/task description - "options": { - "limit": options.limit, - "filter": options.filter.model_dump() if options.filter else None, - }, - } - - # Check payload size before sending (server has 10MB limit) - payload_json = json.dumps(payload) - payload_size = len(payload_json.encode("utf-8")) - if payload_size > MAX_PAYLOAD_BYTES: - raise ValueError( - f"Payload size ({payload_size / 1024 / 1024:.2f}MB) exceeds server limit " - f"({MAX_PAYLOAD_BYTES / 1024 / 1024:.0f}MB). " - f"Try reducing the number of elements on the page or filtering elements." - ) - - headers = { - "Authorization": f"Bearer {api_key}", - "Content-Type": "application/json", - } + payload = _build_snapshot_payload(raw_result, options) try: - response = requests.post( - f"{api_url}/v1/snapshot", - data=payload_json, # Reuse already-serialized JSON - headers=headers, - timeout=30, - ) - response.raise_for_status() - - api_result = response.json() + api_result = _post_snapshot_to_gateway_sync(payload, api_key, api_url) # Merge API result with local data (screenshot, etc.) - snapshot_data = { - "status": api_result.get("status", "success"), - "timestamp": api_result.get("timestamp"), - "url": api_result.get("url", raw_result.get("url", "")), - "viewport": api_result.get("viewport", raw_result.get("viewport")), - "elements": api_result.get("elements", []), - "screenshot": raw_result.get("screenshot"), # Keep local screenshot - "screenshot_format": raw_result.get("screenshot_format"), - "error": api_result.get("error"), - } + snapshot_data = _merge_api_result_with_local(api_result, raw_result) # Show visual overlay if requested (use API-ranked elements) if options.show_overlay: @@ -247,7 +322,7 @@ def _snapshot_via_api( return Snapshot(**snapshot_data) except requests.exceptions.RequestException as e: - raise RuntimeError(f"API request failed: {e}") + raise RuntimeError(f"API request failed: {e}") from e # ========== Async Snapshot Functions ========== diff --git a/tests/test_backends.py b/tests/test_backends.py index a1c7d90..00e4325 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -833,3 +833,126 @@ async def test_tuple_passthrough(self) -> None: assert x == 300 assert y == 400 + + +class TestBackendExceptions: + """Tests for custom backend exceptions.""" + + def test_extension_diagnostics_from_dict(self) -> None: + """Test ExtensionDiagnostics.from_dict.""" + from sentience.backends.exceptions import ExtensionDiagnostics + + data = { + "sentience_defined": True, + "sentience_snapshot": False, + "url": "https://example.com", + } + diag = ExtensionDiagnostics.from_dict(data) + + assert diag.sentience_defined is True + assert diag.sentience_snapshot is False + assert diag.url == "https://example.com" + assert diag.error is None + + def test_extension_diagnostics_to_dict(self) -> None: + """Test ExtensionDiagnostics.to_dict.""" + from sentience.backends.exceptions import ExtensionDiagnostics + + diag = ExtensionDiagnostics( + sentience_defined=True, + sentience_snapshot=True, + url="https://test.com", + error=None, + ) + result = diag.to_dict() + + assert result["sentience_defined"] is True + assert result["sentience_snapshot"] is True + assert result["url"] == "https://test.com" + + def test_extension_not_loaded_error_from_timeout(self) -> None: + """Test ExtensionNotLoadedError.from_timeout creates helpful message.""" + from sentience.backends.exceptions import ExtensionDiagnostics, ExtensionNotLoadedError + + diag = ExtensionDiagnostics( + sentience_defined=False, + sentience_snapshot=False, + url="https://example.com", + ) + error = ExtensionNotLoadedError.from_timeout(timeout_ms=5000, diagnostics=diag) + + assert error.timeout_ms == 5000 + assert error.diagnostics is diag + assert "5000ms" in str(error) + assert "window.sentience defined: False" in str(error) + assert "get_extension_dir" in str(error) # Contains fix suggestion + + def test_extension_not_loaded_error_with_eval_error(self) -> None: + """Test ExtensionNotLoadedError when diagnostics collection failed.""" + from sentience.backends.exceptions import ExtensionDiagnostics, ExtensionNotLoadedError + + diag = ExtensionDiagnostics(error="Could not evaluate JavaScript") + error = ExtensionNotLoadedError.from_timeout(timeout_ms=3000, diagnostics=diag) + + assert "Could not evaluate JavaScript" in str(error) + + def test_snapshot_error_from_null_result(self) -> None: + """Test SnapshotError.from_null_result creates helpful message.""" + from sentience.backends.exceptions import SnapshotError + + error = SnapshotError.from_null_result(url="https://example.com/page") + + assert error.url == "https://example.com/page" + assert "returned null" in str(error) + assert "example.com/page" in str(error) + + def test_snapshot_error_from_null_result_no_url(self) -> None: + """Test SnapshotError.from_null_result without URL.""" + from sentience.backends.exceptions import SnapshotError + + error = SnapshotError.from_null_result(url=None) + + assert error.url is None + assert "returned null" in str(error) + + def test_action_error_message_format(self) -> None: + """Test ActionError formats message correctly.""" + from sentience.backends.exceptions import ActionError + + error = ActionError( + action="click", + message="Element not found", + coordinates=(100, 200), + ) + + assert error.action == "click" + assert error.coordinates == (100, 200) + assert "click failed" in str(error) + assert "Element not found" in str(error) + + def test_sentience_backend_error_inheritance(self) -> None: + """Test all exceptions inherit from SentienceBackendError.""" + from sentience.backends.exceptions import ( + ActionError, + BackendEvalError, + ExtensionInjectionError, + ExtensionNotLoadedError, + SentienceBackendError, + SnapshotError, + ) + + assert issubclass(ExtensionNotLoadedError, SentienceBackendError) + assert issubclass(ExtensionInjectionError, SentienceBackendError) + assert issubclass(BackendEvalError, SentienceBackendError) + assert issubclass(SnapshotError, SentienceBackendError) + assert issubclass(ActionError, SentienceBackendError) + + def test_extension_injection_error_from_page(self) -> None: + """Test ExtensionInjectionError.from_page.""" + from sentience.backends.exceptions import ExtensionInjectionError + + error = ExtensionInjectionError.from_page("https://secure-site.com") + + assert error.url == "https://secure-site.com" + assert "secure-site.com" in str(error) + assert "Content Security Policy" in str(error)