From a7de676faddd0e28a7a15fca0fc55139e94d5036 Mon Sep 17 00:00:00 2001 From: SentienceDEV Date: Tue, 13 Jan 2026 16:28:18 -0800 Subject: [PATCH 1/4] show grid overlay --- examples/show_grid_examples.py | 117 ++++++++++++++++++++++++++++++ sentience/models.py | 44 ++++++++++++ sentience/snapshot.py | 125 +++++++++++++++++++++++++++++---- tests/test_grid_bounds.py | 67 ++++++++++++++---- 4 files changed, 326 insertions(+), 27 deletions(-) create mode 100644 examples/show_grid_examples.py diff --git a/examples/show_grid_examples.py b/examples/show_grid_examples.py new file mode 100644 index 0000000..b5dc3aa --- /dev/null +++ b/examples/show_grid_examples.py @@ -0,0 +1,117 @@ +""" +Example: Grid Overlay Visualization + +Demonstrates how to use the grid overlay feature to visualize detected grids +on a webpage, including highlighting specific grids and identifying the dominant group. +""" + +import os +import time + +from sentience import SentienceBrowser, snapshot +from sentience.models import SnapshotOptions + + +def main(): + # Get API key from environment variable (optional - uses free tier if not set) + api_key = os.environ.get("SENTIENCE_API_KEY") + + try: + with SentienceBrowser(api_key=api_key, headless=False) as browser: + # Navigate to a page with grid layouts (e.g., product listings, article feeds) + browser.page.goto("https://example.com/products", wait_until="domcontentloaded") + time.sleep(2) # Wait for page to fully load + + print("=" * 60) + print("Example 1: Show all detected grids") + print("=" * 60) + # Show all grids (all in purple) + snap = snapshot(browser, SnapshotOptions(show_grid=True)) + print(f"✅ Found {len(snap.elements)} elements") + print(" Purple borders appear around all detected grids for 5 seconds") + time.sleep(6) # Wait to see the overlay + + print("\n" + "=" * 60) + print("Example 2: Highlight a specific grid in red") + print("=" * 60) + # Get grid information first + grids = snap.get_grid_bounds() + if grids: + print(f"✅ Found {len(grids)} grids:") + for grid in grids: + print(f" Grid {grid.grid_id}: {grid.item_count} items, " + f"{grid.row_count}x{grid.col_count} rows/cols, " + f"label: {grid.label or 'none'}") + + # Highlight the first grid in red + if len(grids) > 0: + target_grid_id = grids[0].grid_id + print(f"\n Highlighting Grid {target_grid_id} in red...") + snap = snapshot(browser, SnapshotOptions( + show_grid=True, + grid_id=target_grid_id # This grid will be highlighted in red + )) + time.sleep(6) # Wait to see the overlay + else: + print(" ⚠️ No grids detected on this page") + + print("\n" + "=" * 60) + print("Example 3: Highlight the dominant group") + print("=" * 60) + # Find and highlight the dominant grid + grids = snap.get_grid_bounds() + dominant_grid = next((g for g in grids if g.is_dominant), None) + + if dominant_grid: + print(f"✅ Dominant group detected: Grid {dominant_grid.grid_id}") + print(f" Label: {dominant_grid.label or 'none'}") + print(f" Items: {dominant_grid.item_count}") + print(f" Size: {dominant_grid.row_count}x{dominant_grid.col_count}") + print(f"\n Highlighting dominant grid in red...") + snap = snapshot(browser, SnapshotOptions( + show_grid=True, + grid_id=dominant_grid.grid_id # Highlight dominant grid in red + )) + time.sleep(6) # Wait to see the overlay + else: + print(" ⚠️ No dominant group detected") + + print("\n" + "=" * 60) + print("Example 4: Combine element overlay and grid overlay") + print("=" * 60) + # Show both element borders and grid borders simultaneously + snap = snapshot(browser, SnapshotOptions( + show_overlay=True, # Show element borders (green/blue/red) + show_grid=True # Show grid borders (purple/orange/red) + )) + print("✅ Both overlays are now visible:") + print(" - Element borders: Green (regular), Blue (primary), Red (target)") + print(" - Grid borders: Purple (regular), Orange (dominant), Red (target)") + time.sleep(6) # Wait to see the overlay + + print("\n" + "=" * 60) + print("Example 5: Grid information analysis") + print("=" * 60) + # Analyze grid structure + grids = snap.get_grid_bounds() + print(f"✅ Grid Analysis:") + for grid in grids: + dominant_indicator = "⭐ DOMINANT" if grid.is_dominant else "" + print(f"\n Grid {grid.grid_id} {dominant_indicator}:") + print(f" Label: {grid.label or 'none'}") + print(f" Items: {grid.item_count}") + print(f" Size: {grid.row_count} rows × {grid.col_count} cols") + print(f" BBox: ({grid.bbox.x:.0f}, {grid.bbox.y:.0f}) " + f"{grid.bbox.width:.0f}×{grid.bbox.height:.0f}") + print(f" Confidence: {grid.confidence}") + + print("\n✅ All examples completed!") + + except Exception as e: + print(f"❌ Error: {e}") + import traceback + traceback.print_exc() + + +if __name__ == "__main__": + main() diff --git a/sentience/models.py b/sentience/models.py index fb5ebba..6fe7f4a 100644 --- a/sentience/models.py +++ b/sentience/models.py @@ -118,6 +118,7 @@ class GridInfo(BaseModel): label: str | None = ( None # Optional inferred label (e.g., "product_grid", "search_results", "navigation") ) + is_dominant: bool = False # Whether this grid is the dominant group (main content area) class Snapshot(BaseModel): @@ -190,10 +191,16 @@ def get_grid_bounds(self, grid_id: int | None = None) -> list[GridInfo]: grid_infos = [] + # First pass: compute all grid infos and count dominant group elements + grid_dominant_counts = {} for gid, elements_in_grid in sorted(grid_elements.items()): if not elements_in_grid: continue + # Count dominant group elements in this grid + dominant_count = sum(1 for elem in elements_in_grid if elem.in_dominant_group is True) + grid_dominant_counts[gid] = (dominant_count, len(elements_in_grid)) + # Compute bounding box min_x = min(elem.bbox.x for elem in elements_in_grid) min_y = min(elem.bbox.y for elem in elements_in_grid) @@ -226,9 +233,42 @@ def get_grid_bounds(self, grid_id: int | None = None) -> list[GridInfo]: item_count=len(elements_in_grid), confidence=1.0, label=label, + is_dominant=False, # Will be set below ) ) + # Second pass: identify dominant grid + # The grid with the highest count (or highest percentage >= 50%) of dominant group elements + if grid_dominant_counts: + # Find grid with highest absolute count + max_dominant_count = max(count for count, _ in grid_dominant_counts.values()) + if max_dominant_count > 0: + # Find grid(s) with highest count + dominant_grids = [ + gid + for gid, (count, total) in grid_dominant_counts.items() + if count == max_dominant_count + ] + # If multiple grids tie, prefer the one with highest percentage + if len(dominant_grids) > 1: + dominant_grids.sort( + key=lambda gid: ( + grid_dominant_counts[gid][0] / grid_dominant_counts[gid][1] + if grid_dominant_counts[gid][1] > 0 + else 0 + ), + reverse=True, + ) + # Mark the dominant grid + dominant_gid = dominant_grids[0] + # Only mark as dominant if it has >= 50% dominant group elements or >= 3 elements + dominant_count, total_count = grid_dominant_counts[dominant_gid] + if dominant_count >= 3 or (total_count > 0 and dominant_count / total_count >= 0.5): + for grid_info in grid_infos: + if grid_info.grid_id == dominant_gid: + grid_info.is_dominant = True + break + return grid_infos @staticmethod @@ -456,6 +496,10 @@ class SnapshotOptions(BaseModel): trace_path: str | None = None # Path to save trace (default: "trace_{timestamp}.json") goal: str | None = None # Optional goal/task description for the snapshot show_overlay: bool = False # Show visual overlay highlighting elements in browser + show_grid: bool = False # Show visual overlay highlighting detected grids + grid_id: int | None = ( + None # Optional grid ID to show specific grid (only used if show_grid=True) + ) # API credentials (for browser-use integration without SentienceBrowser) sentience_api_key: str | None = None # Sentience API key for Pro/Enterprise features diff --git a/sentience/snapshot.py b/sentience/snapshot.py index 5720f79..274102b 100644 --- a/sentience/snapshot.py +++ b/sentience/snapshot.py @@ -250,6 +250,9 @@ def _snapshot_via_extension( if options.save_trace: _save_trace_to_file(result.get("raw_elements", []), options.trace_path) + # Validate and parse with Pydantic + snapshot_obj = Snapshot(**result) + # Show visual overlay if requested if options.show_overlay: raw_elements = result.get("raw_elements", []) @@ -265,8 +268,29 @@ def _snapshot_via_extension( raw_elements, ) - # Validate and parse with Pydantic - snapshot_obj = Snapshot(**result) + # Show grid overlay if requested + if options.show_grid: + # Get all grids (don't filter by grid_id here - we want to show all but highlight the target) + grids = snapshot_obj.get_grid_bounds(grid_id=None) + if grids: + # Convert GridInfo to dict for JavaScript + grid_dicts = [grid.model_dump() for grid in grids] + # Pass grid_id as targetGridId to highlight it in red + target_grid_id = options.grid_id if options.grid_id is not None else None + browser.page.evaluate( + """ + (grids, targetGridId) => { + if (window.sentience && window.sentience.showGrid) { + window.sentience.showGrid(grids, targetGridId); + } else { + console.warn('[SDK] showGrid not available in extension'); + } + } + """, + grid_dicts, + target_grid_id, + ) + return snapshot_obj @@ -308,6 +332,9 @@ def _snapshot_via_api( # Merge API result with local data (screenshot, etc.) snapshot_data = _merge_api_result_with_local(api_result, raw_result) + # Create snapshot object + snapshot_obj = Snapshot(**snapshot_data) + # Show visual overlay if requested (use API-ranked elements) if options.show_overlay: elements = api_result.get("elements", []) @@ -323,7 +350,29 @@ def _snapshot_via_api( elements, ) - return Snapshot(**snapshot_data) + # Show grid overlay if requested + if options.show_grid: + # Get all grids (don't filter by grid_id here - we want to show all but highlight the target) + grids = snapshot_obj.get_grid_bounds(grid_id=None) + if grids: + grid_dicts = [grid.model_dump() for grid in grids] + # Pass grid_id as targetGridId to highlight it in red + target_grid_id = options.grid_id if options.grid_id is not None else None + browser.page.evaluate( + """ + (grids, targetGridId) => { + if (window.sentience && window.sentience.showGrid) { + window.sentience.showGrid(grids, targetGridId); + } else { + console.warn('[SDK] showGrid not available in extension'); + } + } + """, + grid_dicts, + target_grid_id, + ) + + return snapshot_obj except requests.exceptions.RequestException as e: raise RuntimeError(f"API request failed: {e}") from e @@ -440,6 +489,18 @@ async def _snapshot_via_extension_async( if options.save_trace: _save_trace_to_file(result.get("raw_elements", []), options.trace_path) + # Extract screenshot_format from data URL if not provided by extension + if result.get("screenshot") and not result.get("screenshot_format"): + screenshot_data_url = result.get("screenshot", "") + if screenshot_data_url.startswith("data:image/"): + # Extract format from "data:image/jpeg;base64,..." or "data:image/png;base64,..." + format_match = screenshot_data_url.split(";")[0].split("/")[-1] + if format_match in ["jpeg", "jpg", "png"]: + result["screenshot_format"] = "jpeg" if format_match in ["jpeg", "jpg"] else "png" + + # Validate and parse with Pydantic + snapshot_obj = Snapshot(**result) + # Show visual overlay if requested if options.show_overlay: raw_elements = result.get("raw_elements", []) @@ -455,17 +516,28 @@ async def _snapshot_via_extension_async( raw_elements, ) - # Extract screenshot_format from data URL if not provided by extension - if result.get("screenshot") and not result.get("screenshot_format"): - screenshot_data_url = result.get("screenshot", "") - if screenshot_data_url.startswith("data:image/"): - # Extract format from "data:image/jpeg;base64,..." or "data:image/png;base64,..." - format_match = screenshot_data_url.split(";")[0].split("/")[-1] - if format_match in ["jpeg", "jpg", "png"]: - result["screenshot_format"] = "jpeg" if format_match in ["jpeg", "jpg"] else "png" + # Show grid overlay if requested + if options.show_grid: + # Get all grids (don't filter by grid_id here - we want to show all but highlight the target) + grids = snapshot_obj.get_grid_bounds(grid_id=None) + if grids: + grid_dicts = [grid.model_dump() for grid in grids] + # Pass grid_id as targetGridId to highlight it in red + target_grid_id = options.grid_id if options.grid_id is not None else None + await browser.page.evaluate( + """ + (grids, targetGridId) => { + if (window.sentience && window.sentience.showGrid) { + window.sentience.showGrid(grids, targetGridId); + } else { + console.warn('[SDK] showGrid not available in extension'); + } + } + """, + grid_dicts, + target_grid_id, + ) - # Validate and parse with Pydantic - snapshot_obj = Snapshot(**result) return snapshot_obj @@ -584,6 +656,9 @@ async def _snapshot_via_api_async( "error": api_result.get("error"), } + # Create snapshot object + snapshot_obj = Snapshot(**snapshot_data) + # Show visual overlay if requested if options.show_overlay: elements = api_result.get("elements", []) @@ -599,7 +674,29 @@ async def _snapshot_via_api_async( elements, ) - return Snapshot(**snapshot_data) + # Show grid overlay if requested + if options.show_grid: + # Get all grids (don't filter by grid_id here - we want to show all but highlight the target) + grids = snapshot_obj.get_grid_bounds(grid_id=None) + if grids: + grid_dicts = [grid.model_dump() for grid in grids] + # Pass grid_id as targetGridId to highlight it in red + target_grid_id = options.grid_id if options.grid_id is not None else None + await browser.page.evaluate( + """ + (grids, targetGridId) => { + if (window.sentience && window.sentience.showGrid) { + window.sentience.showGrid(grids, targetGridId); + } else { + console.warn('[SDK] showGrid not available in extension'); + } + } + """, + grid_dicts, + target_grid_id, + ) + + return snapshot_obj except ImportError: # Fallback to requests if httpx not available (shouldn't happen in async context) raise RuntimeError( diff --git a/tests/test_grid_bounds.py b/tests/test_grid_bounds.py index 9952970..93bb526 100644 --- a/tests/test_grid_bounds.py +++ b/tests/test_grid_bounds.py @@ -221,19 +221,40 @@ def test_label_inference_product_grid(self): """Test that product grids get labeled correctly""" elements = [ create_test_element( - 1, 10, 20, 100, 50, grid_id=0, row_index=0, col_index=0, + 1, + 10, + 20, + 100, + 50, + grid_id=0, + row_index=0, + col_index=0, text="Wireless Headphones $50", - href="https://example.com/product/headphones" + href="https://example.com/product/headphones", ), create_test_element( - 2, 120, 20, 100, 50, grid_id=0, row_index=0, col_index=1, + 2, + 120, + 20, + 100, + 50, + grid_id=0, + row_index=0, + col_index=1, text="Bluetooth Speaker $30", - href="https://example.com/product/speaker" + href="https://example.com/product/speaker", ), create_test_element( - 3, 10, 80, 100, 50, grid_id=0, row_index=1, col_index=0, + 3, + 10, + 80, + 100, + 50, + grid_id=0, + row_index=1, + col_index=0, text="USB-C Cable $10", - href="https://example.com/product/cable" + href="https://example.com/product/cable", ), ] @@ -251,12 +272,26 @@ def test_label_inference_article_feed(self): """Test that article feeds get labeled correctly""" elements = [ create_test_element( - 1, 10, 20, 100, 50, grid_id=0, row_index=0, col_index=0, - text="Breaking News 2 hours ago" + 1, + 10, + 20, + 100, + 50, + grid_id=0, + row_index=0, + col_index=0, + text="Breaking News 2 hours ago", ), create_test_element( - 2, 10, 80, 100, 50, grid_id=0, row_index=1, col_index=0, - text="Tech Update 3 days ago" + 2, + 10, + 80, + 100, + 50, + grid_id=0, + row_index=1, + col_index=0, + text="Tech Update 3 days ago", ), ] @@ -273,9 +308,15 @@ def test_label_inference_article_feed(self): def test_label_inference_navigation(self): """Test that navigation grids get labeled correctly""" elements = [ - create_test_element(1, 10, 20, 80, 30, grid_id=0, row_index=0, col_index=0, text="Home"), - create_test_element(2, 100, 20, 80, 30, grid_id=0, row_index=0, col_index=1, text="About"), - create_test_element(3, 190, 20, 80, 30, grid_id=0, row_index=0, col_index=2, text="Contact"), + create_test_element( + 1, 10, 20, 80, 30, grid_id=0, row_index=0, col_index=0, text="Home" + ), + create_test_element( + 2, 100, 20, 80, 30, grid_id=0, row_index=0, col_index=1, text="About" + ), + create_test_element( + 3, 190, 20, 80, 30, grid_id=0, row_index=0, col_index=2, text="Contact" + ), ] snapshot = Snapshot( From ad6c8ba50559a218b996a8755ef8b6c09ee298f3 Mon Sep 17 00:00:00 2001 From: SentienceDEV Date: Tue, 13 Jan 2026 16:43:28 -0800 Subject: [PATCH 2/4] example for showing grid overlay --- examples/show_grid_examples.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/examples/show_grid_examples.py b/examples/show_grid_examples.py index b5dc3aa..da8ca60 100644 --- a/examples/show_grid_examples.py +++ b/examples/show_grid_examples.py @@ -15,18 +15,22 @@ def main(): # Get API key from environment variable (optional - uses free tier if not set) api_key = os.environ.get("SENTIENCE_API_KEY") + + # Use VPS IP directly if domain is not configured + # Replace with your actual domain once DNS is set up: api_url="https://api.sentienceapi.com" + api_url = os.environ.get("SENTIENCE_API_URL", "http://15.204.243.91:9000") try: - with SentienceBrowser(api_key=api_key, headless=False) as browser: + with SentienceBrowser(api_key=api_key, api_url=api_url, headless=False) as browser: # Navigate to a page with grid layouts (e.g., product listings, article feeds) - browser.page.goto("https://example.com/products", wait_until="domcontentloaded") + browser.page.goto("https://example.com", wait_until="domcontentloaded") time.sleep(2) # Wait for page to fully load print("=" * 60) print("Example 1: Show all detected grids") print("=" * 60) # Show all grids (all in purple) - snap = snapshot(browser, SnapshotOptions(show_grid=True)) + snap = snapshot(browser, SnapshotOptions(show_grid=True, use_api=True)) print(f"✅ Found {len(snap.elements)} elements") print(" Purple borders appear around all detected grids for 5 seconds") time.sleep(6) # Wait to see the overlay From 89e58f87309911d98db8639a2d8ec64d28063f24 Mon Sep 17 00:00:00 2001 From: SentienceDEV Date: Tue, 13 Jan 2026 16:56:37 -0800 Subject: [PATCH 3/4] fix bad code --- examples/show_grid_examples.py | 54 +++++++++++++++++++++------------- sentience/agent_runtime.py | 1 - 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/examples/show_grid_examples.py b/examples/show_grid_examples.py index da8ca60..c428bab 100644 --- a/examples/show_grid_examples.py +++ b/examples/show_grid_examples.py @@ -15,7 +15,7 @@ def main(): # Get API key from environment variable (optional - uses free tier if not set) api_key = os.environ.get("SENTIENCE_API_KEY") - + # Use VPS IP directly if domain is not configured # Replace with your actual domain once DNS is set up: api_url="https://api.sentienceapi.com" api_url = os.environ.get("SENTIENCE_API_URL", "http://15.204.243.91:9000") @@ -43,18 +43,23 @@ def main(): if grids: print(f"✅ Found {len(grids)} grids:") for grid in grids: - print(f" Grid {grid.grid_id}: {grid.item_count} items, " - f"{grid.row_count}x{grid.col_count} rows/cols, " - f"label: {grid.label or 'none'}") - + print( + f" Grid {grid.grid_id}: {grid.item_count} items, " + f"{grid.row_count}x{grid.col_count} rows/cols, " + f"label: {grid.label or 'none'}" + ) + # Highlight the first grid in red if len(grids) > 0: target_grid_id = grids[0].grid_id print(f"\n Highlighting Grid {target_grid_id} in red...") - snap = snapshot(browser, SnapshotOptions( - show_grid=True, - grid_id=target_grid_id # This grid will be highlighted in red - )) + snap = snapshot( + browser, + SnapshotOptions( + show_grid=True, + grid_id=target_grid_id, # This grid will be highlighted in red + ), + ) time.sleep(6) # Wait to see the overlay else: print(" ⚠️ No grids detected on this page") @@ -65,17 +70,20 @@ def main(): # Find and highlight the dominant grid grids = snap.get_grid_bounds() dominant_grid = next((g for g in grids if g.is_dominant), None) - + if dominant_grid: print(f"✅ Dominant group detected: Grid {dominant_grid.grid_id}") print(f" Label: {dominant_grid.label or 'none'}") print(f" Items: {dominant_grid.item_count}") print(f" Size: {dominant_grid.row_count}x{dominant_grid.col_count}") print(f"\n Highlighting dominant grid in red...") - snap = snapshot(browser, SnapshotOptions( - show_grid=True, - grid_id=dominant_grid.grid_id # Highlight dominant grid in red - )) + snap = snapshot( + browser, + SnapshotOptions( + show_grid=True, + grid_id=dominant_grid.grid_id, # Highlight dominant grid in red + ), + ) time.sleep(6) # Wait to see the overlay else: print(" ⚠️ No dominant group detected") @@ -84,10 +92,13 @@ def main(): print("Example 4: Combine element overlay and grid overlay") print("=" * 60) # Show both element borders and grid borders simultaneously - snap = snapshot(browser, SnapshotOptions( - show_overlay=True, # Show element borders (green/blue/red) - show_grid=True # Show grid borders (purple/orange/red) - )) + snap = snapshot( + browser, + SnapshotOptions( + show_overlay=True, # Show element borders (green/blue/red) + show_grid=True, # Show grid borders (purple/orange/red) + ), + ) print("✅ Both overlays are now visible:") print(" - Element borders: Green (regular), Blue (primary), Red (target)") print(" - Grid borders: Purple (regular), Orange (dominant), Red (target)") @@ -105,8 +116,10 @@ def main(): print(f" Label: {grid.label or 'none'}") print(f" Items: {grid.item_count}") print(f" Size: {grid.row_count} rows × {grid.col_count} cols") - print(f" BBox: ({grid.bbox.x:.0f}, {grid.bbox.y:.0f}) " - f"{grid.bbox.width:.0f}×{grid.bbox.height:.0f}") + print( + f" BBox: ({grid.bbox.x:.0f}, {grid.bbox.y:.0f}) " + f"{grid.bbox.width:.0f}×{grid.bbox.height:.0f}" + ) print(f" Confidence: {grid.confidence}") print("\n✅ All examples completed!") @@ -114,6 +127,7 @@ def main(): except Exception as e: print(f"❌ Error: {e}") import traceback + traceback.print_exc() diff --git a/sentience/agent_runtime.py b/sentience/agent_runtime.py index ae1c437..3679397 100644 --- a/sentience/agent_runtime.py +++ b/sentience/agent_runtime.py @@ -343,7 +343,6 @@ def assert_done( True if task is complete (assertion passed), False otherwise """ ok = self.assertTrue(predicate, label=label, required=True) - if ok: self._task_done = True self._task_done_label = label From a65c27d37eb670109ceb324aee9b687807b3aadf Mon Sep 17 00:00:00 2001 From: SentienceDEV Date: Tue, 13 Jan 2026 16:58:38 -0800 Subject: [PATCH 4/4] fix bad code --- sentience/agent_runtime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentience/agent_runtime.py b/sentience/agent_runtime.py index 3679397..a6364bb 100644 --- a/sentience/agent_runtime.py +++ b/sentience/agent_runtime.py @@ -342,7 +342,7 @@ def assert_done( Returns: True if task is complete (assertion passed), False otherwise """ - ok = self.assertTrue(predicate, label=label, required=True) + ok = self.assert_(predicate, label=label, required=True) if ok: self._task_done = True self._task_done_label = label