diff --git a/sentience/agent.py b/sentience/agent.py index deafbd0..ec8433b 100644 --- a/sentience/agent.py +++ b/sentience/agent.py @@ -27,6 +27,7 @@ ) from .protocols import AsyncBrowserProtocol, BrowserProtocol from .snapshot import snapshot, snapshot_async +from .snapshot_diff import SnapshotDiff from .trace_event_builder import TraceEventBuilder if TYPE_CHECKING: @@ -135,6 +136,9 @@ def __init__( # Step counter for tracing self._step_count = 0 + # Previous snapshot for diff detection + self._previous_snapshot: Snapshot | None = None + def _compute_hash(self, text: str) -> str: """Compute SHA256 hash of text.""" return hashlib.sha256(text.encode("utf-8")).hexdigest() @@ -235,13 +239,31 @@ def act( # noqa: C901 if snap.status != "success": raise RuntimeError(f"Snapshot failed: {snap.error}") + # Compute diff_status by comparing with previous snapshot + elements_with_diff = SnapshotDiff.compute_diff_status(snap, self._previous_snapshot) + + # Create snapshot with diff_status populated + snap_with_diff = Snapshot( + status=snap.status, + timestamp=snap.timestamp, + url=snap.url, + viewport=snap.viewport, + elements=elements_with_diff, + screenshot=snap.screenshot, + screenshot_format=snap.screenshot_format, + error=snap.error, + ) + + # Update previous snapshot for next comparison + self._previous_snapshot = snap + # Apply element filtering based on goal - filtered_elements = self.filter_elements(snap, goal) + filtered_elements = self.filter_elements(snap_with_diff, goal) # Emit snapshot trace event if tracer is enabled if self.tracer: - # Build snapshot event data - snapshot_data = TraceEventBuilder.build_snapshot_event(snap) + # Build snapshot event data (use snap_with_diff to include diff_status) + snapshot_data = TraceEventBuilder.build_snapshot_event(snap_with_diff) # Always include screenshot in trace event for studio viewer compatibility # CloudTraceSink will extract and upload screenshots separately, then remove @@ -271,16 +293,16 @@ def act( # noqa: C901 step_id=step_id, ) - # Create filtered snapshot + # Create filtered snapshot (use snap_with_diff to preserve metadata) filtered_snap = Snapshot( - status=snap.status, - timestamp=snap.timestamp, - url=snap.url, - viewport=snap.viewport, + status=snap_with_diff.status, + timestamp=snap_with_diff.timestamp, + url=snap_with_diff.url, + viewport=snap_with_diff.viewport, elements=filtered_elements, - screenshot=snap.screenshot, - screenshot_format=snap.screenshot_format, - error=snap.error, + screenshot=snap_with_diff.screenshot, + screenshot_format=snap_with_diff.screenshot_format, + error=snap_with_diff.error, ) # 2. GROUND: Format elements for LLM context @@ -673,6 +695,9 @@ def __init__( # Step counter for tracing self._step_count = 0 + # Previous snapshot for diff detection + self._previous_snapshot: Snapshot | None = None + def _compute_hash(self, text: str) -> str: """Compute SHA256 hash of text.""" return hashlib.sha256(text.encode("utf-8")).hexdigest() @@ -773,13 +798,31 @@ async def act( # noqa: C901 if snap.status != "success": raise RuntimeError(f"Snapshot failed: {snap.error}") + # Compute diff_status by comparing with previous snapshot + elements_with_diff = SnapshotDiff.compute_diff_status(snap, self._previous_snapshot) + + # Create snapshot with diff_status populated + snap_with_diff = Snapshot( + status=snap.status, + timestamp=snap.timestamp, + url=snap.url, + viewport=snap.viewport, + elements=elements_with_diff, + screenshot=snap.screenshot, + screenshot_format=snap.screenshot_format, + error=snap.error, + ) + + # Update previous snapshot for next comparison + self._previous_snapshot = snap + # Apply element filtering based on goal - filtered_elements = self.filter_elements(snap, goal) + filtered_elements = self.filter_elements(snap_with_diff, goal) # Emit snapshot trace event if tracer is enabled if self.tracer: - # Build snapshot event data - snapshot_data = TraceEventBuilder.build_snapshot_event(snap) + # Build snapshot event data (use snap_with_diff to include diff_status) + snapshot_data = TraceEventBuilder.build_snapshot_event(snap_with_diff) # Always include screenshot in trace event for studio viewer compatibility # CloudTraceSink will extract and upload screenshots separately, then remove @@ -809,16 +852,16 @@ async def act( # noqa: C901 step_id=step_id, ) - # Create filtered snapshot + # Create filtered snapshot (use snap_with_diff to preserve metadata) filtered_snap = Snapshot( - status=snap.status, - timestamp=snap.timestamp, - url=snap.url, - viewport=snap.viewport, + status=snap_with_diff.status, + timestamp=snap_with_diff.timestamp, + url=snap_with_diff.url, + viewport=snap_with_diff.viewport, elements=filtered_elements, - screenshot=snap.screenshot, - screenshot_format=snap.screenshot_format, - error=snap.error, + screenshot=snap_with_diff.screenshot, + screenshot_format=snap_with_diff.screenshot_format, + error=snap_with_diff.error, ) # 2. GROUND: Format elements for LLM context diff --git a/sentience/models.py b/sentience/models.py index db68aa1..7bf48d3 100644 --- a/sentience/models.py +++ b/sentience/models.py @@ -51,6 +51,9 @@ class Element(BaseModel): ml_probability: float | None = None # Confidence score from ONNX model (0.0 - 1.0) ml_score: float | None = None # Raw logit score (optional, for debugging) + # Diff status for frontend Diff Overlay feature + diff_status: Literal["ADDED", "REMOVED", "MODIFIED", "MOVED"] | None = None + class Snapshot(BaseModel): """Snapshot response from extension""" diff --git a/sentience/snapshot_diff.py b/sentience/snapshot_diff.py new file mode 100644 index 0000000..4464837 --- /dev/null +++ b/sentience/snapshot_diff.py @@ -0,0 +1,141 @@ +""" +Snapshot comparison utilities for diff_status detection. + +Implements change detection logic for the Diff Overlay feature. +""" + +from typing import Literal + +from .models import Element, Snapshot + + +class SnapshotDiff: + """ + Utility for comparing snapshots and computing diff_status for elements. + + Implements the logic described in DIFF_STATUS_GAP_ANALYSIS.md: + - ADDED: Element exists in current but not in previous + - REMOVED: Element existed in previous but not in current + - MODIFIED: Element exists in both but has changed + - MOVED: Element exists in both but position changed + """ + + @staticmethod + def _has_bbox_changed(el1: Element, el2: Element, threshold: float = 5.0) -> bool: + """ + Check if element's bounding box has changed significantly. + + Args: + el1: First element + el2: Second element + threshold: Position change threshold in pixels (default: 5.0) + + Returns: + True if position or size changed beyond threshold + """ + return ( + abs(el1.bbox.x - el2.bbox.x) > threshold + or abs(el1.bbox.y - el2.bbox.y) > threshold + or abs(el1.bbox.width - el2.bbox.width) > threshold + or abs(el1.bbox.height - el2.bbox.height) > threshold + ) + + @staticmethod + def _has_content_changed(el1: Element, el2: Element) -> bool: + """ + Check if element's content has changed. + + Args: + el1: First element + el2: Second element + + Returns: + True if text, role, or visual properties changed + """ + # Compare text content + if el1.text != el2.text: + return True + + # Compare role + if el1.role != el2.role: + return True + + # Compare visual cues + if el1.visual_cues.is_primary != el2.visual_cues.is_primary: + return True + if el1.visual_cues.is_clickable != el2.visual_cues.is_clickable: + return True + + return False + + @staticmethod + def compute_diff_status( + current: Snapshot, + previous: Snapshot | None, + ) -> list[Element]: + """ + Compare current snapshot with previous and set diff_status on elements. + + Args: + current: Current snapshot + previous: Previous snapshot (None if this is the first snapshot) + + Returns: + List of elements with diff_status set (includes REMOVED elements from previous) + """ + # If no previous snapshot, all current elements are ADDED + if previous is None: + result = [] + for el in current.elements: + # Create a copy with diff_status set + el_dict = el.model_dump() + el_dict["diff_status"] = "ADDED" + result.append(Element(**el_dict)) + return result + + # Build lookup maps by element ID + current_by_id = {el.id: el for el in current.elements} + previous_by_id = {el.id: el for el in previous.elements} + + current_ids = set(current_by_id.keys()) + previous_ids = set(previous_by_id.keys()) + + result: list[Element] = [] + + # Process current elements + for el in current.elements: + el_dict = el.model_dump() + + if el.id not in previous_ids: + # Element is new - mark as ADDED + el_dict["diff_status"] = "ADDED" + else: + # Element existed before - check for changes + prev_el = previous_by_id[el.id] + + bbox_changed = SnapshotDiff._has_bbox_changed(el, prev_el) + content_changed = SnapshotDiff._has_content_changed(el, prev_el) + + if bbox_changed and content_changed: + # Both position and content changed - mark as MODIFIED + el_dict["diff_status"] = "MODIFIED" + elif bbox_changed: + # Only position changed - mark as MOVED + el_dict["diff_status"] = "MOVED" + elif content_changed: + # Only content changed - mark as MODIFIED + el_dict["diff_status"] = "MODIFIED" + else: + # No change - don't set diff_status (frontend expects undefined) + el_dict["diff_status"] = None + + result.append(Element(**el_dict)) + + # Process removed elements (existed in previous but not in current) + for prev_id in previous_ids - current_ids: + prev_el = previous_by_id[prev_id] + el_dict = prev_el.model_dump() + el_dict["diff_status"] = "REMOVED" + result.append(Element(**el_dict)) + + return result diff --git a/sentience/trace_event_builder.py b/sentience/trace_event_builder.py index 3d4dfb5..560865e 100644 --- a/sentience/trace_event_builder.py +++ b/sentience/trace_event_builder.py @@ -35,9 +35,34 @@ def build_snapshot_event( Returns: Dictionary with snapshot event data """ + # Normalize importance values to importance_score (0-1 range) per snapshot + # Min-max normalization: (value - min) / (max - min) + importance_values = [el.importance for el in snapshot.elements] + + if importance_values: + min_importance = min(importance_values) + max_importance = max(importance_values) + importance_range = max_importance - min_importance + else: + min_importance = 0 + max_importance = 0 + importance_range = 0 + # Include ALL elements with full data for DOM tree display - # Use snap.elements (all elements) not filtered_elements - elements_data = [el.model_dump() for el in snapshot.elements] + # Add importance_score field normalized to [0, 1] + elements_data = [] + for el in snapshot.elements: + el_dict = el.model_dump() + + # Compute normalized importance_score + if importance_range > 0: + importance_score = (el.importance - min_importance) / importance_range + else: + # If all elements have same importance, set to 0.5 + importance_score = 0.5 + + el_dict["importance_score"] = importance_score + elements_data.append(el_dict) return { "url": snapshot.url, diff --git a/sentience/tracing.py b/sentience/tracing.py index fc0405c..0a5fe8b 100644 --- a/sentience/tracing.py +++ b/sentience/tracing.py @@ -4,7 +4,6 @@ Provides abstract interface and JSONL implementation for emitting trace events. """ -import json import time from abc import ABC, abstractmethod from dataclasses import dataclass, field diff --git a/tests/test_importance_score.py b/tests/test_importance_score.py new file mode 100644 index 0000000..05d8ab7 --- /dev/null +++ b/tests/test_importance_score.py @@ -0,0 +1,147 @@ +""" +Tests for importance_score normalization in trace events. +""" + +import pytest + +from sentience.models import BBox, Element, Snapshot, Viewport, VisualCues +from sentience.trace_event_builder import TraceEventBuilder + + +def create_element(element_id: int, importance: int) -> Element: + """Helper to create test elements with specific importance values.""" + return Element( + id=element_id, + role="button", + text=f"Element {element_id}", + importance=importance, + bbox=BBox(x=0, y=0, width=100, height=50), + visual_cues=VisualCues(is_primary=False, is_clickable=True), + ) + + +def create_snapshot(elements: list[Element]) -> Snapshot: + """Helper to create test snapshots.""" + return Snapshot( + status="success", + url="http://example.com", + viewport=Viewport(width=1920, height=1080), + elements=elements, + ) + + +def test_importance_score_normalization_basic(): + """Test basic importance score normalization to [0, 1] range.""" + elements = [ + create_element(1, importance=0), # Min -> 0.0 + create_element(2, importance=500), # Mid -> 0.5 + create_element(3, importance=1000), # Max -> 1.0 + ] + snapshot = create_snapshot(elements) + + event_data = TraceEventBuilder.build_snapshot_event(snapshot) + + assert len(event_data["elements"]) == 3 + + # Check normalization + el1 = event_data["elements"][0] + el2 = event_data["elements"][1] + el3 = event_data["elements"][2] + + assert "importance_score" in el1 + assert "importance_score" in el2 + assert "importance_score" in el3 + + assert el1["importance_score"] == 0.0 # (0 - 0) / (1000 - 0) = 0.0 + assert el2["importance_score"] == 0.5 # (500 - 0) / (1000 - 0) = 0.5 + assert el3["importance_score"] == 1.0 # (1000 - 0) / (1000 - 0) = 1.0 + + +def test_importance_score_with_negative_values(): + """Test normalization with negative importance values.""" + elements = [ + create_element(1, importance=-300), # Min -> 0.0 + create_element(2, importance=500), # Mid -> ~0.44 + create_element(3, importance=1800), # Max -> 1.0 + ] + snapshot = create_snapshot(elements) + + event_data = TraceEventBuilder.build_snapshot_event(snapshot) + + el1 = event_data["elements"][0] + el2 = event_data["elements"][1] + el3 = event_data["elements"][2] + + # Range: 1800 - (-300) = 2100 + assert el1["importance_score"] == 0.0 + assert abs(el2["importance_score"] - 0.380952) < 0.001 # (500 - (-300)) / 2100 ≈ 0.38 + assert el3["importance_score"] == 1.0 + + +def test_importance_score_all_same_values(): + """Test normalization when all elements have same importance.""" + elements = [ + create_element(1, importance=500), + create_element(2, importance=500), + create_element(3, importance=500), + ] + snapshot = create_snapshot(elements) + + event_data = TraceEventBuilder.build_snapshot_event(snapshot) + + # When all have same importance, should default to 0.5 + for el_data in event_data["elements"]: + assert el_data["importance_score"] == 0.5 + + +def test_importance_score_single_element(): + """Test normalization with single element.""" + elements = [create_element(1, importance=500)] + snapshot = create_snapshot(elements) + + event_data = TraceEventBuilder.build_snapshot_event(snapshot) + + # Single element with no range should get 0.5 + assert event_data["elements"][0]["importance_score"] == 0.5 + + +def test_importance_score_empty_snapshot(): + """Test normalization with empty snapshot.""" + snapshot = create_snapshot([]) + + event_data = TraceEventBuilder.build_snapshot_event(snapshot) + + assert event_data["elements"] == [] + assert event_data["element_count"] == 0 + + +def test_importance_score_preserves_original_importance(): + """Test that original importance field is preserved.""" + elements = [ + create_element(1, importance=100), + create_element(2, importance=900), + ] + snapshot = create_snapshot(elements) + + event_data = TraceEventBuilder.build_snapshot_event(snapshot) + + # Original importance should still be present + assert event_data["elements"][0]["importance"] == 100 + assert event_data["elements"][1]["importance"] == 900 + + # And importance_score should be added + assert event_data["elements"][0]["importance_score"] == 0.0 + assert event_data["elements"][1]["importance_score"] == 1.0 + + +def test_importance_score_in_range_0_to_1(): + """Test that all normalized scores are in [0, 1] range.""" + # Create elements with various importance values + elements = [create_element(i, importance=i * 100 - 300) for i in range(20)] + snapshot = create_snapshot(elements) + + event_data = TraceEventBuilder.build_snapshot_event(snapshot) + + for el_data in event_data["elements"]: + score = el_data["importance_score"] + assert 0.0 <= score <= 1.0, f"Score {score} not in [0, 1] range" diff --git a/tests/test_snapshot_diff.py b/tests/test_snapshot_diff.py new file mode 100644 index 0000000..d0e9954 --- /dev/null +++ b/tests/test_snapshot_diff.py @@ -0,0 +1,219 @@ +""" +Tests for snapshot diff functionality (diff_status detection). +""" + +import pytest + +from sentience.models import BBox, Element, Snapshot, Viewport, VisualCues +from sentience.snapshot_diff import SnapshotDiff + + +def create_element( + element_id: int, + role: str = "button", + text: str | None = "Test", + x: float = 100.0, + y: float = 100.0, + width: float = 50.0, + height: float = 20.0, +) -> Element: + """Helper to create test elements.""" + return Element( + id=element_id, + role=role, + text=text, + importance=500, + bbox=BBox(x=x, y=y, width=width, height=height), + visual_cues=VisualCues(is_primary=False, is_clickable=True), + ) + + +def create_snapshot(elements: list[Element], url: str = "http://example.com") -> Snapshot: + """Helper to create test snapshots.""" + return Snapshot( + status="success", + url=url, + viewport=Viewport(width=1920, height=1080), + elements=elements, + ) + + +def test_first_snapshot_all_added(): + """First snapshot should mark all elements as ADDED.""" + elements = [ + create_element(1, text="Button 1"), + create_element(2, text="Button 2"), + ] + current = create_snapshot(elements) + + result = SnapshotDiff.compute_diff_status(current, None) + + assert len(result) == 2 + assert all(el.diff_status == "ADDED" for el in result) + + +def test_unchanged_elements_no_diff_status(): + """Unchanged elements should not have diff_status set.""" + elements = [create_element(1, text="Button 1")] + previous = create_snapshot(elements) + current = create_snapshot(elements) + + result = SnapshotDiff.compute_diff_status(current, previous) + + assert len(result) == 1 + assert result[0].diff_status is None + + +def test_new_element_marked_added(): + """New elements should be marked as ADDED.""" + previous_elements = [create_element(1, text="Button 1")] + current_elements = [ + create_element(1, text="Button 1"), + create_element(2, text="Button 2"), # New element + ] + + previous = create_snapshot(previous_elements) + current = create_snapshot(current_elements) + + result = SnapshotDiff.compute_diff_status(current, previous) + + # Find the new element + new_element = next(el for el in result if el.id == 2) + assert new_element.diff_status == "ADDED" + + # Existing element should have no diff_status + existing_element = next(el for el in result if el.id == 1) + assert existing_element.diff_status is None + + +def test_removed_element_marked_removed(): + """Removed elements should be included in result with REMOVED status.""" + previous_elements = [ + create_element(1, text="Button 1"), + create_element(2, text="Button 2"), + ] + current_elements = [create_element(1, text="Button 1")] + + previous = create_snapshot(previous_elements) + current = create_snapshot(current_elements) + + result = SnapshotDiff.compute_diff_status(current, previous) + + # Should include both current element and removed element + assert len(result) == 2 + + # Find the removed element + removed_element = next(el for el in result if el.id == 2) + assert removed_element.diff_status == "REMOVED" + + +def test_moved_element_marked_moved(): + """Elements that changed position should be marked as MOVED.""" + previous_elements = [create_element(1, x=100.0, y=100.0)] + current_elements = [create_element(1, x=200.0, y=100.0)] # Moved 100px right + + previous = create_snapshot(previous_elements) + current = create_snapshot(current_elements) + + result = SnapshotDiff.compute_diff_status(current, previous) + + assert len(result) == 1 + assert result[0].diff_status == "MOVED" + + +def test_content_changed_marked_modified(): + """Elements that changed content should be marked as MODIFIED.""" + previous_elements = [create_element(1, text="Old Text")] + current_elements = [create_element(1, text="New Text")] + + previous = create_snapshot(previous_elements) + current = create_snapshot(current_elements) + + result = SnapshotDiff.compute_diff_status(current, previous) + + assert len(result) == 1 + assert result[0].diff_status == "MODIFIED" + + +def test_role_changed_marked_modified(): + """Elements that changed role should be marked as MODIFIED.""" + previous_elements = [create_element(1, role="button")] + current_elements = [create_element(1, role="link")] + + previous = create_snapshot(previous_elements) + current = create_snapshot(current_elements) + + result = SnapshotDiff.compute_diff_status(current, previous) + + assert len(result) == 1 + assert result[0].diff_status == "MODIFIED" + + +def test_both_position_and_content_changed_marked_modified(): + """Elements with both position and content changes should be marked as MODIFIED.""" + previous_elements = [create_element(1, text="Old", x=100.0)] + current_elements = [create_element(1, text="New", x=200.0)] + + previous = create_snapshot(previous_elements) + current = create_snapshot(current_elements) + + result = SnapshotDiff.compute_diff_status(current, previous) + + assert len(result) == 1 + assert result[0].diff_status == "MODIFIED" + + +def test_small_position_change_not_detected(): + """Small position changes below threshold should not be detected.""" + previous_elements = [create_element(1, x=100.0, y=100.0)] + current_elements = [create_element(1, x=102.0, y=102.0)] # Moved 2px (< 5px threshold) + + previous = create_snapshot(previous_elements) + current = create_snapshot(current_elements) + + result = SnapshotDiff.compute_diff_status(current, previous) + + assert len(result) == 1 + assert result[0].diff_status is None # No change detected + + +def test_complex_scenario(): + """Test complex scenario with multiple types of changes.""" + previous_elements = [ + create_element(1, text="Unchanged"), + create_element(2, text="Will be removed"), + create_element(3, text="Old text"), + create_element(4, x=100.0), + ] + + current_elements = [ + create_element(1, text="Unchanged"), + # Element 2 removed + create_element(3, text="New text"), # Modified + create_element(4, x=200.0), # Moved + create_element(5, text="New element"), # Added + ] + + previous = create_snapshot(previous_elements) + current = create_snapshot(current_elements) + + result = SnapshotDiff.compute_diff_status(current, previous) + + # Should have 5 elements (4 current + 1 removed) + assert len(result) == 5 + + # Check each element + el1 = next(el for el in result if el.id == 1) + assert el1.diff_status is None # Unchanged + + el2 = next(el for el in result if el.id == 2) + assert el2.diff_status == "REMOVED" + + el3 = next(el for el in result if el.id == 3) + assert el3.diff_status == "MODIFIED" + + el4 = next(el for el in result if el.id == 4) + assert el4.diff_status == "MOVED" + + el5 = next(el for el in result if el.id == 5) + assert el5.diff_status == "ADDED"