diff --git a/sentience/__init__.py b/sentience/__init__.py index 61526a6..76458db 100644 --- a/sentience/__init__.py +++ b/sentience/__init__.py @@ -14,9 +14,6 @@ from .cloud_tracing import CloudTraceSink, SentienceLogger from .conversational_agent import ConversationalAgent from .expect import expect - -# Formatting (v0.12.0+) -from .formatting import format_snapshot_for_llm from .generator import ScriptGenerator, generate from .inspector import Inspector, inspect from .llm_provider import ( @@ -62,6 +59,7 @@ from .tracing import JsonlTraceSink, TraceEvent, Tracer, TraceSink # Utilities (v0.12.0+) +# Import from utils package (re-exports from submodules for backward compatibility) from .utils import ( canonical_snapshot_loose, canonical_snapshot_strict, @@ -69,6 +67,9 @@ save_storage_state, sha256_digest, ) + +# Formatting (v0.12.0+) +from .utils.formatting import format_snapshot_for_llm from .wait import wait_for __version__ = "0.91.1" diff --git a/sentience/action_executor.py b/sentience/action_executor.py new file mode 100644 index 0000000..104e255 --- /dev/null +++ b/sentience/action_executor.py @@ -0,0 +1,191 @@ +""" +Action Executor for Sentience Agent. + +Handles parsing and execution of action commands (CLICK, TYPE, PRESS, FINISH). +This separates action execution concerns from LLM interaction. +""" + +import re +from typing import Any + +from .actions import click, click_async, press, press_async, type_text, type_text_async +from .browser import AsyncSentienceBrowser, SentienceBrowser +from .models import Snapshot + + +class ActionExecutor: + """ + Executes actions and handles parsing of action command strings. + + This class encapsulates all action execution logic, making it easier to: + - Test action execution independently + - Add new action types in one place + - Handle action parsing errors consistently + """ + + def __init__(self, browser: SentienceBrowser | AsyncSentienceBrowser): + """ + Initialize action executor. + + Args: + browser: SentienceBrowser or AsyncSentienceBrowser instance + """ + self.browser = browser + self._is_async = isinstance(browser, AsyncSentienceBrowser) + + def execute(self, action_str: str, snap: Snapshot) -> dict[str, Any]: + """ + Parse action string and execute SDK call (synchronous). + + Args: + action_str: Action string from LLM (e.g., "CLICK(42)", "TYPE(15, \"text\")") + snap: Current snapshot (for context, currently unused but kept for API consistency) + + Returns: + Execution result dictionary with keys: + - success: bool + - action: str (e.g., "click", "type", "press", "finish") + - element_id: Optional[int] (for click/type actions) + - text: Optional[str] (for type actions) + - key: Optional[str] (for press actions) + - outcome: Optional[str] (action outcome) + - url_changed: Optional[bool] (for click actions) + - error: Optional[str] (if action failed) + - message: Optional[str] (for finish action) + + Raises: + ValueError: If action format is unknown + RuntimeError: If called on async browser (use execute_async instead) + """ + if self._is_async: + raise RuntimeError( + "ActionExecutor.execute() called on async browser. Use execute_async() instead." + ) + + # Parse CLICK(42) + if match := re.match(r"CLICK\s*\(\s*(\d+)\s*\)", action_str, re.IGNORECASE): + element_id = int(match.group(1)) + result = click(self.browser, element_id) # type: ignore + return { + "success": result.success, + "action": "click", + "element_id": element_id, + "outcome": result.outcome, + "url_changed": result.url_changed, + } + + # Parse TYPE(42, "hello world") + elif match := re.match( + r'TYPE\s*\(\s*(\d+)\s*,\s*["\']([^"\']*)["\']\s*\)', + action_str, + re.IGNORECASE, + ): + element_id = int(match.group(1)) + text = match.group(2) + result = type_text(self.browser, element_id, text) # type: ignore + return { + "success": result.success, + "action": "type", + "element_id": element_id, + "text": text, + "outcome": result.outcome, + } + + # Parse PRESS("Enter") + elif match := re.match(r'PRESS\s*\(\s*["\']([^"\']+)["\']\s*\)', action_str, re.IGNORECASE): + key = match.group(1) + result = press(self.browser, key) # type: ignore + return { + "success": result.success, + "action": "press", + "key": key, + "outcome": result.outcome, + } + + # Parse FINISH() + elif re.match(r"FINISH\s*\(\s*\)", action_str, re.IGNORECASE): + return { + "success": True, + "action": "finish", + "message": "Task marked as complete", + } + + else: + raise ValueError( + f"Unknown action format: {action_str}\n" + f'Expected: CLICK(id), TYPE(id, "text"), PRESS("key"), or FINISH()' + ) + + async def execute_async(self, action_str: str, snap: Snapshot) -> dict[str, Any]: + """ + Parse action string and execute SDK call (asynchronous). + + Args: + action_str: Action string from LLM (e.g., "CLICK(42)", "TYPE(15, \"text\")") + snap: Current snapshot (for context, currently unused but kept for API consistency) + + Returns: + Execution result dictionary (same format as execute()) + + Raises: + ValueError: If action format is unknown + RuntimeError: If called on sync browser (use execute() instead) + """ + if not self._is_async: + raise RuntimeError( + "ActionExecutor.execute_async() called on sync browser. Use execute() instead." + ) + + # Parse CLICK(42) + if match := re.match(r"CLICK\s*\(\s*(\d+)\s*\)", action_str, re.IGNORECASE): + element_id = int(match.group(1)) + result = await click_async(self.browser, element_id) # type: ignore + return { + "success": result.success, + "action": "click", + "element_id": element_id, + "outcome": result.outcome, + "url_changed": result.url_changed, + } + + # Parse TYPE(42, "hello world") + elif match := re.match( + r'TYPE\s*\(\s*(\d+)\s*,\s*["\']([^"\']*)["\']\s*\)', + action_str, + re.IGNORECASE, + ): + element_id = int(match.group(1)) + text = match.group(2) + result = await type_text_async(self.browser, element_id, text) # type: ignore + return { + "success": result.success, + "action": "type", + "element_id": element_id, + "text": text, + "outcome": result.outcome, + } + + # Parse PRESS("Enter") + elif match := re.match(r'PRESS\s*\(\s*["\']([^"\']+)["\']\s*\)', action_str, re.IGNORECASE): + key = match.group(1) + result = await press_async(self.browser, key) # type: ignore + return { + "success": result.success, + "action": "press", + "key": key, + "outcome": result.outcome, + } + + # Parse FINISH() + elif re.match(r"FINISH\s*\(\s*\)", action_str, re.IGNORECASE): + return { + "success": True, + "action": "finish", + "message": "Task marked as complete", + } + + else: + raise ValueError( + f"Unknown action format: {action_str}\n" + f'Expected: CLICK(id), TYPE(id, "text"), PRESS("key"), or FINISH()' + ) diff --git a/sentience/agent.py b/sentience/agent.py index 585ab48..3238f30 100644 --- a/sentience/agent.py +++ b/sentience/agent.py @@ -5,15 +5,15 @@ import asyncio import hashlib -import re import time from typing import TYPE_CHECKING, Any, Optional -from .actions import click, click_async, press, press_async, type_text, type_text_async +from .action_executor import ActionExecutor from .agent_config import AgentConfig from .base_agent import BaseAgent, BaseAgentAsync from .browser import AsyncSentienceBrowser, SentienceBrowser from .element_filter import ElementFilter +from .llm_interaction_handler import LLMInteractionHandler from .llm_provider import LLMProvider, LLMResponse from .models import ( ActionHistory, @@ -83,6 +83,10 @@ def __init__( self.tracer = tracer self.config = config or AgentConfig() + # Initialize handlers + self.llm_handler = LLMInteractionHandler(llm) + self.action_executor = ActionExecutor(browser) + # Screenshot sequence counter # Execution history self.history: list[dict[str, Any]] = [] @@ -241,10 +245,10 @@ def act( # noqa: C901 ) # 2. GROUND: Format elements for LLM context - context = self._build_context(filtered_snap, goal) + context = self.llm_handler.build_context(filtered_snap, goal) # 3. THINK: Query LLM for next action - llm_response = self._query_llm(context, goal) + llm_response = self.llm_handler.query_llm(context, goal) # Emit LLM query trace event if tracer is enabled if self.tracer: @@ -266,10 +270,10 @@ def act( # noqa: C901 self._track_tokens(goal, llm_response) # Parse action from LLM response - action_str = self._extract_action_from_response(llm_response.content) + action_str = self.llm_handler.extract_action(llm_response.content) # 4. EXECUTE: Parse and run action - result_dict = self._execute_action(action_str, filtered_snap) + result_dict = self.action_executor.execute(action_str, filtered_snap) duration_ms = int((time.time() - start_time) * 1000) @@ -465,187 +469,6 @@ def act( # noqa: C901 ) raise RuntimeError(f"Failed after {max_retries} retries: {e}") - def _build_context(self, snap: Snapshot, goal: str) -> str: - """ - Convert snapshot elements to token-efficient prompt string - - Format: [ID] "text" {cues} @ (x,y) (Imp:score) - - Args: - snap: Snapshot object - goal: User goal (for context) - - Returns: - Formatted element context string - """ - lines = [] - # Note: elements are already filtered by filter_elements() in act() - for el in snap.elements: - # Extract visual cues - cues = [] - if el.visual_cues.is_primary: - cues.append("PRIMARY") - if el.visual_cues.is_clickable: - cues.append("CLICKABLE") - if el.visual_cues.background_color_name: - cues.append(f"color:{el.visual_cues.background_color_name}") - - # Format element line - cues_str = f" {{{','.join(cues)}}}" if cues else "" - text_preview = ( - (el.text[:50] + "...") if el.text and len(el.text) > 50 else (el.text or "") - ) - - lines.append( - f'[{el.id}] <{el.role}> "{text_preview}"{cues_str} ' - f"@ ({int(el.bbox.x)},{int(el.bbox.y)}) (Imp:{el.importance})" - ) - - return "\n".join(lines) - - def _extract_action_from_response(self, response: str) -> str: - """ - Extract action command from LLM response, handling cases where - the LLM adds extra explanation despite instructions. - - Args: - response: Raw LLM response text - - Returns: - Cleaned action command string - """ - import re - - # Remove markdown code blocks if present - response = re.sub(r"```[\w]*\n?", "", response) - response = response.strip() - - # Try to find action patterns in the response - # Pattern matches: CLICK(123), TYPE(123, "text"), PRESS("key"), FINISH() - action_pattern = r'(CLICK\s*\(\s*\d+\s*\)|TYPE\s*\(\s*\d+\s*,\s*["\'].*?["\']\s*\)|PRESS\s*\(\s*["\'].*?["\']\s*\)|FINISH\s*\(\s*\))' - - match = re.search(action_pattern, response, re.IGNORECASE) - if match: - return match.group(1) - - # If no pattern match, return the original response (will likely fail parsing) - return response - - def _query_llm(self, dom_context: str, goal: str) -> LLMResponse: - """ - Query LLM with standardized prompt template - - Args: - dom_context: Formatted element context - goal: User goal - - Returns: - LLMResponse from LLM provider - """ - system_prompt = f"""You are an AI web automation agent. - -GOAL: {goal} - -VISIBLE ELEMENTS (sorted by importance): -{dom_context} - -VISUAL CUES EXPLAINED: -- {{PRIMARY}}: Main call-to-action element on the page -- {{CLICKABLE}}: Element is clickable -- {{color:X}}: Background color name - -CRITICAL RESPONSE FORMAT: -You MUST respond with ONLY ONE of these exact action formats: -- CLICK(id) - Click element by ID -- TYPE(id, "text") - Type text into element -- PRESS("key") - Press keyboard key (Enter, Escape, Tab, ArrowDown, etc) -- FINISH() - Task complete - -DO NOT include any explanation, reasoning, or natural language. -DO NOT use markdown formatting or code blocks. -DO NOT say "The next step is..." or anything similar. - -CORRECT Examples: -CLICK(42) -TYPE(15, "magic mouse") -PRESS("Enter") -FINISH() - -INCORRECT Examples (DO NOT DO THIS): -"The next step is to click..." -"I will type..." -```CLICK(42)``` -""" - - user_prompt = "Return the single action command:" - - return self.llm.generate(system_prompt, user_prompt, temperature=0.0) - - def _execute_action(self, action_str: str, snap: Snapshot) -> dict[str, Any]: - """ - Parse action string and execute SDK call - - Args: - action_str: Action string from LLM (e.g., "CLICK(42)") - snap: Current snapshot (for context) - - Returns: - Execution result dictionary - """ - # Parse CLICK(42) - if match := re.match(r"CLICK\s*\(\s*(\d+)\s*\)", action_str, re.IGNORECASE): - element_id = int(match.group(1)) - result = click(self.browser, element_id) - return { - "success": result.success, - "action": "click", - "element_id": element_id, - "outcome": result.outcome, - "url_changed": result.url_changed, - } - - # Parse TYPE(42, "hello world") - elif match := re.match( - r'TYPE\s*\(\s*(\d+)\s*,\s*["\']([^"\']*)["\']\s*\)', - action_str, - re.IGNORECASE, - ): - element_id = int(match.group(1)) - text = match.group(2) - result = type_text(self.browser, element_id, text) - return { - "success": result.success, - "action": "type", - "element_id": element_id, - "text": text, - "outcome": result.outcome, - } - - # Parse PRESS("Enter") - elif match := re.match(r'PRESS\s*\(\s*["\']([^"\']+)["\']\s*\)', action_str, re.IGNORECASE): - key = match.group(1) - result = press(self.browser, key) - return { - "success": result.success, - "action": "press", - "key": key, - "outcome": result.outcome, - } - - # Parse FINISH() - elif re.match(r"FINISH\s*\(\s*\)", action_str, re.IGNORECASE): - return { - "success": True, - "action": "finish", - "message": "Task marked as complete", - } - - else: - raise ValueError( - f"Unknown action format: {action_str}\n" - f'Expected: CLICK(id), TYPE(id, "text"), PRESS("key"), or FINISH()' - ) - def _track_tokens(self, goal: str, llm_response: LLMResponse): """ Track token usage for analytics @@ -772,6 +595,10 @@ def __init__( self.tracer = tracer self.config = config or AgentConfig() + # Initialize handlers + self.llm_handler = LLMInteractionHandler(llm) + self.action_executor = ActionExecutor(browser) + # Screenshot sequence counter # Execution history self.history: list[dict[str, Any]] = [] @@ -930,10 +757,10 @@ async def act( # noqa: C901 ) # 2. GROUND: Format elements for LLM context - context = self._build_context(filtered_snap, goal) + context = self.llm_handler.build_context(filtered_snap, goal) # 3. THINK: Query LLM for next action - llm_response = self._query_llm(context, goal) + llm_response = self.llm_handler.query_llm(context, goal) # Emit LLM query trace event if tracer is enabled if self.tracer: @@ -955,10 +782,10 @@ async def act( # noqa: C901 self._track_tokens(goal, llm_response) # Parse action from LLM response - action_str = self._extract_action_from_response(llm_response.content) + action_str = self.llm_handler.extract_action(llm_response.content) # 4. EXECUTE: Parse and run action - result_dict = await self._execute_action(action_str, filtered_snap) + result_dict = await self.action_executor.execute_async(action_str, filtered_snap) duration_ms = int((time.time() - start_time) * 1000) @@ -1154,156 +981,6 @@ async def act( # noqa: C901 ) raise RuntimeError(f"Failed after {max_retries} retries: {e}") - def _build_context(self, snap: Snapshot, goal: str) -> str: - """Convert snapshot elements to token-efficient prompt string (same as sync version)""" - lines = [] - # Note: elements are already filtered by filter_elements() in act() - for el in snap.elements: - # Extract visual cues - cues = [] - if el.visual_cues.is_primary: - cues.append("PRIMARY") - if el.visual_cues.is_clickable: - cues.append("CLICKABLE") - if el.visual_cues.background_color_name: - cues.append(f"color:{el.visual_cues.background_color_name}") - - # Format element line - cues_str = f" {{{','.join(cues)}}}" if cues else "" - text_preview = ( - (el.text[:50] + "...") if el.text and len(el.text) > 50 else (el.text or "") - ) - - lines.append( - f'[{el.id}] <{el.role}> "{text_preview}"{cues_str} ' - f"@ ({int(el.bbox.x)},{int(el.bbox.y)}) (Imp:{el.importance})" - ) - - return "\n".join(lines) - - def _extract_action_from_response(self, response: str) -> str: - """Extract action command from LLM response (same as sync version)""" - # Remove markdown code blocks if present - response = re.sub(r"```[\w]*\n?", "", response) - response = response.strip() - - # Try to find action patterns in the response - # Pattern matches: CLICK(123), TYPE(123, "text"), PRESS("key"), FINISH() - action_pattern = r'(CLICK\s*\(\s*\d+\s*\)|TYPE\s*\(\s*\d+\s*,\s*["\'].*?["\']\s*\)|PRESS\s*\(\s*["\'].*?["\']\s*\)|FINISH\s*\(\s*\))' - - match = re.search(action_pattern, response, re.IGNORECASE) - if match: - return match.group(1) - - # If no pattern match, return the original response (will likely fail parsing) - return response - - def _query_llm(self, dom_context: str, goal: str) -> LLMResponse: - """Query LLM with standardized prompt template (same as sync version)""" - system_prompt = f"""You are an AI web automation agent. - -GOAL: {goal} - -VISIBLE ELEMENTS (sorted by importance): -{dom_context} - -VISUAL CUES EXPLAINED: -- {{PRIMARY}}: Main call-to-action element on the page -- {{CLICKABLE}}: Element is clickable -- {{color:X}}: Background color name - -CRITICAL RESPONSE FORMAT: -You MUST respond with ONLY ONE of these exact action formats: -- CLICK(id) - Click element by ID -- TYPE(id, "text") - Type text into element -- PRESS("key") - Press keyboard key (Enter, Escape, Tab, ArrowDown, etc) -- FINISH() - Task complete - -DO NOT include any explanation, reasoning, or natural language. -DO NOT use markdown formatting or code blocks. -DO NOT say "The next step is..." or anything similar. - -CORRECT Examples: -CLICK(42) -TYPE(15, "magic mouse") -PRESS("Enter") -FINISH() - -INCORRECT Examples (DO NOT DO THIS): -"The next step is to click..." -"I will type..." -```CLICK(42)``` -""" - - user_prompt = "Return the single action command:" - - return self.llm.generate(system_prompt, user_prompt, temperature=0.0) - - async def _execute_action(self, action_str: str, snap: Snapshot) -> dict[str, Any]: - """ - Parse action string and execute SDK call (async) - - Args: - action_str: Action string from LLM (e.g., "CLICK(42)") - snap: Current snapshot (for context) - - Returns: - Execution result dictionary - """ - # Parse CLICK(42) - if match := re.match(r"CLICK\s*\(\s*(\d+)\s*\)", action_str, re.IGNORECASE): - element_id = int(match.group(1)) - result = await click_async(self.browser, element_id) - return { - "success": result.success, - "action": "click", - "element_id": element_id, - "outcome": result.outcome, - "url_changed": result.url_changed, - } - - # Parse TYPE(42, "hello world") - elif match := re.match( - r'TYPE\s*\(\s*(\d+)\s*,\s*["\']([^"\']*)["\']\s*\)', - action_str, - re.IGNORECASE, - ): - element_id = int(match.group(1)) - text = match.group(2) - result = await type_text_async(self.browser, element_id, text) - return { - "success": result.success, - "action": "type", - "element_id": element_id, - "text": text, - "outcome": result.outcome, - } - - # Parse PRESS("Enter") - elif match := re.match(r'PRESS\s*\(\s*["\']([^"\']+)["\']\s*\)', action_str, re.IGNORECASE): - key = match.group(1) - result = await press_async(self.browser, key) - return { - "success": result.success, - "action": "press", - "key": key, - "outcome": result.outcome, - } - - # Parse FINISH() - elif re.match(r"FINISH\s*\(\s*\)", action_str, re.IGNORECASE): - return { - "success": True, - "action": "finish", - "message": "Task marked as complete", - } - - else: - raise ValueError( - f"Unknown action format: {action_str}\n" - f'Expected: CLICK(id), TYPE(id, "text"), PRESS("key"), or FINISH()' - ) - def _track_tokens(self, goal: str, llm_response: LLMResponse): """Track token usage for analytics (same as sync version)""" if llm_response.prompt_tokens: diff --git a/sentience/formatting.py b/sentience/formatting.py index f8961c5..b8dd653 100644 --- a/sentience/formatting.py +++ b/sentience/formatting.py @@ -1,59 +1,15 @@ """ Snapshot formatting utilities for LLM prompts. -Provides functions to convert Sentience snapshots into text format suitable -for LLM consumption. -""" - -from typing import List - -from .models import Snapshot - - -def format_snapshot_for_llm(snap: Snapshot, limit: int = 50) -> str: - """ - Convert snapshot elements to text format for LLM consumption. - - This is the canonical way Sentience formats DOM state for LLMs. - The format includes element ID, role, text preview, visual cues, - position, and importance score. +DEPRECATED: This module is maintained for backward compatibility only. +New code should import from sentience.utils.formatting or sentience directly: - Args: - snap: Snapshot object with elements - limit: Maximum number of elements to include (default: 50) - - Returns: - Formatted string with one element per line - - Example: - >>> snap = snapshot(browser) - >>> formatted = format_snapshot_for_llm(snap, limit=10) - >>> print(formatted) - [1]