diff --git a/README.md b/README.md index 618981b..5740ad4 100644 --- a/README.md +++ b/README.md @@ -9,16 +9,66 @@ pip install -e . # Install Playwright browsers (required) playwright install chromium + +# For LLM Agent features (optional) +pip install openai # For OpenAI models +pip install anthropic # For Claude models +pip install transformers torch # For local LLMs +``` + +## Quick Start: Choose Your Abstraction Level + +Sentience SDK offers **three abstraction levels** - use what fits your needs: + +### šŸŽÆ **Level 3: Natural Language (Easiest)** - For non-technical users + +```python +from sentience import SentienceBrowser, ConversationalAgent +from sentience.llm_provider import OpenAIProvider + +browser = SentienceBrowser() +llm = OpenAIProvider(api_key="your-key", model="gpt-4o") +agent = ConversationalAgent(browser, llm) + +with browser: + response = agent.execute("Search for magic mouse on google.com") + print(response) + # → "I searched for 'magic mouse' and found several results. + # The top result is from amazon.com selling Magic Mouse 2 for $79." ``` -## Quick Start +**Best for:** End users, chatbots, no-code platforms +**Code required:** 3-5 lines +**Technical knowledge:** None + +### āš™ļø **Level 2: Technical Commands (Recommended)** - For AI developers + +```python +from sentience import SentienceBrowser, SentienceAgent +from sentience.llm_provider import OpenAIProvider + +browser = SentienceBrowser() +llm = OpenAIProvider(api_key="your-key", model="gpt-4o") +agent = SentienceAgent(browser, llm) + +with browser: + browser.page.goto("https://google.com") + agent.act("Click the search box") + agent.act("Type 'magic mouse' into the search field") + agent.act("Press Enter key") +``` + +**Best for:** Building AI agents, automation scripts +**Code required:** 10-15 lines +**Technical knowledge:** Medium (Python basics) + +### šŸ”§ **Level 1: Direct SDK (Most Control)** - For production automation ```python from sentience import SentienceBrowser, snapshot, find, click -# Start browser with extension with SentienceBrowser(headless=False) as browser: - browser.goto("https://example.com", wait_until="domcontentloaded") + browser.page.goto("https://example.com") # Take snapshot - captures all interactive elements snap = snapshot(browser) @@ -31,6 +81,10 @@ with SentienceBrowser(headless=False) as browser: print(f"Click success: {result.success}") ``` +**Best for:** Maximum control, performance-critical apps +**Code required:** 20-50 lines +**Technical knowledge:** High (SDK API, selectors) + ## Real-World Example: Amazon Shopping Bot This example demonstrates navigating Amazon, finding products, and adding items to cart: diff --git a/examples/agent_layers_demo.py b/examples/agent_layers_demo.py new file mode 100644 index 0000000..c5432e7 --- /dev/null +++ b/examples/agent_layers_demo.py @@ -0,0 +1,222 @@ +""" +Demonstration of all three abstraction layers in Sentience SDK + +Layer 1: Direct SDK (Full Control) +Layer 2: SentienceAgent (Technical Commands) +Layer 3: ConversationalAgent (Natural Language) + +This script shows how the same task can be accomplished at different abstraction levels. +""" + +import os +from dotenv import load_dotenv + +# Load environment variables +load_dotenv() + +def demo_layer1_direct_sdk(): + """ + Layer 1: Direct SDK Usage + - Full control over every action + - Requires knowing exact element selectors + - 50+ lines of code for typical automation + """ + print("\n" + "="*70) + print("LAYER 1: Direct SDK Usage (Full Control)") + print("="*70) + + from sentience import SentienceBrowser, snapshot, find, click, type_text, press + + with SentienceBrowser(headless=False) as browser: + # Navigate + browser.page.goto("https://google.com") + + # Get snapshot + snap = snapshot(browser) + + # Find search box manually + search_box = find(snap, "role=searchbox") + if not search_box: + search_box = find(snap, "role=textbox") + + # Click search box + click(browser, search_box.id) + + # Type query + type_text(browser, search_box.id, "magic mouse") + + # Press Enter + press(browser, "Enter") + + print("\nāœ… Layer 1 Demo Complete") + print(" Code required: ~20 lines") + print(" Technical knowledge: High") + print(" Flexibility: Maximum") + + +def demo_layer2_sentience_agent(): + """ + Layer 2: SentienceAgent (Technical Commands) + - High-level commands with LLM intelligence + - No need to know selectors + - 15 lines of code for typical automation + """ + print("\n" + "="*70) + print("LAYER 2: SentienceAgent (Technical Commands)") + print("="*70) + + from sentience import SentienceBrowser, SentienceAgent + from sentience.llm_provider import OpenAIProvider + + # Initialize + browser = SentienceBrowser(headless=False) + llm = OpenAIProvider(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-mini") + agent = SentienceAgent(browser, llm, verbose=True) + + with browser: + browser.page.goto("https://google.com") + + # Execute technical commands + agent.act("Click the search box") + agent.act("Type 'magic mouse' into the search field") + agent.act("Press Enter key") + + print("\nāœ… Layer 2 Demo Complete") + print(" Code required: ~10 lines") + print(" Technical knowledge: Medium") + print(" Flexibility: High") + print(f" Tokens used: {agent.get_token_stats()['total_tokens']}") + + +def demo_layer3_conversational_agent(): + """ + Layer 3: ConversationalAgent (Natural Language) + - Pure natural language interface + - Automatic planning and execution + - 3 lines of code for typical automation + """ + print("\n" + "="*70) + print("LAYER 3: ConversationalAgent (Natural Language)") + print("="*70) + + from sentience import SentienceBrowser, ConversationalAgent + from sentience.llm_provider import OpenAIProvider + + # Initialize + browser = SentienceBrowser(headless=False) + llm = OpenAIProvider(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + agent = ConversationalAgent(browser, llm, verbose=True) + + with browser: + # Execute in natural language (agent plans and executes automatically) + response = agent.execute("Search for magic mouse on google.com") + + print("\nāœ… Layer 3 Demo Complete") + print(" Code required: ~5 lines") + print(" Technical knowledge: None") + print(" Flexibility: Medium") + print(f" Agent Response: {response}") + + +def demo_layer3_with_local_llm(): + """ + Layer 3 with Local LLM (Zero Cost) + - Uses local Qwen 2.5 3B model + - No API costs + - Runs on your hardware + """ + print("\n" + "="*70) + print("LAYER 3: ConversationalAgent with Local LLM (Zero Cost)") + print("="*70) + + from sentience import SentienceBrowser, ConversationalAgent + from sentience.llm_provider import LocalLLMProvider + + # Initialize with local LLM + browser = SentienceBrowser(headless=False) + llm = LocalLLMProvider( + model_name="Qwen/Qwen2.5-3B-Instruct", + device="auto", # Use CUDA if available + load_in_4bit=True # Save memory with quantization + ) + agent = ConversationalAgent(browser, llm, verbose=True) + + with browser: + # Execute in natural language + response = agent.execute("Go to google.com and search for python tutorials") + + print("\nāœ… Layer 3 with Local LLM Demo Complete") + print(" API Cost: $0 (runs locally)") + print(" Privacy: 100% (no data sent to cloud)") + print(f" Agent Response: {response}") + + +def demo_comparison(): + """ + Side-by-side comparison of all layers + """ + print("\n" + "="*70) + print("COMPARISON: All Three Layers") + print("="*70) + + comparison_table = """ + | Feature | Layer 1 (SDK) | Layer 2 (Agent) | Layer 3 (Conversational) | + |--------------------------|------------------|------------------|--------------------------| + | Lines of code | 50+ | 15 | 3-5 | + | Technical knowledge | High | Medium | None | + | Requires selectors? | Yes | No | No | + | LLM required? | No | Yes | Yes | + | Cost per action | $0 | ~$0.005 | ~$0.010 | + | Speed | Fastest | Fast | Medium | + | Error handling | Manual | Auto-retry | Auto-recovery | + | Multi-step planning | Manual | Manual | Automatic | + | Natural language I/O | No | Commands only | Full conversation | + | Best for | Production | AI developers | End users | + """ + + print(comparison_table) + + +def main(): + """Run all demos""" + print("\n" + "="*70) + print("SENTIENCE SDK: Multi-Layer Abstraction Demo") + print("="*70) + print("\nThis demo shows how to use the SDK at different abstraction levels:") + print(" 1. Layer 1: Direct SDK (maximum control)") + print(" 2. Layer 2: SentienceAgent (technical commands)") + print(" 3. Layer 3: ConversationalAgent (natural language)") + print("\nChoose which demo to run:") + print(" 1 - Layer 1: Direct SDK") + print(" 2 - Layer 2: SentienceAgent") + print(" 3 - Layer 3: ConversationalAgent (OpenAI)") + print(" 4 - Layer 3: ConversationalAgent (Local LLM)") + print(" 5 - Show comparison table") + print(" 0 - Exit") + + choice = input("\nEnter your choice (0-5): ").strip() + + if choice == "1": + demo_layer1_direct_sdk() + elif choice == "2": + if not os.getenv("OPENAI_API_KEY"): + print("\nāŒ Error: OPENAI_API_KEY not set") + return + demo_layer2_sentience_agent() + elif choice == "3": + if not os.getenv("OPENAI_API_KEY"): + print("\nāŒ Error: OPENAI_API_KEY not set") + return + demo_layer3_conversational_agent() + elif choice == "4": + demo_layer3_with_local_llm() + elif choice == "5": + demo_comparison() + elif choice == "0": + print("Goodbye!") + else: + print("Invalid choice") + + +if __name__ == "__main__": + main() diff --git a/sentience/__init__.py b/sentience/__init__.py index f5f59e1..31094a3 100644 --- a/sentience/__init__.py +++ b/sentience/__init__.py @@ -18,6 +18,7 @@ # Agent Layer (Phase 1 & 2) from .llm_provider import LLMProvider, LLMResponse, OpenAIProvider, AnthropicProvider, LocalLLMProvider from .agent import SentienceAgent +from .conversational_agent import ConversationalAgent __version__ = "0.10.7" @@ -49,12 +50,13 @@ "generate", "read", "screenshot", - # Agent Layer + # Agent Layer (Phase 1 & 2) "LLMProvider", "LLMResponse", "OpenAIProvider", "AnthropicProvider", "LocalLLMProvider", "SentienceAgent", + "ConversationalAgent", ] diff --git a/sentience/conversational_agent.py b/sentience/conversational_agent.py new file mode 100644 index 0000000..c3f4e83 --- /dev/null +++ b/sentience/conversational_agent.py @@ -0,0 +1,525 @@ +""" +Conversational Agent: Natural language interface for Sentience SDK +Enables end users to control web automation using plain English +""" + +import json +import time +from typing import Dict, Any, List, Optional +from .llm_provider import LLMProvider, LLMResponse +from .browser import SentienceBrowser +from .agent import SentienceAgent +from .snapshot import snapshot +from .models import Snapshot + + +class ConversationalAgent: + """ + Natural language agent that translates user intent into SDK actions + and returns human-readable results. + + This is Layer 4 - the highest abstraction level for non-technical users. + + Example: + >>> agent = ConversationalAgent(browser, llm) + >>> result = agent.execute("Search for magic mouse on google.com") + >>> print(result) + "I searched for 'magic mouse' on Google and found several results. + The top result is from amazon.com selling the Apple Magic Mouse 2 for $79." + """ + + def __init__( + self, + browser: SentienceBrowser, + llm: LLMProvider, + verbose: bool = True + ): + """ + Initialize conversational agent + + Args: + browser: SentienceBrowser instance + llm: LLM provider (OpenAI, Anthropic, LocalLLM, etc.) + verbose: Print step-by-step execution logs (default: True) + """ + self.browser = browser + self.llm = llm + self.verbose = verbose + + # Underlying technical agent + self.technical_agent = SentienceAgent(browser, llm, verbose=False) + + # Conversation history and context + self.conversation_history: List[Dict[str, Any]] = [] + self.execution_context: Dict[str, Any] = { + "current_url": None, + "last_action": None, + "discovered_elements": [], + "session_data": {} + } + + def execute(self, user_input: str) -> str: + """ + Execute a natural language command and return natural language result + + Args: + user_input: Natural language instruction (e.g., "Search for magic mouse") + + Returns: + Human-readable result description + + Example: + >>> agent.execute("Go to google.com and search for magic mouse") + "I navigated to google.com, searched for 'magic mouse', and found 10 results. + The top result is from amazon.com selling Magic Mouse 2 for $79." + """ + if self.verbose: + print(f"\n{'='*70}") + print(f"šŸ‘¤ User: {user_input}") + print(f"{'='*70}") + + start_time = time.time() + + # Step 1: Plan the execution (break down into atomic steps) + plan = self._create_plan(user_input) + + if self.verbose: + print(f"\nšŸ“‹ Execution Plan:") + for i, step in enumerate(plan['steps'], 1): + print(f" {i}. {step['description']}") + + # Step 2: Execute each step + execution_results = [] + for step in plan['steps']: + step_result = self._execute_step(step) + execution_results.append(step_result) + + if not step_result.get('success', False): + # Early exit on failure + if self.verbose: + print(f"āš ļø Step failed: {step['description']}") + break + + # Step 3: Synthesize natural language response + response = self._synthesize_response(user_input, plan, execution_results) + + duration_ms = int((time.time() - start_time) * 1000) + + # Step 4: Update conversation history + self.conversation_history.append({ + "user_input": user_input, + "plan": plan, + "results": execution_results, + "response": response, + "duration_ms": duration_ms + }) + + if self.verbose: + print(f"\nšŸ¤– Agent: {response}") + print(f"ā±ļø Completed in {duration_ms}ms\n") + + return response + + def _create_plan(self, user_input: str) -> Dict[str, Any]: + """ + Use LLM to break down user input into atomic executable steps + + Args: + user_input: Natural language command + + Returns: + Plan dictionary with list of atomic steps + """ + # Get current page context + current_url = self.browser.page.url if self.browser.page else "None" + + system_prompt = """You are a web automation planning assistant. + +Your job is to analyze a natural language request and break it down into atomic steps +that can be executed by a web automation agent. + +AVAILABLE ACTIONS: +1. NAVIGATE - Go to a URL +2. FIND_AND_CLICK - Find and click an element by description +3. FIND_AND_TYPE - Find input field and type text +4. PRESS_KEY - Press a keyboard key (Enter, Escape, etc.) +5. WAIT - Wait for page to load or element to appear +6. EXTRACT_INFO - Extract specific information from the page +7. VERIFY - Verify a condition is met + +RESPONSE FORMAT (JSON): +{ + "intent": "brief summary of user intent", + "steps": [ + { + "action": "NAVIGATE" | "FIND_AND_CLICK" | "FIND_AND_TYPE" | "PRESS_KEY" | "WAIT" | "EXTRACT_INFO" | "VERIFY", + "description": "human-readable description", + "parameters": { + "url": "https://...", + "element_description": "search box", + "text": "magic mouse", + "key": "Enter", + "duration": 2.0, + "info_type": "product link", + "condition": "page contains results" + } + } + ], + "expected_outcome": "what success looks like" +} + +IMPORTANT: Return ONLY valid JSON, no markdown, no code blocks.""" + + user_prompt = f"""Current URL: {current_url} + +User Request: {user_input} + +Create a step-by-step execution plan.""" + + try: + response = self.llm.generate( + system_prompt, + user_prompt, + json_mode=self.llm.supports_json_mode(), + temperature=0.0 + ) + + # Parse JSON response + plan = json.loads(response.content) + return plan + + except json.JSONDecodeError as e: + # Fallback: create simple plan + if self.verbose: + print(f"āš ļø JSON parsing failed, using fallback plan: {e}") + + return { + "intent": user_input, + "steps": [ + { + "action": "FIND_AND_CLICK", + "description": user_input, + "parameters": {"element_description": user_input} + } + ], + "expected_outcome": "Complete user request" + } + + def _execute_step(self, step: Dict[str, Any]) -> Dict[str, Any]: + """ + Execute a single atomic step from the plan + + Args: + step: Step dictionary with action and parameters + + Returns: + Execution result with success status and data + """ + action = step['action'] + params = step.get('parameters', {}) + + if self.verbose: + print(f"\nāš™ļø Executing: {step['description']}") + + try: + if action == "NAVIGATE": + url = params['url'] + # Add https:// if missing + if not url.startswith(('http://', 'https://')): + url = 'https://' + url + + self.browser.page.goto(url, wait_until="domcontentloaded") + self.execution_context['current_url'] = url + time.sleep(1) # Brief wait for page to settle + + return { + "success": True, + "action": action, + "data": {"url": url} + } + + elif action == "FIND_AND_CLICK": + element_desc = params['element_description'] + # Use technical agent to find and click + result = self.technical_agent.act(f"Click the {element_desc}") + return { + "success": result.get('success', False), + "action": action, + "data": result + } + + elif action == "FIND_AND_TYPE": + element_desc = params['element_description'] + text = params['text'] + # Use technical agent to find input and type + result = self.technical_agent.act(f"Type '{text}' into {element_desc}") + return { + "success": result.get('success', False), + "action": action, + "data": {"text": text} + } + + elif action == "PRESS_KEY": + key = params['key'] + result = self.technical_agent.act(f"Press {key} key") + return { + "success": result.get('success', False), + "action": action, + "data": {"key": key} + } + + elif action == "WAIT": + duration = params.get('duration', 2.0) + time.sleep(duration) + return { + "success": True, + "action": action, + "data": {"duration": duration} + } + + elif action == "EXTRACT_INFO": + info_type = params['info_type'] + # Get current page snapshot and extract info + snap = snapshot(self.browser, limit=50) + + # Use LLM to extract specific information + extracted = self._extract_information(snap, info_type) + + return { + "success": True, + "action": action, + "data": {"extracted": extracted, "info_type": info_type} + } + + elif action == "VERIFY": + condition = params['condition'] + # Verify condition using current page state + is_verified = self._verify_condition(condition) + return { + "success": is_verified, + "action": action, + "data": {"condition": condition, "verified": is_verified} + } + + else: + raise ValueError(f"Unknown action: {action}") + + except Exception as e: + if self.verbose: + print(f"āŒ Step failed: {e}") + return { + "success": False, + "action": action, + "error": str(e) + } + + def _extract_information(self, snap: Snapshot, info_type: str) -> Dict[str, Any]: + """ + Extract specific information from snapshot using LLM + + Args: + snap: Snapshot object + info_type: Type of info to extract (e.g., "product link", "price") + + Returns: + Extracted information dictionary + """ + # Build context from snapshot + elements_text = "\n".join([ + f"[{el.id}] {el.role}: {el.text} (importance: {el.importance})" + for el in snap.elements[:30] # Top 30 elements + ]) + + system_prompt = f"""Extract {info_type} from the following page elements. + +ELEMENTS: +{elements_text} + +Return JSON with extracted information: +{{ + "found": true/false, + "data": {{ + // extracted information fields + }}, + "summary": "brief description of what was found" +}}""" + + user_prompt = f"Extract {info_type} from the elements above." + + try: + response = self.llm.generate( + system_prompt, + user_prompt, + json_mode=self.llm.supports_json_mode() + ) + return json.loads(response.content) + except: + return {"found": False, "data": {}, "summary": "Failed to extract information"} + + def _verify_condition(self, condition: str) -> bool: + """ + Verify a condition is met on current page + + Args: + condition: Natural language condition to verify + + Returns: + True if condition is met, False otherwise + """ + try: + snap = snapshot(self.browser, limit=30) + + # Build context + elements_text = "\n".join([ + f"{el.role}: {el.text}" + for el in snap.elements[:20] + ]) + + system_prompt = f"""Verify if the following condition is met based on page elements. + +CONDITION: {condition} + +PAGE ELEMENTS: +{elements_text} + +Return JSON: +{{ + "verified": true/false, + "reasoning": "explanation" +}}""" + + response = self.llm.generate( + system_prompt, + "", + json_mode=self.llm.supports_json_mode() + ) + result = json.loads(response.content) + return result.get('verified', False) + except: + return False + + def _synthesize_response( + self, + user_input: str, + plan: Dict[str, Any], + execution_results: List[Dict[str, Any]] + ) -> str: + """ + Synthesize a natural language response from execution results + + Args: + user_input: Original user input + plan: Execution plan + execution_results: List of step execution results + + Returns: + Human-readable response string + """ + # Build summary of what happened + successful_steps = [r for r in execution_results if r.get('success')] + failed_steps = [r for r in execution_results if not r.get('success')] + + # Extract key data + extracted_data = [] + for result in execution_results: + if result.get('action') == 'EXTRACT_INFO': + extracted_data.append(result.get('data', {}).get('extracted', {})) + + # Use LLM to create natural response + system_prompt = """You are a helpful assistant that summarizes web automation results +in natural, conversational language. + +Your job is to take technical execution results and convert them into a friendly, +human-readable response that answers the user's original request. + +Be concise but informative. Include key findings or data discovered. +If the task failed, explain what went wrong in simple terms. + +IMPORTANT: Return only the natural language response, no JSON, no markdown.""" + + results_summary = { + "user_request": user_input, + "plan_intent": plan.get('intent'), + "total_steps": len(execution_results), + "successful_steps": len(successful_steps), + "failed_steps": len(failed_steps), + "extracted_data": extracted_data, + "final_url": self.browser.page.url if self.browser.page else None + } + + user_prompt = f"""Summarize these automation results in 1-3 natural sentences: + +{json.dumps(results_summary, indent=2)} + +Respond as if you're talking to a user, not listing technical details.""" + + try: + response = self.llm.generate(system_prompt, user_prompt, temperature=0.3) + return response.content.strip() + except: + # Fallback response + if failed_steps: + return f"I attempted to {user_input}, but encountered an error during execution." + else: + return f"I completed your request: {user_input}" + + def chat(self, message: str) -> str: + """ + Conversational interface with context awareness + + Args: + message: User message (can reference previous context) + + Returns: + Agent response + + Example: + >>> agent.chat("Go to google.com") + "I've navigated to google.com" + >>> agent.chat("Search for magic mouse") # Contextual + "I searched for 'magic mouse' and found 10 results" + """ + return self.execute(message) + + def get_summary(self) -> str: + """ + Get a summary of the entire conversation/session + + Returns: + Natural language summary of all actions taken + """ + if not self.conversation_history: + return "No actions have been performed yet." + + system_prompt = """Summarize this web automation session in a brief, natural paragraph. +Focus on what was accomplished and key findings.""" + + session_data = { + "total_interactions": len(self.conversation_history), + "actions": [ + { + "request": h['user_input'], + "outcome": h['response'] + } + for h in self.conversation_history + ] + } + + user_prompt = f"Summarize this session:\n{json.dumps(session_data, indent=2)}" + + try: + summary = self.llm.generate(system_prompt, user_prompt) + return summary.content.strip() + except: + return f"Session with {len(self.conversation_history)} interactions completed." + + def clear_history(self): + """Clear conversation history""" + self.conversation_history.clear() + self.technical_agent.clear_history() + self.execution_context = { + "current_url": None, + "last_action": None, + "discovered_elements": [], + "session_data": {} + } diff --git a/tests/test_conversational_agent.py b/tests/test_conversational_agent.py new file mode 100644 index 0000000..7411be9 --- /dev/null +++ b/tests/test_conversational_agent.py @@ -0,0 +1,468 @@ +""" +Integration tests for ConversationalAgent (Phase 2) +Tests natural language interface without requiring browser +""" + +import pytest +import json +from unittest.mock import Mock, MagicMock, patch +from sentience.conversational_agent import ConversationalAgent +from sentience.llm_provider import LLMProvider, LLMResponse +from sentience.models import Snapshot, Element, BBox, VisualCues, Viewport + + +class MockLLMProvider(LLMProvider): + """Mock LLM provider for testing conversational agent""" + + def __init__(self, responses=None): + self.responses = responses or {} + self.call_count = 0 + self.calls = [] + + def generate(self, system_prompt: str, user_prompt: str, **kwargs): + self.calls.append({ + "system": system_prompt, + "user": user_prompt, + "kwargs": kwargs + }) + + # Determine response based on content + if "planning assistant" in system_prompt.lower(): + # Return plan + response = self.responses.get('plan', self._default_plan()) + elif "extract" in system_prompt.lower(): + # Return extraction result + response = self.responses.get('extract', '{"found": true, "data": {}, "summary": "Info extracted"}') + elif "verify" in system_prompt.lower(): + # Return verification result + response = self.responses.get('verify', '{"verified": true, "reasoning": "Condition met"}') + elif "summarize" in system_prompt.lower(): + # Return summary + response = self.responses.get('summary', "Task completed successfully") + else: + # Default technical agent response + response = self.responses.get('action', "CLICK(1)") + + self.call_count += 1 + + return LLMResponse( + content=response, + prompt_tokens=100, + completion_tokens=20, + total_tokens=120, + model_name="mock-model" + ) + + def _default_plan(self): + return json.dumps({ + "intent": "Test intent", + "steps": [ + { + "action": "NAVIGATE", + "description": "Go to test.com", + "parameters": {"url": "https://test.com"} + } + ], + "expected_outcome": "Success" + }) + + def supports_json_mode(self) -> bool: + return True + + @property + def model_name(self) -> str: + return "mock-model" + + +def create_mock_browser(): + """Create mock browser for testing""" + browser = Mock() + browser.page = Mock() + browser.page.url = "https://test.com" + browser.page.goto = Mock() + return browser + + +def create_mock_snapshot(): + """Create mock snapshot with test elements""" + elements = [ + Element( + id=1, + role="button", + text="Click Me", + importance=900, + bbox=BBox(x=100, y=200, width=80, height=30), + visual_cues=VisualCues( + is_primary=True, + is_clickable=True, + background_color_name="blue" + ), + in_viewport=True, + is_occluded=False, + z_index=10 + ) + ] + + return Snapshot( + status="success", + timestamp="2024-12-24T10:00:00Z", + url="https://test.com", + viewport=Viewport(width=1920, height=1080), + elements=elements + ) + + +# ========== ConversationalAgent Tests ========== + +def test_conversational_agent_initialization(): + """Test ConversationalAgent initialization""" + browser = create_mock_browser() + llm = MockLLMProvider() + + agent = ConversationalAgent(browser, llm, verbose=False) + + assert agent.browser == browser + assert agent.llm == llm + assert agent.verbose is False + assert len(agent.conversation_history) == 0 + assert agent.technical_agent is not None + + +def test_create_plan(): + """Test plan creation from natural language""" + browser = create_mock_browser() + + plan_json = json.dumps({ + "intent": "Search for magic mouse", + "steps": [ + { + "action": "NAVIGATE", + "description": "Go to google.com", + "parameters": {"url": "https://google.com"} + }, + { + "action": "FIND_AND_CLICK", + "description": "Click search box", + "parameters": {"element_description": "search box"} + } + ], + "expected_outcome": "Search initiated" + }) + + llm = MockLLMProvider(responses={'plan': plan_json}) + agent = ConversationalAgent(browser, llm, verbose=False) + + plan = agent._create_plan("Search for magic mouse on google") + + assert plan['intent'] == "Search for magic mouse" + assert len(plan['steps']) == 2 + assert plan['steps'][0]['action'] == "NAVIGATE" + assert plan['steps'][1]['action'] == "FIND_AND_CLICK" + + +def test_create_plan_json_fallback(): + """Test plan creation with invalid JSON fallback""" + browser = create_mock_browser() + llm = MockLLMProvider(responses={'plan': 'INVALID JSON{'}) + agent = ConversationalAgent(browser, llm, verbose=False) + + plan = agent._create_plan("Click button") + + # Should fall back to simple plan + assert 'intent' in plan + assert 'steps' in plan + assert len(plan['steps']) > 0 + + +def test_execute_navigate_step(): + """Test NAVIGATE step execution""" + browser = create_mock_browser() + llm = MockLLMProvider() + agent = ConversationalAgent(browser, llm, verbose=False) + + step = { + "action": "NAVIGATE", + "description": "Go to google.com", + "parameters": {"url": "google.com"} # Without https:// + } + + result = agent._execute_step(step) + + assert result['success'] is True + assert result['action'] == "NAVIGATE" + browser.page.goto.assert_called_once() + # Should have added https:// + assert "https://google.com" in str(browser.page.goto.call_args) + + +def test_execute_find_and_click_step(): + """Test FIND_AND_CLICK step execution""" + browser = create_mock_browser() + llm = MockLLMProvider(responses={'action': 'CLICK(1)'}) + agent = ConversationalAgent(browser, llm, verbose=False) + + step = { + "action": "FIND_AND_CLICK", + "description": "Click the button", + "parameters": {"element_description": "button"} + } + + # Patch at the agent module level where it's imported + with patch('sentience.agent.snapshot') as mock_snapshot, \ + patch('sentience.agent.click') as mock_click: + + from sentience.models import ActionResult + mock_snapshot.return_value = create_mock_snapshot() + mock_click.return_value = ActionResult( + success=True, + duration_ms=150, + outcome="dom_updated" + ) + + result = agent._execute_step(step) + + assert result['action'] == "FIND_AND_CLICK" + # Technical agent should have been called + assert len(agent.technical_agent.history) > 0 + + +def test_execute_find_and_type_step(): + """Test FIND_AND_TYPE step execution""" + browser = create_mock_browser() + llm = MockLLMProvider(responses={'action': 'TYPE(1, "test")'}) + agent = ConversationalAgent(browser, llm, verbose=False) + + step = { + "action": "FIND_AND_TYPE", + "description": "Type into search box", + "parameters": { + "element_description": "search box", + "text": "magic mouse" + } + } + + # Patch at the agent module level where it's imported + with patch('sentience.agent.snapshot') as mock_snapshot, \ + patch('sentience.agent.type_text') as mock_type: + + from sentience.models import ActionResult + mock_snapshot.return_value = create_mock_snapshot() + mock_type.return_value = ActionResult( + success=True, + duration_ms=200, + outcome="dom_updated" + ) + + result = agent._execute_step(step) + + assert result['action'] == "FIND_AND_TYPE" + assert result['data']['text'] == "magic mouse" + + +def test_execute_wait_step(): + """Test WAIT step execution""" + browser = create_mock_browser() + llm = MockLLMProvider() + agent = ConversationalAgent(browser, llm, verbose=False) + + step = { + "action": "WAIT", + "description": "Wait for page to load", + "parameters": {"duration": 0.1} # Short wait for testing + } + + result = agent._execute_step(step) + + assert result['success'] is True + assert result['action'] == "WAIT" + assert result['data']['duration'] == 0.1 + + +def test_execute_extract_info_step(): + """Test EXTRACT_INFO step execution""" + browser = create_mock_browser() + + extract_response = json.dumps({ + "found": True, + "data": {"price": "$79"}, + "summary": "Found price information" + }) + + llm = MockLLMProvider(responses={'extract': extract_response}) + agent = ConversationalAgent(browser, llm, verbose=False) + + step = { + "action": "EXTRACT_INFO", + "description": "Extract price", + "parameters": {"info_type": "product price"} + } + + with patch('sentience.conversational_agent.snapshot') as mock_snapshot: + mock_snapshot.return_value = create_mock_snapshot() + + result = agent._execute_step(step) + + assert result['success'] is True + assert result['action'] == "EXTRACT_INFO" + assert result['data']['extracted']['found'] is True + + +def test_execute_verify_step(): + """Test VERIFY step execution""" + browser = create_mock_browser() + + verify_response = json.dumps({ + "verified": True, + "reasoning": "Page contains results" + }) + + llm = MockLLMProvider(responses={'verify': verify_response}) + agent = ConversationalAgent(browser, llm, verbose=False) + + step = { + "action": "VERIFY", + "description": "Verify results", + "parameters": {"condition": "page contains search results"} + } + + with patch('sentience.conversational_agent.snapshot') as mock_snapshot: + mock_snapshot.return_value = create_mock_snapshot() + + result = agent._execute_step(step) + + assert result['success'] is True + assert result['action'] == "VERIFY" + assert result['data']['verified'] is True + + +def test_synthesize_response(): + """Test natural language response synthesis""" + browser = create_mock_browser() + + llm = MockLLMProvider(responses={ + 'summary': "I navigated to google.com and found the search results you requested." + }) + + agent = ConversationalAgent(browser, llm, verbose=False) + + plan = { + "intent": "Search for magic mouse", + "steps": [], + "expected_outcome": "Success" + } + + execution_results = [ + {"success": True, "action": "NAVIGATE"} + ] + + response = agent._synthesize_response("Search for magic mouse", plan, execution_results) + + assert isinstance(response, str) + assert len(response) > 0 + + +def test_execute_full_workflow(): + """Test full execute() workflow""" + browser = create_mock_browser() + + plan_json = json.dumps({ + "intent": "Navigate to test site", + "steps": [ + { + "action": "NAVIGATE", + "description": "Go to test.com", + "parameters": {"url": "https://test.com"} + } + ], + "expected_outcome": "Navigation complete" + }) + + llm = MockLLMProvider(responses={ + 'plan': plan_json, + 'summary': "Successfully navigated to test.com" + }) + + agent = ConversationalAgent(browser, llm, verbose=False) + + response = agent.execute("Go to test.com") + + assert isinstance(response, str) + assert len(agent.conversation_history) == 1 + assert agent.conversation_history[0]['user_input'] == "Go to test.com" + + +def test_chat_method(): + """Test chat() method as alias for execute()""" + browser = create_mock_browser() + + plan_json = json.dumps({ + "intent": "Test", + "steps": [], + "expected_outcome": "Done" + }) + + llm = MockLLMProvider(responses={ + 'plan': plan_json, + 'summary': "Task complete" + }) + + agent = ConversationalAgent(browser, llm, verbose=False) + + response = agent.chat("Test message") + + assert isinstance(response, str) + assert len(agent.conversation_history) == 1 + + +def test_get_summary(): + """Test session summary generation""" + browser = create_mock_browser() + + llm = MockLLMProvider(responses={ + 'plan': '{"intent": "test", "steps": [], "expected_outcome": "done"}', + 'summary': "Session completed with 2 interactions" + }) + + agent = ConversationalAgent(browser, llm, verbose=False) + + # Add some history + agent.conversation_history.append({ + "user_input": "Test 1", + "response": "Done 1" + }) + agent.conversation_history.append({ + "user_input": "Test 2", + "response": "Done 2" + }) + + summary = agent.get_summary() + + assert isinstance(summary, str) + assert len(summary) > 0 + + +def test_get_summary_empty_history(): + """Test summary with no history""" + browser = create_mock_browser() + llm = MockLLMProvider() + agent = ConversationalAgent(browser, llm, verbose=False) + + summary = agent.get_summary() + + assert summary == "No actions have been performed yet." + + +def test_clear_history(): + """Test clearing conversation history""" + browser = create_mock_browser() + llm = MockLLMProvider() + agent = ConversationalAgent(browser, llm, verbose=False) + + # Add history + agent.conversation_history.append({"test": "data"}) + agent.technical_agent.history.append({"test": "data"}) + + agent.clear_history() + + assert len(agent.conversation_history) == 0 + assert len(agent.technical_agent.history) == 0