From a40a47277565ced1852ff31d048a7e22dbbe87c0 Mon Sep 17 00:00:00 2001 From: Tryanks Date: Tue, 1 Sep 2026 03:24:41 +0800 Subject: [PATCH 1/4] feat(computer-use): 2.0 background delivery, activation, chromium AX, JPEG --- crates/computer-use-mcp/Cargo.toml | 4 +- crates/computer-use-mcp/examples/cu_probe.rs | 180 +++++ crates/computer-use-mcp/src/backend.rs | 48 +- .../computer-use-mcp/src/backend/macos/ax.rs | 332 ++++++++- .../src/backend/macos/background.rs | 690 ++++++++++++++++++ .../src/backend/macos/capture.rs | 77 +- .../src/backend/macos/focus.rs | 56 +- .../src/backend/macos/input.rs | 145 ++-- .../computer-use-mcp/src/backend/macos/mod.rs | 277 +++++-- .../src/backend/windows/mod.rs | 91 ++- crates/computer-use-mcp/src/config.rs | 25 + crates/computer-use-mcp/src/lib.rs | 5 + crates/computer-use-mcp/src/tools.rs | 56 +- crates/core/src/settings.rs | 17 + docs/computer-use.md | 59 +- 15 files changed, 1831 insertions(+), 231 deletions(-) create mode 100644 crates/computer-use-mcp/examples/cu_probe.rs create mode 100644 crates/computer-use-mcp/src/backend/macos/background.rs diff --git a/crates/computer-use-mcp/Cargo.toml b/crates/computer-use-mcp/Cargo.toml index b5baffd8..006fb35c 100644 --- a/crates/computer-use-mcp/Cargo.toml +++ b/crates/computer-use-mcp/Cargo.toml @@ -20,15 +20,15 @@ schemars = "1" log = "0.4" mcp-host = { path = "../mcp-host" } base64 = "0.23" +image = { version = "0.25.10", default-features = false, features = ["png", "jpeg"] } tcode-services = { path = "../services" } tcode-core = { path = "../core" } [target.'cfg(target_os = "macos")'.dependencies] core-foundation = "0.10" -core-graphics = { version = "0.25", features = ["highsierra"] } +core-graphics = { version = "0.25", features = ["elcapitan", "highsierra"] } [target.'cfg(target_os = "windows")'.dependencies] -image = { version = "0.25.10", default-features = false, features = ["png"] } uiautomation = { version = "0.25", default-features = false, features = [ "control", "input", diff --git a/crates/computer-use-mcp/examples/cu_probe.rs b/crates/computer-use-mcp/examples/cu_probe.rs new file mode 100644 index 00000000..01276b5a --- /dev/null +++ b/crates/computer-use-mcp/examples/cu_probe.rs @@ -0,0 +1,180 @@ +use std::error::Error; +use std::io; + +use computer_use_mcp::backend::{ + self, ActionKind, ActionRequest, CapturePolicy, MouseButton, ObserveRequest, RootFilters, +}; + +enum Operation { + Observe, + Click(String), + ClickCenter, + Type(String), +} + +fn main() -> Result<(), Box> { + let (app, operation) = parse_args()?; + let root = backend::list_roots(&RootFilters { + app: Some(app.clone()), + ..RootFilters::default() + })? + .into_iter() + .next() + .ok_or_else(|| io::Error::other(format!("no window matched app substring {app:?}")))?; + + println!( + "frontmost_pid_before={:?}", + computer_use_mcp::frontmost_pid() + ); + let result = run_operation(&root, operation); + println!( + "frontmost_pid_after={:?}", + computer_use_mcp::frontmost_pid() + ); + result +} + +fn run_operation(root: &backend::RootInfo, operation: Operation) -> Result<(), Box> { + match operation { + Operation::Observe => { + let mut observation = backend::observe( + root, + ObserveRequest { + semantic: true, + capture: CapturePolicy::Never, + }, + )?; + computer_use_mcp::outline::assign_refs(&mut observation.tree); + println!( + "{}", + computer_use_mcp::outline::render_folded(&observation.tree) + ); + } + Operation::Click(ref_id) => { + let mut observation = backend::observe( + root, + ObserveRequest { + semantic: true, + capture: CapturePolicy::Never, + }, + )?; + computer_use_mcp::outline::assign_refs(&mut observation.tree); + let node = observation.tree.find(&ref_id).ok_or_else(|| { + io::Error::other(format!( + "element {ref_id} was not present; run --observe to inspect current refs" + )) + })?; + let path = computer_use_mcp::outline::path_to_ref(&observation.tree, &ref_id) + .ok_or_else(|| io::Error::other(format!("could not resolve path for {ref_id}")))?; + let request = ActionRequest { + kind: ActionKind::Click, + target_path: Some(path), + target_frame: Some(node.frame), + target_role: Some(node.role.clone()), + target_title: Some(node.title.clone()), + target_actions: node.actions.clone(), + x: None, + y: None, + text: None, + keys: None, + scroll_x: None, + scroll_y: None, + path: None, + button: MouseButton::Left, + click_count: 1, + }; + print_action_result(backend::perform_action(root, &request)?); + } + Operation::ClickCenter => { + let (x, y) = root.frame.center(); + let request = coordinate_request(ActionKind::Click, Some(x), Some(y), None); + print_action_result(backend::perform_action(root, &request)?); + } + Operation::Type(text) => { + let request = coordinate_request(ActionKind::TypeText, None, None, Some(text)); + print_action_result(backend::perform_action(root, &request)?); + } + } + Ok(()) +} + +fn coordinate_request( + kind: ActionKind, + x: Option, + y: Option, + text: Option, +) -> ActionRequest { + ActionRequest { + kind, + target_path: None, + target_frame: None, + target_role: None, + target_title: None, + target_actions: Vec::new(), + x, + y, + text, + keys: None, + scroll_x: None, + scroll_y: None, + path: None, + button: MouseButton::Left, + click_count: 1, + } +} + +fn print_action_result(result: backend::ActionResult) { + let json = serde_json::to_string_pretty(&result).unwrap_or_else(|_| format!("{result:?}")); + println!("{json}"); +} + +fn parse_args() -> Result<(String, Operation), Box> { + let mut args = std::env::args().skip(1); + let mut app = None; + let mut operation = None; + while let Some(argument) = args.next() { + match argument.as_str() { + "--app" => app = Some(next_value(&mut args, "--app")?), + "--observe" => set_operation(&mut operation, Operation::Observe)?, + "--click" => { + let ref_id = next_value(&mut args, "--click")?; + set_operation(&mut operation, Operation::Click(ref_id))?; + } + "--click-center" => set_operation(&mut operation, Operation::ClickCenter)?, + "--type" => { + let text = next_value(&mut args, "--type")?; + set_operation(&mut operation, Operation::Type(text))?; + } + _ => return Err(usage(format!("unknown argument {argument:?}"))), + } + } + let app = app.ok_or_else(|| usage("--app is required"))?; + let operation = operation.ok_or_else(|| { + usage("choose one of --observe, --click , --click-center, or --type ") + })?; + Ok((app, operation)) +} + +fn next_value( + args: &mut impl Iterator, + flag: &str, +) -> Result> { + args.next() + .ok_or_else(|| usage(format!("{flag} requires a value"))) +} + +fn set_operation(slot: &mut Option, value: Operation) -> Result<(), Box> { + if slot.replace(value).is_some() { + Err(usage("only one probe operation may be selected")) + } else { + Ok(()) + } +} + +fn usage(message: impl Into) -> Box { + let message = message.into(); + io::Error::other(format!( + "{message}\nusage: cu_probe --app (--observe | --click | --click-center | --type )" + )) + .into() +} diff --git a/crates/computer-use-mcp/src/backend.rs b/crates/computer-use-mcp/src/backend.rs index 7b0ac715..5836872e 100644 --- a/crates/computer-use-mcp/src/backend.rs +++ b/crates/computer-use-mcp/src/backend.rs @@ -96,7 +96,8 @@ pub struct RootObservation { pub root: RootInfo, pub tree: UiNode, pub text_sparse: bool, - pub screenshot_png: Option>, + pub screenshot: Option>, + pub screenshot_mime: &'static str, } /// Supported desktop input action. @@ -151,31 +152,46 @@ pub enum ActionOutcome { Unknown, } +/// Mechanism used to deliver an action to the target application. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Delivery { + Ax, + BackgroundPid, + ForegroundHid, + #[default] + None, +} + #[derive(Debug, Clone, Serialize)] pub struct ActionResult { pub outcome: ActionOutcome, pub message: String, + pub delivery: Delivery, } impl ActionResult { - pub fn worked(message: impl Into) -> Self { + pub fn worked(message: impl Into, delivery: Delivery) -> Self { Self { outcome: ActionOutcome::Worked, message: message.into(), + delivery, } } - pub fn didnt(message: impl Into) -> Self { + pub fn didnt(message: impl Into, delivery: Delivery) -> Self { Self { outcome: ActionOutcome::Didnt, message: message.into(), + delivery, } } - pub fn unknown(message: impl Into) -> Self { + pub fn unknown(message: impl Into, delivery: Delivery) -> Self { Self { outcome: ActionOutcome::Unknown, message: message.into(), + delivery, } } } @@ -269,6 +285,18 @@ pub fn perform_action( } } +/// Process identifier of the currently frontmost macOS application. +pub fn frontmost_pid() -> Option { + #[cfg(target_os = "macos")] + { + macos::frontmost_pid() + } + #[cfg(not(target_os = "macos"))] + { + None + } +} + #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct KeyModifiers { pub command: bool, @@ -502,6 +530,18 @@ fn contains_case_insensitive(haystack: &str, needle: &str) -> bool { mod tests { use super::*; + #[test] + fn delivery_serializes_as_snake_case() { + for (delivery, expected) in [ + (Delivery::Ax, "ax"), + (Delivery::BackgroundPid, "background_pid"), + (Delivery::ForegroundHid, "foreground_hid"), + (Delivery::None, "none"), + ] { + assert_eq!(serde_json::to_value(delivery).unwrap(), expected); + } + } + #[test] fn key_names_and_chords_map_to_macos_virtual_codes() { assert_eq!(macos_keycode_for_name("enter"), Some(0x24)); diff --git a/crates/computer-use-mcp/src/backend/macos/ax.rs b/crates/computer-use-mcp/src/backend/macos/ax.rs index a97cd69d..5fe5adea 100644 --- a/crates/computer-use-mcp/src/backend/macos/ax.rs +++ b/crates/computer-use-mcp/src/backend/macos/ax.rs @@ -1,20 +1,28 @@ -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::ffi::c_void; use std::fmt; +use std::os::raw::{c_char, c_int}; use std::ptr; +use std::sync::{Mutex, OnceLock}; +use std::time::Duration; use core_foundation::array::{CFArrayGetCount, CFArrayGetValueAtIndex, CFArrayRef}; use core_foundation::base::{CFGetTypeID, CFRelease, CFRetain, CFTypeID, CFTypeRef, TCFType}; use core_foundation::boolean::CFBoolean; use core_foundation::number::CFNumber; +use core_foundation::runloop::{ + CFRunLoopAddSource, CFRunLoopGetCurrent, CFRunLoopRun, CFRunLoopSourceRef, CFRunLoopWakeUp, + kCFRunLoopDefaultMode, +}; use core_foundation::string::{CFString, CFStringRef}; use core_graphics::geometry::{CGPoint, CGRect, CGSize}; use super::super::{BackendError, BackendErrorCode, RootInfo, RootKind}; -use crate::outline::{Frame, UiNode, canonical_role}; +use crate::outline::{Frame, UiNode, canonical_role, is_text_sparse}; type AXUIElementRef = CFTypeRef; type AXValueRef = CFTypeRef; +type AXObserverRef = CFTypeRef; type AXError = i32; type AXValueType = u32; @@ -26,6 +34,20 @@ const MAX_DEPTH: usize = 18; const MAX_NODES: usize = 3_000; const MAX_CHILDREN_PER_NODE: usize = 500; +type AddNotificationAndCheckRemote = + unsafe extern "C" fn(AXObserverRef, AXUIElementRef, CFStringRef, *mut c_void) -> AXError; + +static ACTIVATED_PIDS: OnceLock>> = OnceLock::new(); +static WEB_SPARSE_PIDS: OnceLock>> = OnceLock::new(); +static AX_OBSERVERS: OnceLock>> = OnceLock::new(); + +const RTLD_LAZY: c_int = 1; + +unsafe extern "C" { + fn dlopen(path: *const c_char, mode: c_int) -> *mut c_void; + fn dlsym(handle: *mut c_void, symbol: *const c_char) -> *mut c_void; +} + #[link(name = "ApplicationServices", kind = "framework")] unsafe extern "C" { fn AXUIElementCreateApplication(pid: i32) -> AXUIElementRef; @@ -57,6 +79,18 @@ unsafe extern "C" { fn AXValueGetTypeID() -> CFTypeID; fn AXValueGetType(value: AXValueRef) -> AXValueType; fn AXValueGetValue(value: AXValueRef, value_type: AXValueType, value_ptr: *mut c_void) -> bool; + fn AXObserverCreateWithInfoCallback( + pid: i32, + callback: extern "C" fn(AXObserverRef, AXUIElementRef, CFStringRef, CFTypeRef, *mut c_void), + observer: *mut AXObserverRef, + ) -> AXError; + fn AXObserverGetRunLoopSource(observer: AXObserverRef) -> CFRunLoopSourceRef; + fn AXObserverAddNotification( + observer: AXObserverRef, + element: AXUIElementRef, + notification: CFStringRef, + refcon: *mut c_void, + ) -> AXError; } #[link(name = "AppKit", kind = "framework")] @@ -317,8 +351,263 @@ pub(super) fn root_kind(root: &RootInfo) -> RootKind { } } +fn should_activate_chromium(root: &RootInfo) -> bool { + is_chromium_bundle(&root.bundle_id) || lock_set(&WEB_SPARSE_PIDS).contains(&root.pid) +} + +fn is_chromium_bundle(bundle_id: &str) -> bool { + let bundle_id = bundle_id.to_ascii_lowercase(); + ["chrome", "chromium", "electron"] + .into_iter() + .any(|marker| bundle_id.contains(marker)) +} + +fn activate_chromium_accessibility(pid: u32, application: AXUIElementRef) { + for attribute_name in ["AXManualAccessibility", "AXEnhancedUserInterface"] { + let attribute = CFString::new(attribute_name); + // SAFETY: application is a live AX application element and both the + // attribute string and kCFBooleanTrue remain live for this call. + let code = unsafe { + AXUIElementSetAttributeValue( + application, + attribute.as_concrete_TypeRef(), + CFBoolean::true_value().as_CFTypeRef(), + ) + }; + if code != AX_SUCCESS { + log::debug!( + "Chromium AX activation attribute {attribute_name} was rejected for pid {pid}: {}", + ax_error_name(code) + ); + } + } + + let first_activation = lock_set(&ACTIVATED_PIDS).insert(pid); + if !first_activation { + return; + } + register_chromium_observer(pid, application); + std::thread::sleep(Duration::from_millis(300)); +} + +fn register_chromium_observer(pid: u32, application: AXUIElementRef) { + let Ok(pid_i32) = i32::try_from(pid) else { + log::debug!("Chromium AX observer pid {pid} is out of range"); + return; + }; + let mut observer: AXObserverRef = ptr::null(); + // SAFETY: the output pointer is valid, the callback has the verified AX + // ABI, and a null/failed observer is handled without dereferencing it. + let code = unsafe { + AXObserverCreateWithInfoCallback(pid_i32, chromium_observer_callback, &mut observer) + }; + if code != AX_SUCCESS || observer.is_null() { + log::debug!( + "could not create Chromium AX observer for pid {pid}: {}", + ax_error_name(code) + ); + return; + } + + for notification_name in CHROMIUM_NOTIFICATIONS { + let notification = CFString::new(notification_name); + let code = + add_chromium_notification(observer, application, notification.as_concrete_TypeRef()); + if code != AX_SUCCESS && code != -25210 { + log::debug!( + "Chromium AX notification {notification_name} was rejected for pid {pid}: {}", + ax_error_name(code) + ); + } + } + + // SAFETY: observer is a live create-rule AXObserver returned above; the + // borrowed run-loop source remains live while the observer is retained. + let source = unsafe { AXObserverGetRunLoopSource(observer) }; + if source.is_null() || add_observer_run_loop_source(source).is_none() { + log::debug!("could not attach Chromium AX observer for pid {pid} to its run loop"); + // SAFETY: observer is the create-rule object returned above and has not + // been handed to the persistent registry on this failure path. + unsafe { CFRelease(observer) }; + return; + } + + observer_registry() + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .insert(pid, observer as usize); +} + +const CHROMIUM_NOTIFICATIONS: [&str; 13] = [ + "AXFocusedUIElementChanged", + "AXFocusedWindowChanged", + "AXApplicationActivated", + "AXApplicationDeactivated", + "AXApplicationHidden", + "AXApplicationShown", + "AXWindowCreated", + "AXWindowMoved", + "AXWindowResized", + "AXValueChanged", + "AXTitleChanged", + "AXSelectedChildrenChanged", + "AXLayoutChanged", +]; + +extern "C" fn chromium_observer_callback( + _observer: AXObserverRef, + _element: AXUIElementRef, + _notification: CFStringRef, + _info: CFTypeRef, + _refcon: *mut c_void, +) { +} + +fn add_chromium_notification( + observer: AXObserverRef, + application: AXUIElementRef, + notification: CFStringRef, +) -> AXError { + if let Some(add_remote) = remote_notification_adder() { + // SAFETY: the function pointer was resolved with the verified private + // AX observer ABI and all supplied AX/CF references are live. + return unsafe { add_remote(observer, application, notification, ptr::null_mut()) }; + } + // SAFETY: observer, application, and notification are live values using + // the public AXObserverAddNotification ABI; null refcon is permitted. + unsafe { AXObserverAddNotification(observer, application, notification, ptr::null_mut()) } +} + +fn remote_notification_adder() -> Option { + static ADDER: OnceLock> = OnceLock::new(); + *ADDER.get_or_init(|| { + // SAFETY: the framework path and flags are valid C inputs; null means + // the optional private helper is unavailable. + let handle = unsafe { + dlopen( + c"/System/Library/Frameworks/ApplicationServices.framework/Frameworks/HIServices.framework/HIServices".as_ptr(), + RTLD_LAZY, + ) + }; + if handle.is_null() { + log::debug!( + "HIServices private AX helpers are unavailable; using public observer registration" + ); + return None; + } + // SAFETY: RTLD_DEFAULT is the verified macOS sentinel and the private + // symbol name is a static NUL-terminated C string. + let symbol = unsafe { + dlsym( + (-2_isize) as *mut c_void, + c"_AXObserverAddNotificationAndCheckRemote".as_ptr(), + ) + }; + if symbol.is_null() { + log::debug!( + "remote AX observer registration helper is unavailable; using public API" + ); + None + } else { + // SAFETY: the symbol uses the verified private AX observer + // registration ABI represented by AddNotificationAndCheckRemote. + Some(unsafe { + std::mem::transmute::<*mut c_void, AddNotificationAndCheckRemote>(symbol) + }) + } + }) +} + +struct ObserverRunLoop { + run_loop: usize, + first_source: usize, +} + +fn add_observer_run_loop_source(source: CFRunLoopSourceRef) -> Option<()> { + static RUN_LOOP: OnceLock> = OnceLock::new(); + let runtime = RUN_LOOP + .get_or_init(|| start_observer_run_loop(source)) + .as_ref()?; + if runtime.first_source != source as usize { + // SAFETY: the persistent run loop and borrowed AXObserver source are + // live; CFRunLoopAddSource retains the source in the default mode. + unsafe { + CFRunLoopAddSource( + runtime.run_loop as core_foundation::runloop::CFRunLoopRef, + source, + kCFRunLoopDefaultMode, + ); + CFRunLoopWakeUp(runtime.run_loop as core_foundation::runloop::CFRunLoopRef); + } + } + Some(()) +} + +fn start_observer_run_loop(source: CFRunLoopSourceRef) -> Option { + let source_address = source as usize; + let (ready_tx, ready_rx) = std::sync::mpsc::sync_channel(1); + let thread = std::thread::Builder::new() + .name("tcode-cu-ax-observer".into()) + .spawn(move || { + // SAFETY: this call obtains the current dedicated thread's live + // Core Foundation run loop. + let run_loop = unsafe { CFRunLoopGetCurrent() }; + // SAFETY: source_address is the live AXObserver source passed into + // this thread setup, and the default mode is a static CF value. + unsafe { + CFRunLoopAddSource( + run_loop, + source_address as CFRunLoopSourceRef, + kCFRunLoopDefaultMode, + ); + } + if ready_tx.send(run_loop as usize).is_err() { + return; + } + // SAFETY: this dedicated thread owns and runs its current run loop + // for the process lifetime to service persistent AX observers. + unsafe { CFRunLoopRun() }; + log::debug!("persistent Chromium AX observer run loop stopped unexpectedly"); + }); + if let Err(error) = thread { + log::debug!("could not spawn Chromium AX observer run-loop thread: {error}"); + return None; + } + match ready_rx.recv() { + Ok(run_loop) => Some(ObserverRunLoop { + run_loop, + first_source: source_address, + }), + Err(error) => { + log::debug!("Chromium AX observer run-loop thread failed during setup: {error}"); + None + } + } +} + +fn lock_set( + cell: &'static OnceLock>>, +) -> std::sync::MutexGuard<'static, HashSet> { + cell.get_or_init(|| Mutex::new(HashSet::new())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +fn observer_registry() -> &'static Mutex> { + AX_OBSERVERS.get_or_init(|| Mutex::new(HashMap::new())) +} + pub(super) fn observe_tree(root: &RootInfo) -> Result { - let (_application, window) = locate_window(root)?; + let application = create_application(root.pid).ok_or_else(|| { + BackendError::new( + BackendErrorCode::RootNotFound, + format!("could not create an AX application for pid {}", root.pid), + ) + })?; + if should_activate_chromium(root) { + activate_chromium_accessibility(root.pid, application.as_ax()); + } + let window = locate_window_in_application(application.as_ax(), root)?; let mut context = WalkContext { count: 0, visited: HashSet::new(), @@ -336,6 +625,9 @@ pub(super) fn observe_tree(root: &RootInfo) -> Result { if !tree.frame.has_area() { tree.frame = root.frame; } + if is_text_sparse(&tree) { + lock_set(&WEB_SPARSE_PIDS).insert(root.pid); + } Ok(tree) } @@ -398,8 +690,16 @@ fn locate_window(root: &RootInfo) -> Result<(OwnedCf, OwnedCf), BackendError> { format!("could not create an AX application for pid {}", root.pid), ) })?; - let windows = copy_attribute_elements(application.as_ax(), "AXWindows", 200); - let window = windows + let window = locate_window_in_application(application.as_ax(), root)?; + Ok((application, window)) +} + +fn locate_window_in_application( + application: AXUIElementRef, + root: &RootInfo, +) -> Result { + let windows = copy_attribute_elements(application, "AXWindows", 200); + windows .into_iter() .max_by(|left, right| { window_match_score(left.as_ax(), root) @@ -410,8 +710,7 @@ fn locate_window(root: &RootInfo) -> Result<(OwnedCf, OwnedCf), BackendError> { BackendErrorCode::RootNotFound, format!("no AX window matched root {}", root.ref_id), ) - })?; - Ok((application, window)) + }) } fn window_match_score(window: AXUIElementRef, root: &RootInfo) -> f64 { @@ -727,3 +1026,22 @@ fn ax_error_name(code: AXError) -> &'static str { _ => "unknown", } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn chromium_bundle_detection_is_case_insensitive_and_narrow() { + for bundle_id in [ + "com.google.Chrome", + "org.chromium.Chromium", + "com.example.ELECTRON.shell", + ] { + assert!(is_chromium_bundle(bundle_id)); + } + for bundle_id in ["com.apple.Safari", "com.example.chromatic", ""] { + assert!(!is_chromium_bundle(bundle_id)); + } + } +} diff --git a/crates/computer-use-mcp/src/backend/macos/background.rs b/crates/computer-use-mcp/src/backend/macos/background.rs new file mode 100644 index 00000000..6fbbb480 --- /dev/null +++ b/crates/computer-use-mcp/src/backend/macos/background.rs @@ -0,0 +1,690 @@ +use std::cell::Cell; +use std::ffi::c_void; +use std::os::raw::{c_char, c_int}; +use std::ptr; +use std::sync::OnceLock; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc; +use std::thread::{self, JoinHandle}; +use std::time::Duration; + +use core_foundation::base::{CFRelease, CFTypeRef}; +use core_foundation::mach_port::{ + CFMachPortCreateRunLoopSource, CFMachPortInvalidate, CFMachPortRef, +}; +use core_foundation::runloop::{ + CFRunLoopAddSource, CFRunLoopGetCurrent, CFRunLoopRef, CFRunLoopRun, CFRunLoopStop, + kCFRunLoopCommonModes, +}; +use core_graphics::event::CGEvent; +use core_graphics::geometry::CGPoint; +use core_graphics::sys::CGEventRef; + +use super::super::{BackendError, BackendErrorCode, MouseButton, RootInfo}; +use super::{ax, input}; +use crate::outline::Frame; + +const FIELD_MOUSE_CLICK_STATE: u32 = 1; +const FIELD_MOUSE_PRESSURE: u32 = 2; +const FIELD_TARGET_PID: u32 = 39; +const FIELD_TARGET_WINDOW: u32 = 51; +const FIELD_PRIVATE_ROUTING: u32 = 58; +const FIELD_WINDOW_UNDER_POINTER: u32 = 91; +const FIELD_WINDOW_UNDER_POINTER_CAN_HANDLE: u32 = 92; + +type SetWindowLocationFn = unsafe extern "C" fn(CGEventRef, CGPoint); +type Pid = c_int; + +const RTLD_LAZY: c_int = 1; + +#[link(name = "CoreGraphics", kind = "framework")] +unsafe extern "C" { + fn CGEventTapCreateForPid( + pid: Pid, + place: u32, + options: u32, + mask: u64, + callback: extern "C" fn(*const c_void, u32, CGEventRef, *mut c_void) -> CGEventRef, + user_info: *mut c_void, + ) -> CFMachPortRef; + fn CGEventTapEnable(tap: CFMachPortRef, enable: bool); + fn CGEventSetIntegerValueField(event: CGEventRef, field: u32, value: i64); + fn CGEventPostToPid(pid: Pid, event: CGEventRef); +} + +unsafe extern "C" { + fn dlopen(path: *const c_char, mode: c_int) -> *mut c_void; + fn dlsym(handle: *mut c_void, symbol: *const c_char) -> *mut c_void; +} + +#[link(name = "AppKit", kind = "framework")] +unsafe extern "C" {} + +#[link(name = "objc")] +unsafe extern "C" { + fn objc_getClass(name: *const std::ffi::c_char) -> *mut c_void; + fn sel_registerName(name: *const std::ffi::c_char) -> *mut c_void; + fn objc_msgSend(); +} + +pub(super) struct BackgroundDispatcher { + pid: Pid, + window_id: i64, + window_frame: Frame, + last_point: Cell, +} + +impl BackgroundDispatcher { + pub(super) fn new(root: &RootInfo) -> Result { + let pid = Pid::try_from(root.pid) + .map_err(|_| operation(format!("target pid {} is out of range", root.pid)))?; + if root.window_id == 0 || !root.frame.has_area() { + return Err(operation( + "background PID delivery requires a concrete window id and frame", + )); + } + let (center_x, center_y) = root.frame.center(); + Ok(Self { + pid, + window_id: i64::from(root.window_id), + window_frame: root.frame, + last_point: Cell::new(CGPoint::new(center_x, center_y)), + }) + } + + pub(super) fn click( + &self, + x: f64, + y: f64, + button: MouseButton, + click_count: u32, + ) -> Result<(), BackendError> { + let point = CGPoint::new(x, y); + for (index, (down, up)) in input::click_events(x, y, button, click_count)? + .into_iter() + .enumerate() + { + self.post_mouse(&down, point, i64::from(click_count), 1.0); + thread::sleep(Duration::from_millis(30)); + self.post_mouse(&up, point, i64::from(click_count), 0.0); + if index + 1 < click_count as usize { + thread::sleep(Duration::from_millis(45)); + } + } + self.last_point.set(point); + Ok(()) + } + + pub(super) fn move_mouse(&self, x: f64, y: f64) -> Result<(), BackendError> { + let point = CGPoint::new(x, y); + let event = input::move_mouse_event(x, y)?; + self.post_mouse(&event, point, 0, 0.0); + self.last_point.set(point); + Ok(()) + } + + pub(super) fn scroll(&self, x: f64, y: f64) -> Result<(), BackendError> { + let point = self.last_point.get(); + let event = input::scroll_event(x, y)?; + event.set_location(point); + event.set_integer_value_field(FIELD_WINDOW_UNDER_POINTER, self.window_id); + event.set_integer_value_field(FIELD_WINDOW_UNDER_POINTER_CAN_HANDLE, self.window_id); + self.stamp_addressing(&event); + set_window_location(&event, window_local_point(point, self.window_frame)); + event.post_to_pid(self.pid); + Ok(()) + } + + pub(super) fn drag(&self, path: &[[f64; 2]], button: MouseButton) -> Result<(), BackendError> { + let events = input::drag_events(path, button)?; + let final_index = events.len().saturating_sub(1); + for (index, event) in events.into_iter().enumerate() { + let point_index = index.min(path.len().saturating_sub(1)); + let point = CGPoint::new(path[point_index][0], path[point_index][1]); + let pressure = if index == final_index { 0.0 } else { 1.0 }; + self.post_mouse(&event, point, 1, pressure); + self.last_point.set(point); + if index > 0 && index < final_index { + thread::sleep(Duration::from_millis(12)); + } + } + Ok(()) + } + + pub(super) fn keypress(&self, keys: &[String]) -> Result<(), BackendError> { + for event in input::keypress_events(keys)? { + self.stamp_addressing(&event); + event.post_to_pid(self.pid); + } + Ok(()) + } + + pub(super) fn type_text(&self, text: &str) -> Result<(), BackendError> { + for [down, up] in input::text_event_pairs(text)? { + self.stamp_addressing(&down); + down.post_to_pid(self.pid); + self.stamp_addressing(&up); + up.post_to_pid(self.pid); + thread::sleep(Duration::from_millis(5)); + } + Ok(()) + } + + fn post_mouse(&self, event: &CGEvent, point: CGPoint, click_state: i64, pressure: f64) { + event.set_integer_value_field(FIELD_MOUSE_CLICK_STATE, click_state); + event.set_double_value_field(FIELD_MOUSE_PRESSURE, pressure); + event.set_integer_value_field(FIELD_WINDOW_UNDER_POINTER, self.window_id); + event.set_integer_value_field(FIELD_WINDOW_UNDER_POINTER_CAN_HANDLE, self.window_id); + self.stamp_addressing(event); + set_window_location(event, window_local_point(point, self.window_frame)); + event.post_to_pid(self.pid); + } + + fn stamp_addressing(&self, event: &CGEvent) { + stamp_addressing(event, self.pid, self.window_id); + } +} + +fn stamp_addressing(event: &CGEvent, pid: Pid, window_id: i64) { + event.set_integer_value_field(FIELD_TARGET_PID, i64::from(pid)); + event.set_integer_value_field(FIELD_TARGET_WINDOW, window_id); + event.set_integer_value_field(FIELD_PRIVATE_ROUTING, 1); +} + +fn window_local_point(point: CGPoint, frame: Frame) -> CGPoint { + CGPoint::new(point.x - frame.x, point.y - frame.y) +} + +fn set_window_location(event: &CGEvent, point: CGPoint) { + let Some(setter) = sky_light_set_window_location() else { + return; + }; + // SAFETY: the optional SkyLight symbol was resolved with the verified + // CGEventSetWindowLocation ABI, and `event` is live for this call. + unsafe { setter(raw_event_ref(event), point) }; +} + +fn raw_event_ref(event: &CGEvent) -> CGEventRef { + let borrowed: &core_graphics::event::CGEventRef = event; + std::ptr::from_ref(borrowed).cast_mut().cast() +} + +fn sky_light_set_window_location() -> Option { + static SETTER: OnceLock> = OnceLock::new(); + *SETTER.get_or_init(|| { + // SAFETY: the path and flags are valid C inputs; a null handle is + // treated as an unavailable optional private framework. + let handle = unsafe { + dlopen( + c"/System/Library/PrivateFrameworks/SkyLight.framework/SkyLight".as_ptr(), + RTLD_LAZY, + ) + }; + if handle.is_null() { + log::debug!("SkyLight is unavailable; skipping window-local event stamping"); + return None; + } + // SAFETY: RTLD_DEFAULT is the verified macOS sentinel and the symbol + // name is a static NUL-terminated C string. + let symbol = unsafe { + dlsym( + (-2_isize) as *mut c_void, + c"CGEventSetWindowLocation".as_ptr(), + ) + }; + if symbol.is_null() { + log::debug!( + "CGEventSetWindowLocation is unavailable; skipping window-local event stamping" + ); + None + } else { + // SAFETY: the resolved symbol uses the verified + // CGEventSetWindowLocation(CGEventRef, CGPoint) ABI. + Some(unsafe { std::mem::transmute::<*mut c_void, SetWindowLocationFn>(symbol) }) + } + }) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum TapKind { + Previous, + Target, +} + +pub(super) struct TapContext { + kind: TapKind, + armed: AtomicBool, +} + +impl TapContext { + fn new(kind: TapKind) -> Self { + Self { + kind, + armed: AtomicBool::new(true), + } + } +} + +pub(super) struct TapRuntime { + ports: [CFMachPortRef; 2], + run_loop: CFRunLoopRef, + thread: Option>, +} + +pub(super) enum BackgroundActivation { + AlreadyForeground, + Active { + target_pid: Pid, + window_id: i64, + runtime: TapRuntime, + previous_context: Box, + target_context: Box, + }, +} + +impl BackgroundActivation { + pub(super) fn acquire(root: &RootInfo) -> Result { + let previous_pid = ax::frontmost_application_pid().ok_or_else(|| { + operation("could not determine the frontmost application for background delivery") + })?; + if previous_pid == root.pid { + return Ok(Self::AlreadyForeground); + } + let previous_pid = Pid::try_from(previous_pid) + .map_err(|_| operation("frontmost application pid is out of range"))?; + let target_pid = Pid::try_from(root.pid) + .map_err(|_| operation("target application pid is out of range"))?; + if root.window_id == 0 || !root.frame.has_area() { + return Err(operation( + "background activation requires a concrete target window", + )); + } + + let mut previous_context = Box::new(TapContext::new(TapKind::Previous)); + let mut target_context = Box::new(TapContext::new(TapKind::Target)); + let previous_context_ptr = (&mut *previous_context as *mut TapContext) as usize; + let target_context_ptr = (&mut *target_context as *mut TapContext) as usize; + let (ready_tx, ready_rx) = mpsc::sync_channel(1); + let thread = thread::Builder::new() + .name("tcode-cu-background-taps".into()) + .spawn(move || { + run_tap_thread( + previous_pid, + target_pid, + previous_context_ptr, + target_context_ptr, + ready_tx, + ); + }) + .map_err(|error| operation(format!("could not spawn focus-tap thread: {error}")))?; + + let ready = match ready_rx.recv() { + Ok(Ok(ready)) => ready, + Ok(Err(error)) => { + let _ = thread.join(); + return Err(operation(error)); + } + Err(error) => { + let _ = thread.join(); + return Err(operation(format!( + "focus-tap thread stopped during setup: {error}" + ))); + } + }; + let guard = Self::Active { + target_pid, + window_id: i64::from(root.window_id), + runtime: TapRuntime { + ports: ready.ports.map(|port| port as CFMachPortRef), + run_loop: ready.run_loop as CFRunLoopRef, + thread: Some(thread), + }, + previous_context, + target_context, + }; + + post_appkit_event(target_pid, i64::from(root.window_id), 1); + thread::sleep(Duration::from_millis(20)); + let dispatcher = BackgroundDispatcher::new(root)?; + let (center_x, center_y) = root.frame.center(); + dispatcher.click(center_x, center_y, MouseButton::Left, 1)?; + Ok(guard) + } +} + +impl Drop for BackgroundActivation { + fn drop(&mut self) { + let Self::Active { + target_pid, + window_id, + runtime, + previous_context, + target_context, + } = self + else { + return; + }; + + if ax::frontmost_application_pid() != u32::try_from(*target_pid).ok() { + post_appkit_event(*target_pid, *window_id, 2); + thread::sleep(Duration::from_millis(20)); + } + previous_context.armed.store(false, Ordering::Release); + target_context.armed.store(false, Ordering::Release); + for port in runtime.ports { + // SAFETY: each port is a live create-rule event tap owned by the + // tap thread and remains live until that thread is joined below. + unsafe { CFMachPortInvalidate(port) }; + } + // SAFETY: this is the live dedicated run loop returned by the tap + // thread; stopping it is thread-safe and causes the thread to exit. + unsafe { CFRunLoopStop(runtime.run_loop) }; + if let Some(thread) = runtime.thread.take() + && thread.join().is_err() + { + log::debug!("background focus-tap thread panicked while stopping"); + } + } +} + +struct TapThreadReady { + ports: [usize; 2], + run_loop: usize, +} + +fn run_tap_thread( + previous_pid: Pid, + target_pid: Pid, + previous_context: usize, + target_context: usize, + ready: mpsc::SyncSender>, +) { + let previous = create_tap(previous_pid, previous_context as *mut c_void); + let previous = match previous { + Ok(tap) => tap, + Err(error) => { + let _ = ready.send(Err(error)); + return; + } + }; + let target = create_tap(target_pid, target_context as *mut c_void); + let target = match target { + Ok(tap) => tap, + Err(error) => { + release_tap(previous); + let _ = ready.send(Err(error)); + return; + } + }; + + // SAFETY: this runs on the dedicated tap thread and returns that thread's + // live run loop, to which the two valid sources are added. + let run_loop = unsafe { CFRunLoopGetCurrent() }; + // SAFETY: the run loop, sources, and tap ports are live; common-modes is a + // static Core Foundation mode constant and enabling a new tap is valid. + unsafe { + CFRunLoopAddSource(run_loop, previous.source, kCFRunLoopCommonModes); + CFRunLoopAddSource(run_loop, target.source, kCFRunLoopCommonModes); + CGEventTapEnable(previous.port, true); + CGEventTapEnable(target.port, true); + } + if ready + .send(Ok(TapThreadReady { + ports: [previous.port as usize, target.port as usize], + run_loop: run_loop as usize, + })) + .is_err() + { + release_tap(previous); + release_tap(target); + return; + } + // SAFETY: the current thread owns this dedicated run loop and runs it + // until BackgroundActivation::drop calls CFRunLoopStop. + unsafe { CFRunLoopRun() }; + release_tap(previous); + release_tap(target); +} + +struct RawTap { + port: CFMachPortRef, + source: core_foundation::runloop::CFRunLoopSourceRef, +} + +fn create_tap(pid: Pid, context: *mut c_void) -> Result { + // SAFETY: the callback has C ABI and `context` points to a boxed TapContext + // that the guard keeps alive until after this tap thread is joined. + let port = unsafe { CGEventTapCreateForPid(pid, 0, 0, u64::MAX, focus_tap_callback, context) }; + if port.is_null() { + return Err(format!( + "could not create the background focus-suppression tap for pid {pid}" + )); + } + // SAFETY: `port` is a live CFMachPort create-rule object; null source is + // handled as a recoverable setup failure. + let source = unsafe { CFMachPortCreateRunLoopSource(ptr::null(), port, 0) }; + if source.is_null() { + // SAFETY: `port` is the create-rule object returned above and is being + // invalidated and released exactly once on this failure path. + unsafe { + CFMachPortInvalidate(port); + CFRelease(port.cast::() as CFTypeRef); + } + return Err(format!( + "could not create a run-loop source for the background tap on pid {pid}" + )); + } + Ok(RawTap { port, source }) +} + +fn release_tap(tap: RawTap) { + // SAFETY: both values are create-rule Core Foundation objects owned by + // this tap thread and are released exactly once here. + unsafe { + CFMachPortInvalidate(tap.port); + CFRelease(tap.source.cast::() as CFTypeRef); + CFRelease(tap.port.cast::() as CFTypeRef); + } +} + +extern "C" fn focus_tap_callback( + _proxy: *const c_void, + event_type: u32, + event: CGEventRef, + user_info: *mut c_void, +) -> CGEventRef { + if user_info.is_null() { + return event; + } + // SAFETY: user_info points to the boxed TapContext kept alive by the guard + // until its tap has been invalidated and its run-loop thread joined. + let context = unsafe { &*(user_info as *const TapContext) }; + if should_drop_focus_event( + context.kind, + event_type, + context.armed.load(Ordering::Acquire), + ) { + ptr::null_mut() + } else { + event + } +} + +fn should_drop_focus_event(kind: TapKind, event_type: u32, armed: bool) -> bool { + armed && kind == TapKind::Previous && matches!(event_type, 13 | 19 | 20) +} + +fn post_appkit_event(pid: Pid, window_id: i64, subtype: i16) { + if window_id == 0 { + return; + } + let Some(pool) = ObjcPool::new() else { + log::debug!("could not create an autorelease pool for appKitDefined primer"); + return; + }; + // SAFETY: NSEvent is a stable AppKit class lookup; a null result is + // handled by skipping this optional primer. + let event_class = unsafe { objc_getClass(c"NSEvent".as_ptr()) }; + if event_class.is_null() { + log::debug!("NSEvent is unavailable; skipping appKitDefined primer"); + drop(pool); + return; + } + // SAFETY: this selector is the verified NSEvent class method and all + // arguments use their macOS ABI types. + let selector = unsafe { + sel_registerName(c"otherEventWithType:location:modifierFlags:timestamp:windowNumber:context:subtype:data1:data2:".as_ptr()) + }; + if selector.is_null() { + log::debug!("NSEvent primer selector is unavailable; skipping primer"); + return; + } + type OtherEvent = unsafe extern "C" fn( + *mut c_void, + *mut c_void, + usize, + CGPoint, + usize, + f64, + isize, + *mut c_void, + i16, + isize, + isize, + ) -> *mut c_void; + // SAFETY: objc_msgSend is cast to the exact verified NSEvent class-method + // ABI used immediately below. + let send_other: OtherEvent = + unsafe { std::mem::transmute(objc_msgSend as unsafe extern "C" fn()) }; + // SAFETY: receiver, selector, CGPoint, and scalar arguments match the + // OtherEvent ABI; returned nil is handled gracefully. + let ns_event = unsafe { + send_other( + event_class, + selector, + 13, + CGPoint::new(0.0, 0.0), + 0, + 0.0, + window_id as isize, + ptr::null_mut(), + subtype, + 0, + 0, + ) + }; + if ns_event.is_null() { + log::debug!("NSEvent returned nil; skipping appKitDefined primer subtype {subtype}"); + return; + } + // SAFETY: CGEvent is the stable property selector on a live NSEvent. + let cg_selector = unsafe { sel_registerName(c"CGEvent".as_ptr()) }; + if cg_selector.is_null() { + log::debug!("NSEvent CGEvent selector is unavailable; skipping primer"); + return; + } + type GetCgEvent = unsafe extern "C" fn(*mut c_void, *mut c_void) -> CGEventRef; + // SAFETY: objc_msgSend is cast to the verified zero-argument CGEvent + // property getter ABI used immediately below. + let get_cg_event: GetCgEvent = + unsafe { std::mem::transmute(objc_msgSend as unsafe extern "C" fn()) }; + // SAFETY: ns_event and cg_selector are live Objective-C values with the + // getter ABI above; returned nil is handled gracefully. + let cg_event = unsafe { get_cg_event(ns_event, cg_selector) }; + if cg_event.is_null() { + log::debug!("NSEvent CGEvent returned nil; skipping primer subtype {subtype}"); + return; + } + // SAFETY: cg_event is borrowed from the live NSEvent for this scope, and + // both CoreGraphics calls use the verified raw CGEvent ABI synchronously. + unsafe { + CGEventSetIntegerValueField(cg_event, FIELD_TARGET_PID, i64::from(pid)); + CGEventSetIntegerValueField(cg_event, FIELD_TARGET_WINDOW, window_id); + CGEventSetIntegerValueField(cg_event, FIELD_PRIVATE_ROUTING, 1); + CGEventPostToPid(pid, cg_event); + } +} + +struct ObjcPool(*mut c_void); + +impl ObjcPool { + fn new() -> Option { + // SAFETY: NSAutoreleasePool is a stable Foundation class lookup; null + // is handled as an unavailable optional primer path. + let class = unsafe { objc_getClass(c"NSAutoreleasePool".as_ptr()) }; + if class.is_null() { + return None; + } + // SAFETY: `new` is the standard zero-argument Objective-C selector. + let selector = unsafe { sel_registerName(c"new".as_ptr()) }; + if selector.is_null() { + return None; + } + type SendId = unsafe extern "C" fn(*mut c_void, *mut c_void) -> *mut c_void; + // SAFETY: objc_msgSend is cast to the exact zero-argument object-return + // ABI used for +[NSAutoreleasePool new]. + let send: SendId = unsafe { std::mem::transmute(objc_msgSend as unsafe extern "C" fn()) }; + // SAFETY: class and selector are valid for the SendId ABI above. + let pool = unsafe { send(class, selector) }; + (!pool.is_null()).then_some(Self(pool)) + } +} + +impl Drop for ObjcPool { + fn drop(&mut self) { + // SAFETY: `drain` is the stable zero-argument selector for the live + // NSAutoreleasePool owned by this guard. + let selector = unsafe { sel_registerName(c"drain".as_ptr()) }; + if selector.is_null() { + return; + } + type SendVoid = unsafe extern "C" fn(*mut c_void, *mut c_void); + // SAFETY: objc_msgSend is cast to the exact void-return, zero-argument + // ABI for -[NSAutoreleasePool drain]. + let send: SendVoid = unsafe { std::mem::transmute(objc_msgSend as unsafe extern "C" fn()) }; + // SAFETY: self.0 is the pool created in ObjcPool::new and selector is + // valid for the SendVoid ABI above. + unsafe { send(self.0, selector) }; + } +} + +fn operation(message: impl Into) -> BackendError { + BackendError::new(BackendErrorCode::OperationFailed, message) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn window_local_transform_uses_ax_top_left_origin() { + let frame = Frame { + x: -420.0, + y: 180.0, + w: 800.0, + h: 600.0, + }; + let point = window_local_point(CGPoint::new(-20.0, 530.0), frame); + assert_eq!((point.x, point.y), (400.0, 350.0)); + } + + #[test] + fn focus_tap_drops_only_previous_pid_focus_types_while_armed() { + for event_type in [13, 19, 20] { + assert!(should_drop_focus_event(TapKind::Previous, event_type, true)); + assert!(!should_drop_focus_event(TapKind::Target, event_type, true)); + assert!(!should_drop_focus_event( + TapKind::Previous, + event_type, + false + )); + } + for event_type in [0, 1, 12, 14, 18, 21, u32::MAX] { + assert!(!should_drop_focus_event( + TapKind::Previous, + event_type, + true + )); + } + } +} diff --git a/crates/computer-use-mcp/src/backend/macos/capture.rs b/crates/computer-use-mcp/src/backend/macos/capture.rs index db47d27a..ffb44fc0 100644 --- a/crates/computer-use-mcp/src/backend/macos/capture.rs +++ b/crates/computer-use-mcp/src/backend/macos/capture.rs @@ -1,6 +1,11 @@ use super::super::{BackendError, BackendErrorCode, RootInfo}; +use image::GenericImageView; use std::time::{SystemTime, UNIX_EPOCH}; +const MAX_LONG_EDGE: u32 = 1_568; +const MAX_PIXEL_AREA: u64 = 629_145; +const JPEG_QUALITY: u8 = 80; + pub(super) fn capture_window(root: &RootInfo) -> Result, BackendError> { let path = std::env::temp_dir().join(format!( "tcode-computer-use-{}-{}-{}.png", @@ -8,7 +13,7 @@ pub(super) fn capture_window(root: &RootInfo) -> Result, BackendError> { std::process::id(), SystemTime::now() .duration_since(UNIX_EPOCH) - .unwrap() + .unwrap_or_default() .as_nanos() )); let result = tcode_services::process::command("screencapture") @@ -27,12 +32,13 @@ pub(super) fn capture_window(root: &RootInfo) -> Result, BackendError> { }) .and_then(|status| { if status.success() { - std::fs::read(&path).map_err(|error| { + let png = std::fs::read(&path).map_err(|error| { BackendError::new( BackendErrorCode::CaptureFailed, format!("failed to read captured PNG: {error}"), ) - }) + })?; + png_to_scaled_jpeg(&png) } else { Err(BackendError::new( BackendErrorCode::CaptureFailed, @@ -43,3 +49,68 @@ pub(super) fn capture_window(root: &RootInfo) -> Result, BackendError> { let _ = std::fs::remove_file(path); result } + +fn png_to_scaled_jpeg(png: &[u8]) -> Result, BackendError> { + let image = + image::load_from_memory_with_format(png, image::ImageFormat::Png).map_err(|error| { + BackendError::new( + BackendErrorCode::CaptureFailed, + format!("failed to decode captured PNG: {error}"), + ) + })?; + let (width, height) = image.dimensions(); + let (scaled_width, scaled_height) = scaled_dimensions(width, height); + let image = if (scaled_width, scaled_height) == (width, height) { + image + } else { + image.resize_exact( + scaled_width, + scaled_height, + image::imageops::FilterType::Lanczos3, + ) + }; + let rgb = image.to_rgb8(); + let mut jpeg = Vec::new(); + image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg, JPEG_QUALITY) + .encode_image(&rgb) + .map_err(|error| { + BackendError::new( + BackendErrorCode::CaptureFailed, + format!("failed to encode captured JPEG: {error}"), + ) + })?; + Ok(jpeg) +} + +fn scaled_dimensions(width: u32, height: u32) -> (u32, u32) { + if width == 0 || height == 0 { + return (width, height); + } + let long_edge_scale = f64::from(MAX_LONG_EDGE) / f64::from(width.max(height)); + let area = u64::from(width) * u64::from(height); + let area_scale = (MAX_PIXEL_AREA as f64 / area as f64).sqrt(); + let scale = 1.0_f64.min(long_edge_scale).min(area_scale); + let scaled_width = (f64::from(width) * scale).floor().max(1.0) as u32; + let scaled_height = (f64::from(height) * scale).floor().max(1.0) as u32; + (scaled_width, scaled_height) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn screenshot_scaling_respects_edge_area_and_aspect_ratio() { + for (width, height) in [(320, 240), (4_000, 1_000), (1_920, 1_080), (900, 3_000)] { + let (scaled_width, scaled_height) = scaled_dimensions(width, height); + assert!(scaled_width <= width && scaled_height <= height); + assert!(scaled_width.max(scaled_height) <= MAX_LONG_EDGE); + assert!(u64::from(scaled_width) * u64::from(scaled_height) <= MAX_PIXEL_AREA); + let original_ratio = f64::from(width) / f64::from(height); + let scaled_ratio = f64::from(scaled_width) / f64::from(scaled_height); + let rounding_tolerance = 2.0 / f64::from(scaled_width.min(scaled_height)); + assert!((original_ratio - scaled_ratio).abs() <= rounding_tolerance); + } + assert_eq!(scaled_dimensions(320, 240), (320, 240)); + } +} diff --git a/crates/computer-use-mcp/src/backend/macos/focus.rs b/crates/computer-use-mcp/src/backend/macos/focus.rs index 17c2c074..454879f0 100644 --- a/crates/computer-use-mcp/src/backend/macos/focus.rs +++ b/crates/computer-use-mcp/src/backend/macos/focus.rs @@ -1,14 +1,11 @@ use std::time::{Duration, Instant}; -use core_graphics::display::CGDisplay; -use core_graphics::event::CGEvent; -use core_graphics::geometry::CGPoint; - use super::super::RootInfo; -use super::{ax, input}; +use super::ax; pub(super) struct FocusGuard { previous_pid: Option, + ready: bool, } impl FocusGuard { @@ -20,7 +17,10 @@ impl FocusGuard { return Self::noop(); }; if previous_pid == root.pid { - return Self::noop(); + return Self { + previous_pid: None, + ready: true, + }; } if !ax::activate_application(root.pid) { log::debug!( @@ -41,6 +41,7 @@ impl FocusGuard { Some(pid) if pid == root.pid => { return Self { previous_pid: Some(previous_pid), + ready: true, }; } Some(_) if Instant::now() < deadline => { @@ -66,7 +67,14 @@ impl FocusGuard { } fn noop() -> Self { - Self { previous_pid: None } + Self { + previous_pid: None, + ready: false, + } + } + + pub(super) fn is_ready(&self) -> bool { + self.ready } fn restore_after_failed_acquire(previous_pid: u32) { @@ -87,37 +95,3 @@ impl Drop for FocusGuard { } } } - -pub(super) struct CursorGuard { - saved_point: Option, -} - -impl CursorGuard { - pub(super) fn acquire() -> Self { - let saved_point = match input::event_source().and_then(|source| { - CGEvent::new(source).map_err(|()| { - super::super::BackendError::new( - super::super::BackendErrorCode::OperationFailed, - "CoreGraphics could not create an event to read the cursor position", - ) - }) - }) { - Ok(event) => Some(event.location()), - Err(error) => { - log::debug!("could not save the macOS cursor position: {error}"); - None - } - }; - Self { saved_point } - } -} - -impl Drop for CursorGuard { - fn drop(&mut self) { - if let Some(point) = self.saved_point - && let Err(error) = CGDisplay::warp_mouse_cursor_position(point) - { - log::debug!("could not restore the macOS cursor position: {error:?}"); - } - } -} diff --git a/crates/computer-use-mcp/src/backend/macos/input.rs b/crates/computer-use-mcp/src/backend/macos/input.rs index 0c274de0..7515f278 100644 --- a/crates/computer-use-mcp/src/backend/macos/input.rs +++ b/crates/computer-use-mcp/src/backend/macos/input.rs @@ -15,46 +15,24 @@ pub(super) fn click( button: MouseButton, click_count: u32, ) -> Result<(), BackendError> { - if !x.is_finite() || !y.is_finite() { - return Err(invalid("click coordinates must be finite")); - } - if !(1..=3).contains(&click_count) { - return Err(invalid("click_count must be between 1 and 3")); - } - let point = CGPoint::new(x, y); - let (down, up, cg_button) = mouse_types(button); - for click_index in 0..click_count { - let down_event = mouse_event(down, point, cg_button)?; - down_event.set_integer_value_field(EventField::MOUSE_EVENT_CLICK_STATE, click_count.into()); + for (click_index, (down_event, up_event)) in click_events(x, y, button, click_count)? + .into_iter() + .enumerate() + { down_event.post(CGEventTapLocation::HID); - let up_event = mouse_event(up, point, cg_button)?; - up_event.set_integer_value_field(EventField::MOUSE_EVENT_CLICK_STATE, click_count.into()); up_event.post(CGEventTapLocation::HID); - if click_index + 1 < click_count { + if click_index + 1 < click_count as usize { std::thread::sleep(Duration::from_millis(45)); } } Ok(()) } -pub(super) fn move_mouse(x: f64, y: f64) -> Result<(), BackendError> { - if !x.is_finite() || !y.is_finite() { - return Err(invalid("mouse coordinates must be finite")); - } - mouse_event( - CGEventType::MouseMoved, - CGPoint::new(x, y), - CGMouseButton::Left, - )? - .post(CGEventTapLocation::HID); - Ok(()) -} - -pub(super) fn scroll(x: f64, y: f64) -> Result<(), BackendError> { +pub(super) fn scroll_event(x: f64, y: f64) -> Result { if !x.is_finite() || !y.is_finite() { return Err(invalid("scroll deltas must be finite")); } - let event = CGEvent::new_scroll_event( + CGEvent::new_scroll_event( event_source()?, ScrollEventUnit::PIXEL, 2, @@ -62,12 +40,13 @@ pub(super) fn scroll(x: f64, y: f64) -> Result<(), BackendError> { x.round() as i32, 0, ) - .map_err(|()| operation("CoreGraphics could not create a scroll event"))?; - event.post(CGEventTapLocation::HID); - Ok(()) + .map_err(|()| operation("CoreGraphics could not create a scroll event")) } -pub(super) fn drag(path: &[[f64; 2]], button: MouseButton) -> Result<(), BackendError> { +pub(super) fn drag_events( + path: &[[f64; 2]], + button: MouseButton, +) -> Result, BackendError> { if path.len() < 2 { return Err(invalid("drag requires at least two path points")); } @@ -90,51 +69,101 @@ pub(super) fn drag(path: &[[f64; 2]], button: MouseButton) -> Result<(), Backend MouseButton::Middle => CGEventType::OtherMouseDragged, }; let first = CGPoint::new(path[0][0], path[0][1]); - mouse_event(down, first, cg_button)?.post(CGEventTapLocation::HID); + let mut events = vec![mouse_event(down, first, cg_button)?]; for point in &path[1..path.len() - 1] { - mouse_event(dragged, CGPoint::new(point[0], point[1]), cg_button)? - .post(CGEventTapLocation::HID); - std::thread::sleep(Duration::from_millis(12)); + events.push(mouse_event( + dragged, + CGPoint::new(point[0], point[1]), + cg_button, + )?); } let last = path[path.len() - 1]; let last = CGPoint::new(last[0], last[1]); - mouse_event(dragged, last, cg_button)?.post(CGEventTapLocation::HID); - mouse_event(up, last, cg_button)?.post(CGEventTapLocation::HID); - Ok(()) + events.push(mouse_event(dragged, last, cg_button)?); + events.push(mouse_event(up, last, cg_button)?); + Ok(events) } pub(super) fn keypress(keys: &[String]) -> Result<(), BackendError> { + for event in keypress_events(keys)? { + event.post(CGEventTapLocation::HID); + } + Ok(()) +} + +pub(super) fn keypress_events(keys: &[String]) -> Result<[CGEvent; 2], BackendError> { let chord = parse_key_chord(keys).map_err(invalid)?; let flags = modifier_flags(chord.modifiers); let down = CGEvent::new_keyboard_event(event_source()?, chord.keycode, true) .map_err(|()| operation("CoreGraphics could not create a key-down event"))?; down.set_flags(flags); - down.post(CGEventTapLocation::HID); let up = CGEvent::new_keyboard_event(event_source()?, chord.keycode, false) .map_err(|()| operation("CoreGraphics could not create a key-up event"))?; up.set_flags(flags); - up.post(CGEventTapLocation::HID); - Ok(()) + Ok([down, up]) } pub(super) fn type_text(text: &str) -> Result<(), BackendError> { - let chunks = unicode_chunks(text, 20); - for chunk in chunks { - let down = CGEvent::new_keyboard_event(event_source()?, 0, true) - .map_err(|()| operation("CoreGraphics could not create a Unicode key-down event"))?; - down.set_flags(CGEventFlags::CGEventFlagNull); - down.set_string_from_utf16_unchecked(&chunk); + for [down, up] in text_event_pairs(text)? { down.post(CGEventTapLocation::HID); - let up = CGEvent::new_keyboard_event(event_source()?, 0, false) - .map_err(|()| operation("CoreGraphics could not create a Unicode key-up event"))?; - up.set_flags(CGEventFlags::CGEventFlagNull); - up.set_string_from_utf16_unchecked(&chunk); up.post(CGEventTapLocation::HID); std::thread::sleep(Duration::from_millis(5)); } Ok(()) } +pub(super) fn text_event_pairs(text: &str) -> Result, BackendError> { + unicode_chunks(text, 20) + .into_iter() + .map(|chunk| { + let down = CGEvent::new_keyboard_event(event_source()?, 0, true).map_err(|()| { + operation("CoreGraphics could not create a Unicode key-down event") + })?; + down.set_flags(CGEventFlags::CGEventFlagNull); + down.set_string_from_utf16_unchecked(&chunk); + let up = CGEvent::new_keyboard_event(event_source()?, 0, false) + .map_err(|()| operation("CoreGraphics could not create a Unicode key-up event"))?; + up.set_flags(CGEventFlags::CGEventFlagNull); + up.set_string_from_utf16_unchecked(&chunk); + Ok([down, up]) + }) + .collect() +} + +pub(super) fn click_events( + x: f64, + y: f64, + button: MouseButton, + click_count: u32, +) -> Result, BackendError> { + validate_point(x, y, "click coordinates must be finite")?; + if !(1..=3).contains(&click_count) { + return Err(invalid("click_count must be between 1 and 3")); + } + let point = CGPoint::new(x, y); + let (down, up, cg_button) = mouse_types(button); + (0..click_count) + .map(|_| { + let down_event = mouse_event(down, point, cg_button)?; + down_event + .set_integer_value_field(EventField::MOUSE_EVENT_CLICK_STATE, click_count.into()); + let up_event = mouse_event(up, point, cg_button)?; + up_event + .set_integer_value_field(EventField::MOUSE_EVENT_CLICK_STATE, click_count.into()); + Ok((down_event, up_event)) + }) + .collect() +} + +pub(super) fn move_mouse_event(x: f64, y: f64) -> Result { + validate_point(x, y, "mouse coordinates must be finite")?; + mouse_event( + CGEventType::MouseMoved, + CGPoint::new(x, y), + CGMouseButton::Left, + ) +} + fn unicode_chunks(text: &str, maximum_units: usize) -> Vec> { let mut chunks = Vec::new(); let mut current = Vec::new(); @@ -152,7 +181,7 @@ fn unicode_chunks(text: &str, maximum_units: usize) -> Vec> { chunks } -pub(super) fn event_source() -> Result { +fn event_source() -> Result { CGEventSource::new(CGEventSourceStateID::HIDSystemState) .map_err(|()| operation("CoreGraphics could not create an event source")) } @@ -186,6 +215,14 @@ fn mouse_types(button: MouseButton) -> (CGEventType, CGEventType, CGMouseButton) } } +fn validate_point(x: f64, y: f64, message: &'static str) -> Result<(), BackendError> { + if x.is_finite() && y.is_finite() { + Ok(()) + } else { + Err(invalid(message)) + } +} + fn modifier_flags(modifiers: KeyModifiers) -> CGEventFlags { let mut flags = CGEventFlags::CGEventFlagNull; if modifiers.command { diff --git a/crates/computer-use-mcp/src/backend/macos/mod.rs b/crates/computer-use-mcp/src/backend/macos/mod.rs index 0f4d0f7d..2e2fac2e 100644 --- a/crates/computer-use-mcp/src/backend/macos/mod.rs +++ b/crates/computer-use-mcp/src/backend/macos/mod.rs @@ -1,4 +1,5 @@ mod ax; +mod background; mod capture; mod focus; mod input; @@ -18,12 +19,13 @@ use core_graphics::window::{ }; use super::{ - ActionKind, ActionRequest, ActionResult, BackendError, BackendErrorCode, ObserveRequest, - RootFilters, RootInfo, RootObservation, matches_root_filters, + ActionKind, ActionRequest, ActionResult, BackendError, BackendErrorCode, Delivery, + ObserveRequest, RootFilters, RootInfo, RootObservation, matches_root_filters, }; use crate::outline::{UiNode, is_text_sparse}; -use self::focus::{CursorGuard, FocusGuard}; +use self::background::{BackgroundActivation, BackgroundDispatcher}; +use self::focus::FocusGuard; pub(super) struct MacosBackend; @@ -113,14 +115,15 @@ impl MacosBackend { }; let text_sparse = is_text_sparse(&tree); let should_capture = request.capture.should_capture(text_sparse); - let screenshot_png = should_capture + let screenshot = should_capture .then(|| capture::capture_window(root)) .transpose()?; Ok(RootObservation { root: root.clone(), tree, text_sparse, - screenshot_png, + screenshot, + screenshot_mime: "image/jpeg", }) } @@ -133,8 +136,8 @@ impl MacosBackend { ActionKind::Press => { let target = target(root, request)?; Ok(match target.press() { - Ok(()) => ActionResult::worked("AXPress completed"), - Err(error) => ActionResult::didnt(error.to_string()), + Ok(()) => ActionResult::worked("AXPress completed", Delivery::Ax), + Err(error) => ActionResult::didnt(error.to_string(), Delivery::None), }) } ActionKind::Click => { @@ -143,21 +146,44 @@ impl MacosBackend { { let target = target(root, request)?; if target.press().is_ok() { - return Ok(ActionResult::worked("AXPress completed for click target")); + return Ok(ActionResult::worked( + "AXPress completed for click target", + Delivery::Ax, + )); } let (x, y) = target.frame().center(); - let _cursor_guard = CursorGuard::acquire(); - let _focus_guard = FocusGuard::acquire(root); - input::click(x, y, request.button, request.click_count)?; - return Ok(ActionResult::unknown( - "AXPress was rejected; physical click events were posted", - )); + return Ok( + match background(root, |dispatcher| { + dispatcher.click(x, y, request.button, request.click_count) + }) { + Ok(()) => ActionResult::unknown( + "AXPress was rejected; click events were posted directly to the target pid", + Delivery::BackgroundPid, + ), + Err(error) => ActionResult::didnt( + format!( + "AXPress was rejected; background click delivery failed: {error}" + ), + Delivery::None, + ), + }, + ); } let (x, y) = action_point(root, request)?; - let _cursor_guard = CursorGuard::acquire(); - let _focus_guard = FocusGuard::acquire(root); - input::click(x, y, request.button, request.click_count)?; - Ok(ActionResult::unknown("physical click events were posted")) + Ok( + match background(root, |dispatcher| { + dispatcher.click(x, y, request.button, request.click_count) + }) { + Ok(()) => ActionResult::unknown( + "click events were posted directly to the target pid", + Delivery::BackgroundPid, + ), + Err(error) => ActionResult::didnt( + format!("background click delivery failed: {error}"), + Delivery::None, + ), + }, + ) } ActionKind::SetText => { let text = request.text.as_deref().ok_or_else(|| { @@ -165,29 +191,44 @@ impl MacosBackend { })?; let target = target(root, request)?; match target.set_text(text) { - Ok(()) => Ok(ActionResult::worked("AXValue was set")), + Ok(()) => Ok(ActionResult::worked("AXValue was set", Delivery::Ax)), Err(ax_error) => { let click_point = if target.focus().is_err() { let frame = target.frame(); if !frame.has_area() { - return Ok(ActionResult::didnt(format!( - "{ax_error}; the target also rejected focus and has no clickable frame" - ))); + return Ok(ActionResult::didnt( + format!( + "{ax_error}; the target also rejected focus and has no clickable frame" + ), + Delivery::None, + )); } Some(frame.center()) } else { None }; - let _cursor_guard = click_point.map(|_| CursorGuard::acquire()); - let _focus_guard = FocusGuard::acquire(root); - if let Some((x, y)) = click_point { - input::click(x, y, super::MouseButton::Left, 1)?; - } - input::keypress(&["cmd+a".into()])?; - input::type_text(text)?; - Ok(ActionResult::unknown(format!( - "{ax_error}; keyboard replacement events were posted instead" - ))) + Ok( + match background(root, |dispatcher| { + if let Some((x, y)) = click_point { + dispatcher.click(x, y, super::MouseButton::Left, 1)?; + } + dispatcher.keypress(&["cmd+a".into()])?; + dispatcher.type_text(text) + }) { + Ok(()) => ActionResult::unknown( + format!( + "{ax_error}; keyboard replacement events were posted directly to the target pid" + ), + Delivery::BackgroundPid, + ), + Err(error) => ActionResult::didnt( + format!( + "{ax_error}; background keyboard replacement delivery failed: {error}" + ), + Delivery::None, + ), + }, + ) } } } @@ -202,6 +243,7 @@ impl MacosBackend { if !frame.has_area() { return Ok(ActionResult::didnt( "target rejected focus and has no clickable frame", + Delivery::None, )); } Some(frame.center()) @@ -211,13 +253,30 @@ impl MacosBackend { } else { None }; - let _cursor_guard = click_point.map(|_| CursorGuard::acquire()); - let _focus_guard = FocusGuard::acquire(root); - if let Some((x, y)) = click_point { - input::click(x, y, super::MouseButton::Left, 1)?; - } - input::type_text(text)?; - Ok(ActionResult::unknown("Unicode keyboard events were posted")) + let attempt = background(root, |dispatcher| { + if let Some((x, y)) = click_point { + dispatcher.click(x, y, super::MouseButton::Left, 1)?; + } + dispatcher.type_text(text) + }); + Ok(keyboard_result_with_optional_foreground( + root, + attempt, + |root| { + let focus_guard = FocusGuard::acquire(root); + if !focus_guard.is_ready() { + return Err(BackendError::new( + BackendErrorCode::OperationFailed, + "foreground HID retry could not activate and raise the target window", + )); + } + if let Some((x, y)) = click_point { + input::click(x, y, super::MouseButton::Left, 1)?; + } + input::type_text(text) + }, + "Unicode keyboard events", + )) } ActionKind::Keypress => { let keys = request.keys.as_deref().ok_or_else(|| { @@ -230,6 +289,7 @@ impl MacosBackend { if !frame.has_area() { return Ok(ActionResult::didnt( "keypress target rejected focus and has no clickable frame", + Delivery::None, )); } Some(frame.center()) @@ -239,13 +299,30 @@ impl MacosBackend { } else { None }; - let _cursor_guard = click_point.map(|_| CursorGuard::acquire()); - let _focus_guard = FocusGuard::acquire(root); - if let Some((x, y)) = click_point { - input::click(x, y, super::MouseButton::Left, 1)?; - } - input::keypress(keys)?; - Ok(ActionResult::unknown("keyboard events were posted")) + let attempt = background(root, |dispatcher| { + if let Some((x, y)) = click_point { + dispatcher.click(x, y, super::MouseButton::Left, 1)?; + } + dispatcher.keypress(keys) + }); + Ok(keyboard_result_with_optional_foreground( + root, + attempt, + |root| { + let focus_guard = FocusGuard::acquire(root); + if !focus_guard.is_ready() { + return Err(BackendError::new( + BackendErrorCode::OperationFailed, + "foreground HID retry could not activate and raise the target window", + )); + } + if let Some((x, y)) = click_point { + input::click(x, y, super::MouseButton::Left, 1)?; + } + input::keypress(keys) + }, + "keyboard events", + )) } ActionKind::Scroll => { let action_point = if request.target_path.is_some() @@ -255,33 +332,107 @@ impl MacosBackend { } else { None }; - let _cursor_guard = CursorGuard::acquire(); - let _focus_guard = FocusGuard::acquire(root); - if let Some((x, y)) = action_point { - input::move_mouse(x, y)?; - } - input::scroll( - request.scroll_x.unwrap_or(0.0), - request.scroll_y.unwrap_or(0.0), - )?; - Ok(ActionResult::unknown("scroll-wheel events were posted")) + Ok( + match background(root, |dispatcher| { + if let Some((x, y)) = action_point { + dispatcher.move_mouse(x, y)?; + } + dispatcher.scroll( + request.scroll_x.unwrap_or(0.0), + request.scroll_y.unwrap_or(0.0), + ) + }) { + Ok(()) => ActionResult::unknown( + "scroll-wheel events were posted directly to the target pid", + Delivery::BackgroundPid, + ), + Err(error) => ActionResult::didnt( + format!("background scroll delivery failed: {error}"), + Delivery::None, + ), + }, + ) } ActionKind::Drag => { let path = request.path.as_deref().ok_or_else(|| { BackendError::new(BackendErrorCode::InvalidAction, "drag requires a path") })?; - let _cursor_guard = CursorGuard::acquire(); - let _focus_guard = FocusGuard::acquire(root); - input::drag(path, request.button)?; - Ok(ActionResult::unknown("drag events were posted")) + Ok( + match background(root, |dispatcher| dispatcher.drag(path, request.button)) { + Ok(()) => ActionResult::unknown( + "drag events were posted directly to the target pid", + Delivery::BackgroundPid, + ), + Err(error) => ActionResult::didnt( + format!("background drag delivery failed: {error}"), + Delivery::None, + ), + }, + ) } ActionKind::MoveMouse => { let (x, y) = action_point(root, request)?; - let _focus_guard = FocusGuard::acquire(root); - input::move_mouse(x, y)?; - Ok(ActionResult::unknown("mouse-move event was posted")) + Ok( + match background(root, |dispatcher| dispatcher.move_mouse(x, y)) { + Ok(()) => ActionResult::unknown( + "mouse-move event was posted directly to the target pid", + Delivery::BackgroundPid, + ), + Err(error) => ActionResult::didnt( + format!("background mouse-move delivery failed: {error}"), + Delivery::None, + ), + }, + ) + } + } + } +} + +pub(super) fn frontmost_pid() -> Option { + ax::frontmost_application_pid() +} + +fn background( + root: &RootInfo, + action: impl FnOnce(&BackgroundDispatcher) -> Result<(), BackendError>, +) -> Result<(), BackendError> { + let _activation = BackgroundActivation::acquire(root)?; + let dispatcher = BackgroundDispatcher::new(root)?; + action(&dispatcher) +} + +fn keyboard_result_with_optional_foreground( + root: &RootInfo, + background_attempt: Result<(), BackendError>, + foreground_attempt: impl FnOnce(&RootInfo) -> Result<(), BackendError>, + action_name: &str, +) -> ActionResult { + match background_attempt { + Ok(()) => ActionResult::unknown( + format!("{action_name} were posted directly to the target pid"), + Delivery::BackgroundPid, + ), + Err(background_error) if crate::config::get().allow_foreground_fallback => { + match foreground_attempt(root) { + Ok(()) => ActionResult::unknown( + format!( + "background PID delivery failed ({background_error}); {action_name} were retried through foreground HID delivery" + ), + Delivery::ForegroundHid, + ), + Err(foreground_error) => ActionResult::didnt( + format!( + "background PID delivery failed ({background_error}); foreground HID retry also failed: {foreground_error}" + ), + Delivery::None, + ), } } + Err(error) => ActionResult::didnt( + format!("background PID delivery failed: {error}"), + Delivery::None, + ), } } diff --git a/crates/computer-use-mcp/src/backend/windows/mod.rs b/crates/computer-use-mcp/src/backend/windows/mod.rs index 90e9829e..8e0e074f 100644 --- a/crates/computer-use-mcp/src/backend/windows/mod.rs +++ b/crates/computer-use-mcp/src/backend/windows/mod.rs @@ -24,8 +24,8 @@ use windows::Win32::System::Threading::{ use windows::core::PWSTR; use super::{ - ActionKind, ActionRequest, ActionResult, BackendError, BackendErrorCode, ObserveRequest, - RootFilters, RootInfo, RootKind, RootObservation, matches_root_filters, + ActionKind, ActionRequest, ActionResult, BackendError, BackendErrorCode, Delivery, + ObserveRequest, RootFilters, RootInfo, RootKind, RootObservation, matches_root_filters, }; use crate::outline::{Frame, UiNode, canonical_role, is_text_sparse}; @@ -76,7 +76,7 @@ impl WindowsBackend { } }; let text_sparse = is_text_sparse(&tree); - let screenshot_png = if request.capture.should_capture(text_sparse) { + let screenshot = if request.capture.should_capture(text_sparse) { Some(match capture::capture_window(root) { Ok(png) => png, Err(error) => { @@ -107,7 +107,8 @@ impl WindowsBackend { root: root.clone(), tree, text_sparse, - screenshot_png, + screenshot, + screenshot_mime: "image/png", }) } @@ -120,7 +121,7 @@ impl WindowsBackend { ActionKind::Press => { let target = target(root, request)?; Ok(match target.press() { - Ok(message) => ActionResult::worked(message), + Ok(message) => ActionResult::worked(message, Delivery::Ax), Err(uia_error) => { let live_frame = target.frame(); let frame = live_frame @@ -128,17 +129,19 @@ impl WindowsBackend { .then_some(live_frame) .or_else(|| request.target_frame.filter(|frame| frame.has_area())); let Some(frame) = frame else { - return Ok(ActionResult::didnt(format!( - "{uia_error}; the target has no clickable frame" - ))); + return Ok(ActionResult::didnt( + format!("{uia_error}; the target has no clickable frame"), + Delivery::None, + )); }; let (x, y) = frame.center(); let _cursor_guard = CursorGuard::acquire(); let _foreground_guard = ForegroundGuard::acquire(root); input::click(x, y, super::MouseButton::Left, 1)?; - ActionResult::unknown(format!( - "{uia_error}; uiautomation mouse events were posted instead" - )) + ActionResult::unknown( + format!("{uia_error}; uiautomation mouse events were posted instead"), + Delivery::ForegroundHid, + ) } }) } @@ -149,7 +152,7 @@ impl WindowsBackend { { let target = target(root, request)?; match target.press() { - Ok(message) => return Ok(ActionResult::worked(message)), + Ok(message) => return Ok(ActionResult::worked(message, Delivery::Ax)), Err(uia_error) => { let live_frame = target.frame(); let frame = live_frame @@ -157,17 +160,21 @@ impl WindowsBackend { .then_some(live_frame) .or_else(|| request.target_frame.filter(|frame| frame.has_area())); let Some(frame) = frame else { - return Ok(ActionResult::didnt(format!( - "{uia_error}; the target has no clickable frame" - ))); + return Ok(ActionResult::didnt( + format!("{uia_error}; the target has no clickable frame"), + Delivery::None, + )); }; let (x, y) = frame.center(); let _cursor_guard = CursorGuard::acquire(); let _foreground_guard = ForegroundGuard::acquire(root); input::click(x, y, request.button, request.click_count)?; - return Ok(ActionResult::unknown(format!( - "{uia_error}; uiautomation mouse events were posted instead" - ))); + return Ok(ActionResult::unknown( + format!( + "{uia_error}; uiautomation mouse events were posted instead" + ), + Delivery::ForegroundHid, + )); } } } @@ -177,6 +184,7 @@ impl WindowsBackend { input::click(x, y, request.button, request.click_count)?; Ok(ActionResult::unknown( "uiautomation mouse events were posted", + Delivery::ForegroundHid, )) } ActionKind::SetText => { @@ -185,15 +193,18 @@ impl WindowsBackend { })?; let target = target(root, request)?; match target.set_text(text) { - Ok(message) => Ok(ActionResult::worked(message)), + Ok(message) => Ok(ActionResult::worked(message, Delivery::Ax)), Err(uia_error) => { let focus_failed = target.focus().is_err(); let click_point = if focus_failed { let frame = target.frame(); if !frame.has_area() { - return Ok(ActionResult::didnt(format!( - "{uia_error}; the target also rejected focus and has no clickable frame" - ))); + return Ok(ActionResult::didnt( + format!( + "{uia_error}; the target also rejected focus and has no clickable frame" + ), + Delivery::None, + )); } Some(frame.center()) } else { @@ -206,9 +217,12 @@ impl WindowsBackend { } input::keypress(&["ctrl+a".into()])?; input::type_text(text)?; - Ok(ActionResult::unknown(format!( - "{uia_error}; uiautomation keyboard replacement events were posted instead" - ))) + Ok(ActionResult::unknown( + format!( + "{uia_error}; uiautomation keyboard replacement events were posted instead" + ), + Delivery::ForegroundHid, + )) } } } @@ -223,6 +237,7 @@ impl WindowsBackend { if !frame.has_area() { return Ok(ActionResult::didnt( "target rejected focus and has no clickable frame", + Delivery::None, )); } Some(frame.center()) @@ -240,6 +255,7 @@ impl WindowsBackend { input::type_text(text)?; Ok(ActionResult::unknown( "uiautomation Unicode keyboard events were posted", + Delivery::ForegroundHid, )) } ActionKind::Keypress => { @@ -253,6 +269,7 @@ impl WindowsBackend { if !frame.has_area() { return Ok(ActionResult::didnt( "keypress target rejected focus and has no clickable frame", + Delivery::None, )); } Some(frame.center()) @@ -270,6 +287,7 @@ impl WindowsBackend { input::keypress(keys)?; Ok(ActionResult::unknown( "uiautomation keyboard events were posted", + Delivery::ForegroundHid, )) } ActionKind::Scroll => { @@ -279,19 +297,23 @@ impl WindowsBackend { if scroll_x == 0.0 && scroll_y == 0.0 { return Ok(ActionResult::worked( "scroll deltas were zero; no action was needed", + Delivery::None, )); } let target = scroll_target(root, request)?; match target.scroll(scroll_x, scroll_y) { - Ok(message) => Ok(ActionResult::worked(message)), + Ok(message) => Ok(ActionResult::worked(message, Delivery::Ax)), Err(uia_error) => { let frame = target.frame(); let focus_failed = target.focus().is_err(); let mouse_action = if focus_failed { if !frame.has_area() { - return Ok(ActionResult::didnt(format!( - "{uia_error}; the target rejected focus and has no frame for keyboard fallback" - ))); + return Ok(ActionResult::didnt( + format!( + "{uia_error}; the target rejected focus and has no frame for keyboard fallback" + ), + Delivery::None, + )); } Some((frame.center(), true)) } else if frame.has_area() { @@ -309,9 +331,12 @@ impl WindowsBackend { } } input::scroll_with_keyboard(scroll_x, scroll_y)?; - Ok(ActionResult::unknown(format!( - "{uia_error}; uiautomation keyboard scroll events were posted instead" - ))) + Ok(ActionResult::unknown( + format!( + "{uia_error}; uiautomation keyboard scroll events were posted instead" + ), + Delivery::ForegroundHid, + )) } } } @@ -324,6 +349,7 @@ impl WindowsBackend { input::drag(path, request.button)?; Ok(ActionResult::unknown( "uiautomation mouse drag events were posted", + Delivery::ForegroundHid, )) } ActionKind::MoveMouse => { @@ -332,6 +358,7 @@ impl WindowsBackend { input::move_mouse(x, y)?; Ok(ActionResult::unknown( "a uiautomation mouse-move event was posted", + Delivery::ForegroundHid, )) } } diff --git a/crates/computer-use-mcp/src/config.rs b/crates/computer-use-mcp/src/config.rs index e5ae44ad..5949e327 100644 --- a/crates/computer-use-mcp/src/config.rs +++ b/crates/computer-use-mcp/src/config.rs @@ -10,6 +10,8 @@ static CONFIG: RwLock = RwLock::new(ComputerUseSettings { enabled: false, allow_input: true, image_mode: ImageMode::Auto, + allow_foreground_fallback: false, + show_agent_cursor: true, }); pub fn set(config: ComputerUseSettings) { @@ -19,3 +21,26 @@ pub fn set(config: ComputerUseSettings) { pub fn get() -> ComputerUseSettings { CONFIG.read().unwrap().clone() } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn new_background_settings_default_for_empty_and_legacy_values() { + for json in [ + "{}", + r#"{"enabled":true,"image_mode":"always","allow_input":false}"#, + ] { + let settings: ComputerUseSettings = serde_json::from_str(json).unwrap(); + assert!(!settings.allow_foreground_fallback); + assert!(settings.show_agent_cursor); + } + + let configured: ComputerUseSettings = + serde_json::from_str(r#"{"allow_foreground_fallback":true,"show_agent_cursor":false}"#) + .unwrap(); + assert!(configured.allow_foreground_fallback); + assert!(!configured.show_agent_cursor); + } +} diff --git a/crates/computer-use-mcp/src/lib.rs b/crates/computer-use-mcp/src/lib.rs index 1782ab34..66bf94c2 100644 --- a/crates/computer-use-mcp/src/lib.rs +++ b/crates/computer-use-mcp/src/lib.rs @@ -13,6 +13,11 @@ pub mod permissions; pub mod state; pub mod tools; +/// Return the frontmost application pid on macOS, or `None` elsewhere. +pub fn frontmost_pid() -> Option { + backend::frontmost_pid() +} + /// A running computer-use MCP server and the bearer token required to access it. pub struct ComputerUseMcpServer { /// Streamable-HTTP endpoint, e.g. `http://127.0.0.1:53211/computer-use`. diff --git a/crates/computer-use-mcp/src/tools.rs b/crates/computer-use-mcp/src/tools.rs index 0ed9a653..9eece001 100644 --- a/crates/computer-use-mcp/src/tools.rs +++ b/crates/computer-use-mcp/src/tools.rs @@ -309,8 +309,8 @@ mod dispatch { use serde_json::json; use crate::backend::{ - ActionOutcome, ActionRequest, ActionResult, CapturePolicy, ObserveRequest, RootFilters, - RootInfo, RootObservation, + ActionOutcome, ActionRequest, ActionResult, CapturePolicy, Delivery, ObserveRequest, + RootFilters, RootInfo, RootObservation, }; use crate::outline::{self, UiNode}; @@ -540,18 +540,25 @@ mod dispatch { }; let mut step_results = Vec::new(); let mut stopped_at = None; + let mut activation = "none"; for (index, action) in params.actions.iter().enumerate() { let result = match prepare_action(&previous.tree, action) { Ok(request) => crate::backend::perform_action(&previous.root, &request) - .unwrap_or_else(|error| ActionResult::didnt(error.to_string())), - Err(error) => ActionResult::didnt(error), + .unwrap_or_else(|error| ActionResult::didnt(error.to_string(), Delivery::None)), + Err(error) => ActionResult::didnt(error, Delivery::None), }; let didnt = result.outcome == ActionOutcome::Didnt; + activation = match (activation, result.delivery) { + (_, Delivery::ForegroundHid) => "foreground", + ("none", Delivery::BackgroundPid) => "background", + (current, _) => current, + }; step_results.push(json!({ "index": index + 1, "action": action_name(action.action), "outcome": result.outcome, "message": result.message, + "delivery": result.delivery, })); if didnt { stopped_at = Some(index + 1); @@ -577,7 +584,7 @@ mod dispatch { let successor = crate::state::global().lock().unwrap().insert_observation( successor.root, successor.tree, - successor.screenshot_png, + successor.screenshot, ); let diff = outline::diff_trees(&previous.tree, &successor.tree); let expectation_failed = expectation_status == "failed"; @@ -597,6 +604,7 @@ mod dispatch { "state_id": successor.state_id, "previous_state_id": previous.state_id, "outcome": outcome, + "activation": activation, "stopped_at": stopped_at, "steps": step_results, "expect": expectation_status, @@ -719,7 +727,7 @@ mod dispatch { let successor = crate::state::global().lock().unwrap().insert_observation( observed.root, observed.tree, - observed.screenshot_png, + observed.screenshot, ); let status = if matched { "matched" } else { "timeout" }; let report = json!({ @@ -852,12 +860,18 @@ mod dispatch { } fn save_observation(observed: RootObservation, warning: Option<&str>) -> CallToolResult { - let screenshot_for_response = observed.screenshot_png.clone(); - let observation = crate::state::global().lock().unwrap().insert_observation( - observed.root, - observed.tree, - observed.screenshot_png, - ); + let RootObservation { + root, + tree, + text_sparse, + screenshot, + screenshot_mime, + } = observed; + let screenshot_for_response = screenshot.clone(); + let observation = crate::state::global() + .lock() + .unwrap() + .insert_observation(root, tree, screenshot); let mut text = format!( "state_id: {}\nroot: {} app=\"{}\" title=\"{}\"\nelements: {} interactive: {}", observation.state_id, @@ -867,7 +881,7 @@ mod dispatch { count_nodes(&observation.tree), outline::interactive_count(&observation.tree) ); - if observed.text_sparse { + if text_sparse { text.push_str("\ntext_sparse: true"); } if let Some(warning) = warning { @@ -877,10 +891,10 @@ mod dispatch { text.push('\n'); text.push_str(&outline::render_folded(&observation.tree)); let extra = screenshot_for_response - .map(|png| { + .map(|screenshot| { ContentBlock::image( - base64::engine::general_purpose::STANDARD.encode(png), - "image/png", + base64::engine::general_purpose::STANDARD.encode(screenshot), + screenshot_mime, ) }) .into_iter() @@ -1084,7 +1098,7 @@ mod dispatch { use super::*; use crate::outline::Frame; - fn sparse_observation(screenshot_png: Option>) -> RootObservation { + fn sparse_observation(screenshot: Option>) -> RootObservation { RootObservation { root: RootInfo { ref_id: "@r1".into(), @@ -1110,7 +1124,8 @@ mod dispatch { ..UiNode::default() }, text_sparse: true, - screenshot_png, + screenshot, + screenshot_mime: "image/jpeg", } } @@ -1129,13 +1144,12 @@ mod dispatch { assert_eq!(policy, CapturePolicy::IfSparse); assert!(policy.should_capture(true)); - let result = - save_observation(sparse_observation(Some(vec![0x89, b'P', b'N', b'G'])), None); + let result = save_observation(sparse_observation(Some(vec![0xff, 0xd8, 0xff])), None); assert!(matches!( result.content.as_slice(), [ContentBlock::Text(text), ContentBlock::Image(image)] if text.text.contains("text_sparse: true") - && image.mime_type == "image/png" + && image.mime_type == "image/jpeg" )); } diff --git a/crates/core/src/settings.rs b/crates/core/src/settings.rs index 76ea9c5c..9bb8ea35 100644 --- a/crates/core/src/settings.rs +++ b/crates/core/src/settings.rs @@ -567,6 +567,13 @@ pub struct ComputerUseSettings { /// Defaults to TRUE and tolerates an absent field in legacy files. #[serde(default = "default_true")] pub allow_input: bool, + /// Permit an opt-in foreground HID retry for keyboard actions when + /// background PID delivery cannot be initialized. + #[serde(default)] + pub allow_foreground_fallback: bool, + /// Show the agent cursor overlay. The overlay consumes this in a later PR. + #[serde(default = "default_true")] + pub show_agent_cursor: bool, } impl Default for ComputerUseSettings { @@ -575,6 +582,8 @@ impl Default for ComputerUseSettings { enabled: false, image_mode: ImageMode::default(), allow_input: true, + allow_foreground_fallback: false, + show_agent_cursor: true, } } } @@ -1270,6 +1279,8 @@ mod tests { // New fields tolerate an absent block: image mode auto, input allowed. assert_eq!(legacy.computer_use.image_mode, ImageMode::Auto); assert!(legacy.computer_use.allow_input); + assert!(!legacy.computer_use.allow_foreground_fallback); + assert!(legacy.computer_use.show_agent_cursor); // A legacy block that predates image_mode / allow_input still defaults // input ON (observe-only is opt-in, never the silent legacy behavior). @@ -1278,12 +1289,16 @@ mod tests { assert!(partial.computer_use.enabled); assert_eq!(partial.computer_use.image_mode, ImageMode::Auto); assert!(partial.computer_use.allow_input); + assert!(!partial.computer_use.allow_foreground_fallback); + assert!(partial.computer_use.show_agent_cursor); let settings = Settings { computer_use: ComputerUseSettings { enabled: true, image_mode: ImageMode::Always, allow_input: false, + allow_foreground_fallback: true, + show_agent_cursor: false, }, ..Settings::default() }; @@ -1293,6 +1308,8 @@ mod tests { assert!(back.computer_use.enabled); assert_eq!(back.computer_use.image_mode, ImageMode::Always); assert!(!back.computer_use.allow_input); + assert!(back.computer_use.allow_foreground_fallback); + assert!(!back.computer_use.show_agent_cursor); } #[test] diff --git a/docs/computer-use.md b/docs/computer-use.md index cabc9b5c..60a3866c 100644 --- a/docs/computer-use.md +++ b/docs/computer-use.md @@ -30,7 +30,9 @@ Core contract, inherited from pi-computer-use: `inspect_ui` query the full stored tree without touching the live UI. - **Honest outcomes.** `act_ui` reports `worked` / `didnt` / `unknown` per step, stops at the first failure (`stopped_at`), and never treats event delivery alone as semantic success when an - `expect` condition was given. + `expect` condition was given. Each step also reports `delivery` (`ax`, `background_pid`, + `foreground_hid`, or `none`), while the transaction reports `activation` (`none`, `background`, + or `foreground`). AX-only transactions therefore have `activation: "none"`. - **Bounded output.** Model-visible text is capped; oversized results return a preview plus a continuation ref for `read_text`. @@ -48,7 +50,8 @@ not run OCR or synthesize `pictureOnly` nodes; the model reads the attached pixe - `tools.rs` — rmcp `ToolRouter` (same streamable-HTTP + bearer-token shape as `preview-mcp` / `orchestrate-mcp`). - `backend/` — platform dispatch plus shared contracts. `backend/macos/` uses the AX C API - (`AXUIElement*`), CGEvent input synthesis, and `screencapture -l ` capture. + (`AXUIElement*`), per-process CGEvent input synthesis, and `screencapture -l ` + capture. `backend/windows/` is a thin adapter over the `uiautomation` crate for COM setup, Control View traversal, patterns, input, and GDI-backed screenshot capture. Other platforms get a stub backend whose tools return a clear "unsupported platform" error. @@ -80,8 +83,52 @@ observations: marker. - `never` never captures or attaches an image; sparse observations still include the marker. -The window is captured at most once per observation. The fallback is intentionally OCR-free and -does not add `pictureOnly` or other synthesized nodes. +The window is captured at most once per observation. On macOS, `screencapture`'s PNG is decoded, +downscaled while preserving aspect ratio (long edge at most 1568 pixels and area at most about +629,145 pixels), and returned as JPEG at quality 80 with MIME `image/jpeg`. Windows capture stays +PNG and is labeled `image/png`. The fallback is intentionally OCR-free and does not add +`pictureOnly` or other synthesized nodes. + +## macOS background input delivery + +AX-first actions remain background-safe: `AXPress`, setting `AXValue`, and setting `AXFocused` +are attempted before synthesized input. Coordinate, pointer, scroll, drag, and keyboard fallbacks +use `CGEventPostToPid` instead of the global HID tap. Every event is stamped with the target pid, +window number, and private window-routing field. Mouse events additionally carry click state, +pressure, and both window-under-pointer fields. When SkyLight's optional +`CGEventSetWindowLocation` symbol is available, events also receive a window-local point computed +directly from the AX top-left screen coordinates. Missing private symbols are treated as an +optional capability, not a crash condition. Because these events are never posted globally, the +system cursor does not move. + +When the target is not already frontmost, a `BackgroundActivation` guard installs one per-pid +event tap for the current app and one for the target on a dedicated CFRunLoop thread. While armed, +the current-app tap drops only focus-message event types 13, 19, and 20; other events and all +target-tap events pass through. The guard then sends the target window an AppKit-defined +application-activated event (subtype 1) and a PID-directed down/up click at the window center as a +readiness primer. On teardown it sends application-deactivated subtype 2 when the target is still +backgrounded, invalidates the taps, stops the run loop, joins its thread, and only then releases +the callback contexts. A nil NSEvent or unavailable private API is logged and skipped; inability +to establish safe focus suppression makes the action return `didnt` unless an eligible foreground +keyboard fallback is enabled. + +`allow_foreground_fallback` defaults to `false`. When enabled, only `type_text` and `keypress` may +retry through the legacy activate/raise foreground HID path, and only after background setup or +delivery fails. Pointer action kinds never activate or raise an app. `show_agent_cursor` defaults +to `true` and is persisted/plumbed through the computer-use configuration as the seam for the +later overlay work; this part does not render an overlay. + +## Chromium accessibility activation + +Before walking a Chromium/Electron AX tree (bundle id contains `chrome`, `chromium`, or `electron`, +case-insensitively), the backend best-effort sets `AXManualAccessibility` and +`AXEnhancedUserInterface` on the application element. A pid whose earlier tree was text-sparse is +also activated on its next observation, covering branded Electron apps whose bundle id does not +advertise the runtime. The first activation per pid creates a persistent AX observer, attaches its +source to a dedicated process-lifetime run loop, and subscribes to focus, application visibility, +window create/move/resize, value/title/selection, and layout notifications. It prefers the optional +remote-check registration symbol and falls back to public `AXObserverAddNotification`, then waits +about 300 ms before the first walk so the renderer can publish its complete tree. ## Windows backend @@ -128,6 +175,10 @@ Settings gains two pages: the next explicit action becomes **Open System Settings** and deep-links the matching `x-apple.systempreferences` pane. Returning to tcode also triggers a recheck. +The persisted computer-use block additionally accepts `allow_foreground_fallback` (default +`false`) and `show_agent_cursor` (default `true`). Both use serde defaults, so settings files from +before background delivery continue to load without migration. + ### Restart continuity macOS applies some grants (notably Screen Recording) only after the app restarts, and shows its From d57b61eb5708b6948f9fca211770b0d54055748a Mon Sep 17 00:00:00 2001 From: Tryanks Date: Tue, 1 Sep 2026 03:32:45 +0800 Subject: [PATCH 2/4] feat(computer-use): wire agent-cursor overlay into perform_action --- .../computer-use-mcp/src/backend/macos/mod.rs | 40 ++ .../src/backend/macos/overlay/border.rs | 267 +++++++++++ .../src/backend/macos/overlay/cursor.rs | 240 ++++++++++ .../src/backend/macos/overlay/ffi.rs | 432 ++++++++++++++++++ .../src/backend/macos/overlay/geometry.rs | 116 +++++ .../src/backend/macos/overlay/mod.rs | 210 +++++++++ 6 files changed, 1305 insertions(+) create mode 100644 crates/computer-use-mcp/src/backend/macos/overlay/border.rs create mode 100644 crates/computer-use-mcp/src/backend/macos/overlay/cursor.rs create mode 100644 crates/computer-use-mcp/src/backend/macos/overlay/ffi.rs create mode 100644 crates/computer-use-mcp/src/backend/macos/overlay/geometry.rs create mode 100644 crates/computer-use-mcp/src/backend/macos/overlay/mod.rs diff --git a/crates/computer-use-mcp/src/backend/macos/mod.rs b/crates/computer-use-mcp/src/backend/macos/mod.rs index 2e2fac2e..ca009ad4 100644 --- a/crates/computer-use-mcp/src/backend/macos/mod.rs +++ b/crates/computer-use-mcp/src/backend/macos/mod.rs @@ -3,6 +3,7 @@ mod background; mod capture; mod focus; mod input; +mod overlay; use std::collections::HashMap; use std::ffi::c_void; @@ -132,6 +133,7 @@ impl MacosBackend { root: &RootInfo, request: &ActionRequest, ) -> Result { + reflect_overlay(root, request); match request.kind { ActionKind::Press => { let target = target(root, request)?; @@ -436,6 +438,44 @@ fn keyboard_result_with_optional_foreground( } } +fn reflect_overlay(root: &RootInfo, request: &ActionRequest) { + let enabled = crate::config::get().show_agent_cursor; + overlay::set_enabled(enabled); + if !enabled { + return; + } + use overlay::OverlayActionKind as K; + let frame = root.frame; + match request.kind { + ActionKind::Drag => { + if let Some(path) = request.path.as_ref() + && let (Some(first), Some(last)) = (path.first(), path.last()) + { + overlay::show_drag((first[0], first[1]), (last[0], last[1]), frame); + return; + } + overlay::highlight_window(frame); + } + ActionKind::TypeText | ActionKind::SetText | ActionKind::Keypress => { + match action_point(root, request) { + Ok(point) => overlay::show_action(K::Keyboard, point, frame), + Err(_) => overlay::highlight_window(frame), + } + } + other => { + let kind = match other { + ActionKind::Scroll => K::Scroll, + ActionKind::MoveMouse => K::Move, + _ => K::Click, + }; + match action_point(root, request) { + Ok(point) => overlay::show_action(kind, point, frame), + Err(_) => overlay::highlight_window(frame), + } + } + } +} + fn target(root: &RootInfo, request: &ActionRequest) -> Result { let path = request.target_path.as_deref().ok_or_else(|| { BackendError::new( diff --git a/crates/computer-use-mcp/src/backend/macos/overlay/border.rs b/crates/computer-use-mcp/src/backend/macos/overlay/border.rs new file mode 100644 index 00000000..d525af1c --- /dev/null +++ b/crates/computer-use-mcp/src/backend/macos/overlay/border.rs @@ -0,0 +1,267 @@ +use std::ptr; + +use core_graphics::geometry::{CGPoint, CGRect, CGSize}; + +use super::ffi::{ + Id, class, send_id, send_id_color, send_id_cstr, send_id_f32, send_id_id, send_id_objects, + send_id_rect, send_id_rounded_rect, send_id_window_init, send_void, send_void_bool, + send_void_f32, send_void_f64, send_void_id, send_void_isize, send_void_point, send_void_rect, + send_void_rect_bool, send_void_size, send_void_two_ids, send_void_usize, status_window_level, +}; +use super::geometry::{BORDER_PADDING, DisplayGeometry, border_frame}; +use crate::outline::Frame; + +const NS_WINDOW_STYLE_BORDERLESS: usize = 0; +const NS_BACKING_STORE_BUFFERED: usize = 2; +const NS_WINDOW_COLLECTION_BEHAVIOR: usize = (1 << 0) | (1 << 3) | (1 << 9); +const FADE_DURATION: f64 = 0.3; +const CORNER_RADIUS: f64 = 12.0; + +pub(super) struct BorderUi { + window: Id, + container: Id, + glow: Id, + gradient: Id, + mask: Id, + visible: bool, +} + +impl BorderUi { + /// Must only be called from the process main queue. + pub(super) fn new() -> Option { + let window_class = class(c"NSWindow")?; + let allocated = send_id(window_class, c"alloc")?; + let window = send_id_window_init( + allocated, + c"initWithContentRect:styleMask:backing:defer:", + rect(0.0, 0.0, 1.0, 1.0), + NS_WINDOW_STYLE_BORDERLESS, + NS_BACKING_STORE_BUFFERED, + false, + )?; + let clear = send_id(class(c"NSColor")?, c"clearColor")?; + let configured = send_void_bool(window, c"setReleasedWhenClosed:", false) + && send_void_bool(window, c"setIgnoresMouseEvents:", true) + && send_void_bool(window, c"setOpaque:", false) + && send_void_bool(window, c"setHasShadow:", false) + && send_void_bool(window, c"setHidesOnDeactivate:", false) + && send_void_id(window, c"setBackgroundColor:", clear) + && send_void_usize( + window, + c"setCollectionBehavior:", + NS_WINDOW_COLLECTION_BEHAVIOR, + ) + && send_void_isize(window, c"setLevel:", status_window_level()); + if !configured { + return None; + } + + let view = send_id_rect( + send_id(class(c"NSView")?, c"alloc")?, + c"initWithFrame:", + rect(0.0, 0.0, 1.0, 1.0), + )?; + if !send_void_bool(view, c"setWantsLayer:", true) { + return None; + } + let root = send_id(view, c"layer")?; + let container = send_id(class(c"CALayer")?, c"layer")?; + let glow = send_id(class(c"CAShapeLayer")?, c"layer")?; + let gradient = send_id(class(c"CAGradientLayer")?, c"layer")?; + let mask = send_id(class(c"CAShapeLayer")?, c"layer")?; + + let glow_color = cg_color(0.91, 0.25, 0.72, 0.34)?; + let shadow_color = cg_color(0.30, 0.68, 1.0, 0.9)?; + let mask_color = cg_color(1.0, 1.0, 1.0, 1.0)?; + let colors = gradient_colors()?; + + let layers_configured = send_void_f32(container, c"setOpacity:", 0.0) + && send_void_id(glow, c"setFillColor:", ptr::null_mut()) + && send_void_id(glow, c"setStrokeColor:", glow_color) + && send_void_f64(glow, c"setLineWidth:", 13.0) + && send_void_id(glow, c"setShadowColor:", shadow_color) + && send_void_f32(glow, c"setShadowOpacity:", 0.75) + && send_void_f64(glow, c"setShadowRadius:", 22.0) + && send_void_size(glow, c"setShadowOffset:", CGSize::new(0.0, 0.0)) + && send_void_id(gradient, c"setColors:", colors) + && send_void_point(gradient, c"setStartPoint:", CGPoint::new(0.0, 0.25)) + && send_void_point(gradient, c"setEndPoint:", CGPoint::new(1.0, 0.75)) + && send_void_id(mask, c"setFillColor:", ptr::null_mut()) + && send_void_id(mask, c"setStrokeColor:", mask_color) + && send_void_f64(mask, c"setLineWidth:", 6.0) + && send_void_id(gradient, c"setMask:", mask) + && send_void_id(container, c"addSublayer:", glow) + && send_void_id(container, c"addSublayer:", gradient) + && send_void_id(root, c"addSublayer:", container) + && send_void_id(window, c"setContentView:", view); + if !layers_configured { + return None; + } + + Some(Self { + window, + container, + glow, + gradient, + mask, + visible: false, + }) + } + + /// Must only be called from the process main queue. + pub(super) fn show(&mut self, window_frame: Frame, display: DisplayGeometry) { + let outer = border_frame(window_frame, display); + let outer_rect = frame_rect(outer); + let bounds = rect(0.0, 0.0, outer.w, outer.h); + let inner = rect( + BORDER_PADDING, + BORDER_PADDING, + window_frame.w, + window_frame.h, + ); + let Some(path) = send_id_rounded_rect( + class(c"NSBezierPath").unwrap_or(ptr::null_mut()), + c"bezierPathWithRoundedRect:xRadius:yRadius:", + inner, + CORNER_RADIUS, + CORNER_RADIUS, + ) + .and_then(|path| send_id(path, c"CGPath")) else { + return; + }; + + let transaction = begin_without_implicit_animations(); + let updated = send_void_rect_bool(self.window, c"setFrame:display:", outer_rect, true) + && send_void_rect(self.container, c"setFrame:", bounds) + && send_void_rect(self.glow, c"setFrame:", bounds) + && send_void_rect(self.gradient, c"setFrame:", bounds) + && send_void_rect(self.mask, c"setFrame:", bounds) + && send_void_id(self.glow, c"setPath:", path) + && send_void_id(self.mask, c"setPath:", path); + end_transaction(transaction); + if !updated { + return; + } + + let _ = send_void(self.window, c"orderFrontRegardless"); + let from = if self.visible { 1.0 } else { 0.0 }; + animate_opacity(self.container, from, 1.0); + self.visible = true; + } + + /// Must only be called from the process main queue. + pub(super) fn hide(&mut self) { + if self.visible { + animate_opacity(self.container, 1.0, 0.0); + self.visible = false; + } + } +} + +fn gradient_colors() -> Option { + let colors = [ + cg_color(0.98, 0.66, 0.26, 0.95)?, + cg_color(0.94, 0.29, 0.48, 0.96)?, + cg_color(0.75, 0.42, 0.96, 0.94)?, + cg_color(0.31, 0.72, 1.0, 0.95)?, + ]; + send_id_objects( + class(c"NSArray")?, + c"arrayWithObjects:count:", + colors.as_ptr(), + colors.len(), + ) +} + +fn animate_opacity(layer: Id, from: f32, to: f32) { + let Some(key) = ns_string(c"tcode.agent-overlay.opacity") else { + let _ = send_void_f32(layer, c"setOpacity:", to); + return; + }; + let Some(key_path) = ns_string(c"opacity") else { + let _ = send_void_f32(layer, c"setOpacity:", to); + return; + }; + let Some(animation) = send_id_id( + class(c"CABasicAnimation").unwrap_or(ptr::null_mut()), + c"animationWithKeyPath:", + key_path, + ) else { + let _ = send_void_f32(layer, c"setOpacity:", to); + return; + }; + let Some(from_value) = number(from) else { + let _ = send_void_f32(layer, c"setOpacity:", to); + return; + }; + let Some(to_value) = number(to) else { + let _ = send_void_f32(layer, c"setOpacity:", to); + return; + }; + + let configured = send_void_id(animation, c"setFromValue:", from_value) + && send_void_id(animation, c"setToValue:", to_value) + && send_void_f64(animation, c"setDuration:", FADE_DURATION); + if let Some(timing) = timing_function() { + let _ = send_void_id(animation, c"setTimingFunction:", timing); + } + + let transaction = begin_without_implicit_animations(); + let model_updated = send_void_f32(layer, c"setOpacity:", to); + end_transaction(transaction); + if configured && model_updated { + let _ = send_void_two_ids(layer, c"addAnimation:forKey:", animation, key); + } +} + +fn begin_without_implicit_animations() -> Option { + let transaction = class(c"CATransaction")?; + if !send_void(transaction, c"begin") { + return None; + } + let _ = send_void_bool(transaction, c"setDisableActions:", true); + Some(transaction) +} + +fn end_transaction(transaction: Option) { + if let Some(transaction) = transaction { + let _ = send_void(transaction, c"commit"); + } +} + +fn timing_function() -> Option { + let name = ns_string(c"easeInEaseOut")?; + send_id_id(class(c"CAMediaTimingFunction")?, c"functionWithName:", name) +} + +fn number(value: f32) -> Option { + send_id_f32(class(c"NSNumber")?, c"numberWithFloat:", value) +} + +fn ns_string(value: &std::ffi::CStr) -> Option { + send_id_cstr( + class(c"NSString")?, + c"stringWithUTF8String:", + value.as_ptr(), + ) +} + +fn cg_color(red: f64, green: f64, blue: f64, alpha: f64) -> Option { + let color = send_id_color( + class(c"NSColor")?, + c"colorWithSRGBRed:green:blue:alpha:", + red, + green, + blue, + alpha, + )?; + send_id(color, c"CGColor") +} + +fn frame_rect(frame: Frame) -> CGRect { + rect(frame.x, frame.y, frame.w, frame.h) +} + +fn rect(x: f64, y: f64, width: f64, height: f64) -> CGRect { + CGRect::new(&CGPoint::new(x, y), &CGSize::new(width, height)) +} diff --git a/crates/computer-use-mcp/src/backend/macos/overlay/cursor.rs b/crates/computer-use-mcp/src/backend/macos/overlay/cursor.rs new file mode 100644 index 00000000..6c6ae368 --- /dev/null +++ b/crates/computer-use-mcp/src/backend/macos/overlay/cursor.rs @@ -0,0 +1,240 @@ +use std::ptr; + +use core_graphics::geometry::{CGPoint, CGRect, CGSize}; + +use super::OverlayActionKind; +use super::ffi::{ + Id, class, send_id, send_id_color, send_id_cstr, send_id_id, send_id_rect, send_id_window_init, + send_void, send_void_bool, send_void_f32, send_void_f64, send_void_id, send_void_isize, + send_void_point, send_void_rect, send_void_size, send_void_usize, status_window_level, +}; +use super::geometry::{DisplayGeometry, ax_screen_to_appkit}; + +const CURSOR_SIZE: f64 = 40.0; +const HOTSPOT_X: f64 = 4.0; +const HOTSPOT_Y: f64 = 35.0; +const ANIMATION_DURATION: f64 = 0.25; + +const NS_WINDOW_STYLE_NONACTIVATING_PANEL: usize = 1 << 7; +const NS_BACKING_STORE_BUFFERED: usize = 2; +const NS_WINDOW_COLLECTION_BEHAVIOR: usize = (1 << 0) | (1 << 3) | (1 << 9); + +pub(super) struct CursorUi { + window: Id, + shape: Id, + visible: bool, +} + +impl CursorUi { + /// Must only be called from the process main queue. + pub(super) fn new() -> Option { + let panel_class = class(c"NSPanel")?; + let panel = send_id(panel_class, c"alloc")?; + let window = send_id_window_init( + panel, + c"initWithContentRect:styleMask:backing:defer:", + rect(-CURSOR_SIZE, -CURSOR_SIZE, CURSOR_SIZE, CURSOR_SIZE), + NS_WINDOW_STYLE_NONACTIVATING_PANEL, + NS_BACKING_STORE_BUFFERED, + false, + )?; + + let clear = send_id(class(c"NSColor")?, c"clearColor")?; + let configured = send_void_bool(window, c"setReleasedWhenClosed:", false) + && send_void_bool(window, c"setIgnoresMouseEvents:", true) + && send_void_bool(window, c"setOpaque:", false) + && send_void_bool(window, c"setHasShadow:", false) + && send_void_bool(window, c"setHidesOnDeactivate:", false) + && send_void_id(window, c"setBackgroundColor:", clear) + && send_void_usize( + window, + c"setCollectionBehavior:", + NS_WINDOW_COLLECTION_BEHAVIOR, + ) + && send_void_isize(window, c"setLevel:", status_window_level()); + if !configured { + return None; + } + + let view = send_id_rect( + send_id(class(c"NSView")?, c"alloc")?, + c"initWithFrame:", + rect(0.0, 0.0, CURSOR_SIZE, CURSOR_SIZE), + )?; + if !send_void_bool(view, c"setWantsLayer:", true) { + return None; + } + let root_layer = send_id(view, c"layer")?; + let shape = send_id(class(c"CAShapeLayer")?, c"layer")?; + let path = cursor_path()?; + let fill = cg_color(0.08, 0.09, 0.12, 0.98)?; + let stroke = cg_color(0.93, 0.35, 0.72, 1.0)?; + let shadow = cg_color(0.0, 0.0, 0.0, 0.8)?; + + let shape_configured = send_void_id(shape, c"setPath:", path) + && send_void_rect( + shape, + c"setFrame:", + rect(0.0, 0.0, CURSOR_SIZE, CURSOR_SIZE), + ) + && send_void_id(shape, c"setFillColor:", fill) + && send_void_id(shape, c"setStrokeColor:", stroke) + && send_void_f64(shape, c"setLineWidth:", 2.0) + && send_void_id(shape, c"setShadowColor:", shadow) + && send_void_f32(shape, c"setShadowOpacity:", 0.55) + && send_void_f64(shape, c"setShadowRadius:", 2.5) + && send_void_size(shape, c"setShadowOffset:", CGSize::new(0.0, -1.0)) + && send_void_id(root_layer, c"addSublayer:", shape) + && send_void_id(window, c"setContentView:", view); + if !shape_configured { + return None; + } + + Some(Self { + window, + shape, + visible: false, + }) + } + + /// Must only be called from the process main queue. + pub(super) fn show( + &mut self, + kind: OverlayActionKind, + ax_point: (f64, f64), + display: DisplayGeometry, + ) { + self.set_kind(kind); + let appkit_point = ax_screen_to_appkit(ax_point, display); + if !self.visible { + let spawn = (display.appkit.x - 48.0, display.appkit.y - 48.0); + let _ = send_void_point(self.window, c"setFrameOrigin:", window_origin(spawn)); + self.visible = true; + } + let _ = send_void(self.window, c"orderFrontRegardless"); + let _ = send_void(self.window, c"displayIfNeeded"); + animate_window_origin(self.window, window_origin(appkit_point)); + } + + /// Must only be called from the process main queue. + pub(super) fn show_drag( + &mut self, + from_ax: (f64, f64), + to_ax: (f64, f64), + from_display: DisplayGeometry, + to_display: DisplayGeometry, + ) { + self.set_kind(OverlayActionKind::Drag); + let from = ax_screen_to_appkit(from_ax, from_display); + let to = ax_screen_to_appkit(to_ax, to_display); + let _ = send_void_point(self.window, c"setFrameOrigin:", window_origin(from)); + let _ = send_void(self.window, c"orderFrontRegardless"); + let _ = send_void(self.window, c"displayIfNeeded"); + self.visible = true; + animate_window_origin(self.window, window_origin(to)); + } + + /// Must only be called from the process main queue. + pub(super) fn hide(&mut self) { + if self.visible { + let _ = send_void_id(self.window, c"orderOut:", ptr::null_mut()); + self.visible = false; + } + } + + fn set_kind(&self, kind: OverlayActionKind) { + let (red, green, blue) = match kind { + OverlayActionKind::Click => (0.93, 0.35, 0.72), + OverlayActionKind::Scroll => (0.30, 0.78, 0.96), + OverlayActionKind::Drag => (0.66, 0.43, 0.96), + OverlayActionKind::Keyboard => (0.98, 0.66, 0.28), + OverlayActionKind::Move => (0.32, 0.66, 1.0), + }; + if let Some(color) = cg_color(red, green, blue, 1.0) { + let _ = send_void_id(self.shape, c"setStrokeColor:", color); + } + } +} + +fn cursor_path() -> Option { + let path = send_id(class(c"NSBezierPath")?, c"bezierPath")?; + for (index, point) in [ + (4.0, 35.0), + (4.0, 7.0), + (11.5, 14.0), + (17.5, 3.5), + (22.0, 6.0), + (16.0, 16.5), + (26.0, 16.5), + ] + .into_iter() + .enumerate() + { + let selector = if index == 0 { + c"moveToPoint:" + } else { + c"lineToPoint:" + }; + if !send_void_point(path, selector, CGPoint::new(point.0, point.1)) { + return None; + } + } + if !send_void(path, c"closePath") { + return None; + } + send_id(path, c"CGPath") +} + +fn animate_window_origin(window: Id, origin: CGPoint) { + let Some(context_class) = class(c"NSAnimationContext") else { + let _ = send_void_point(window, c"setFrameOrigin:", origin); + return; + }; + if !send_void(context_class, c"beginGrouping") { + let _ = send_void_point(window, c"setFrameOrigin:", origin); + return; + } + + let animated = send_id(context_class, c"currentContext").is_some_and(|context| { + let duration_set = send_void_f64(context, c"setDuration:", ANIMATION_DURATION); + if let Some(timing) = timing_function() { + let _ = send_void_id(context, c"setTimingFunction:", timing); + } + let moved = send_id(window, c"animator") + .is_some_and(|animator| send_void_point(animator, c"setFrameOrigin:", origin)); + duration_set && moved + }); + let _ = send_void(context_class, c"endGrouping"); + if !animated { + let _ = send_void_point(window, c"setFrameOrigin:", origin); + } +} + +fn timing_function() -> Option { + let name = send_id_cstr( + class(c"NSString")?, + c"stringWithUTF8String:", + c"easeOut".as_ptr(), + )?; + send_id_id(class(c"CAMediaTimingFunction")?, c"functionWithName:", name) +} + +fn cg_color(red: f64, green: f64, blue: f64, alpha: f64) -> Option { + let color = send_id_color( + class(c"NSColor")?, + c"colorWithSRGBRed:green:blue:alpha:", + red, + green, + blue, + alpha, + )?; + send_id(color, c"CGColor") +} + +fn window_origin(point: (f64, f64)) -> CGPoint { + CGPoint::new(point.0 - HOTSPOT_X, point.1 - HOTSPOT_Y) +} + +fn rect(x: f64, y: f64, width: f64, height: f64) -> CGRect { + CGRect::new(&CGPoint::new(x, y), &CGSize::new(width, height)) +} diff --git a/crates/computer-use-mcp/src/backend/macos/overlay/ffi.rs b/crates/computer-use-mcp/src/backend/macos/overlay/ffi.rs new file mode 100644 index 00000000..e9dc770c --- /dev/null +++ b/crates/computer-use-mcp/src/backend/macos/overlay/ffi.rs @@ -0,0 +1,432 @@ +use std::ffi::{CStr, c_char, c_void}; +use std::mem; + +use core_graphics::geometry::{CGPoint, CGRect, CGSize}; + +use super::geometry::DisplayGeometry; +use crate::outline::Frame; + +pub(super) type Id = *mut c_void; +type Sel = *mut c_void; +// SAFETY: this callback ABI is the dispatch_function_t signature from libdispatch. +pub(super) type DispatchFn = unsafe extern "C" fn(*mut c_void); + +// SAFETY: these declarations match the Objective-C runtime's public C ABI. +#[link(name = "objc")] +unsafe extern "C" { + fn objc_getClass(name: *const c_char) -> Id; + fn sel_registerName(name: *const c_char) -> Sel; + fn objc_msgSend(); +} + +// SAFETY: AppKit is linked for the Objective-C window and view classes used below. +#[link(name = "AppKit", kind = "framework")] +unsafe extern "C" {} + +// SAFETY: QuartzCore is linked for the Objective-C layer classes used below. +#[link(name = "QuartzCore", kind = "framework")] +unsafe extern "C" {} + +// SAFETY: these declarations match CoreGraphics' public display C ABI. +#[link(name = "CoreGraphics", kind = "framework")] +unsafe extern "C" { + fn CGDisplayBounds(display: u32) -> CGRect; + fn CGGetDisplaysWithPoint( + point: CGPoint, + max_displays: u32, + displays: *mut u32, + display_count: *mut u32, + ) -> i32; + fn CGMainDisplayID() -> u32; + fn CGWindowLevelForKey(key: i32) -> i32; +} + +// SAFETY: libdispatch is a libSystem component and these declarations match its C ABI. +#[link(name = "System")] +unsafe extern "C" { + static _dispatch_main_q: c_void; + fn dispatch_async_f(queue: Id, context: *mut c_void, work: DispatchFn); +} + +macro_rules! invoke { + ($return_type:ty, $receiver:expr, $selector:expr $(, $argument_type:ty => $argument:expr)* $(,)?) => {{ + // SAFETY: each wrapper below fixes the function signature to the documented + // Objective-C ABI of the selectors for which that wrapper is used. + let function: unsafe extern "C" fn(Id, Sel $(, $argument_type)*) -> $return_type = + unsafe { mem::transmute(objc_msgSend as unsafe extern "C" fn()) }; + // SAFETY: receiver and selector were checked for null, and each caller uses + // the wrapper whose fixed signature matches the selector's documented ABI. + unsafe { function($receiver, $selector $(, $argument)*) } + }}; +} + +pub(super) fn class(name: &CStr) -> Option { + // SAFETY: name is a live, nul-terminated C string for this call. + let value = unsafe { objc_getClass(name.as_ptr()) }; + (!value.is_null()).then_some(value) +} + +fn selector(name: &CStr) -> Option { + // SAFETY: name is a live, nul-terminated C string for this call. + let value = unsafe { sel_registerName(name.as_ptr()) }; + (!value.is_null()).then_some(value) +} + +fn can_send(receiver: Id, target: Sel) -> bool { + if receiver.is_null() || target.is_null() { + return false; + } + let Some(check) = selector(c"respondsToSelector:") else { + return false; + }; + invoke!(i8, receiver, check, Sel => target) != 0 +} + +pub(super) fn send_id(receiver: Id, name: &CStr) -> Option { + let selector = selector(name)?; + if !can_send(receiver, selector) { + return None; + } + let value = invoke!(Id, receiver, selector); + (!value.is_null()).then_some(value) +} + +pub(super) fn send_id_cstr(receiver: Id, name: &CStr, value: *const c_char) -> Option { + let selector = selector(name)?; + if !can_send(receiver, selector) || value.is_null() { + return None; + } + let result = invoke!(Id, receiver, selector, *const c_char => value); + (!result.is_null()).then_some(result) +} + +pub(super) fn send_id_id(receiver: Id, name: &CStr, value: Id) -> Option { + let selector = selector(name)?; + if !can_send(receiver, selector) { + return None; + } + let result = invoke!(Id, receiver, selector, Id => value); + (!result.is_null()).then_some(result) +} + +pub(super) fn send_id_rect(receiver: Id, name: &CStr, rect: CGRect) -> Option { + let selector = selector(name)?; + if !can_send(receiver, selector) { + return None; + } + let value = invoke!(Id, receiver, selector, CGRect => rect); + (!value.is_null()).then_some(value) +} + +pub(super) fn send_id_window_init( + receiver: Id, + name: &CStr, + rect: CGRect, + style: usize, + backing: usize, + defer: bool, +) -> Option { + let selector = selector(name)?; + if !can_send(receiver, selector) { + return None; + } + let value = invoke!( + Id, + receiver, + selector, + CGRect => rect, + usize => style, + usize => backing, + i8 => i8::from(defer), + ); + (!value.is_null()).then_some(value) +} + +pub(super) fn send_id_rounded_rect( + receiver: Id, + name: &CStr, + rect: CGRect, + x_radius: f64, + y_radius: f64, +) -> Option { + let selector = selector(name)?; + if !can_send(receiver, selector) { + return None; + } + let value = invoke!( + Id, + receiver, + selector, + CGRect => rect, + f64 => x_radius, + f64 => y_radius, + ); + (!value.is_null()).then_some(value) +} + +pub(super) fn send_id_color( + receiver: Id, + name: &CStr, + red: f64, + green: f64, + blue: f64, + alpha: f64, +) -> Option { + let selector = selector(name)?; + if !can_send(receiver, selector) { + return None; + } + let value = invoke!( + Id, + receiver, + selector, + f64 => red, + f64 => green, + f64 => blue, + f64 => alpha, + ); + (!value.is_null()).then_some(value) +} + +pub(super) fn send_id_f32(receiver: Id, name: &CStr, value: f32) -> Option { + let selector = selector(name)?; + if !can_send(receiver, selector) { + return None; + } + let result = invoke!(Id, receiver, selector, f32 => value); + (!result.is_null()).then_some(result) +} + +pub(super) fn send_id_objects( + receiver: Id, + name: &CStr, + objects: *const Id, + count: usize, +) -> Option { + let selector = selector(name)?; + if !can_send(receiver, selector) || (objects.is_null() && count != 0) { + return None; + } + let value = invoke!( + Id, + receiver, + selector, + *const Id => objects, + usize => count, + ); + (!value.is_null()).then_some(value) +} + +pub(super) fn send_void(receiver: Id, name: &CStr) -> bool { + let Some(selector) = selector(name) else { + return false; + }; + if !can_send(receiver, selector) { + return false; + } + invoke!((), receiver, selector); + true +} + +pub(super) fn send_void_id(receiver: Id, name: &CStr, value: Id) -> bool { + let Some(selector) = selector(name) else { + return false; + }; + if !can_send(receiver, selector) { + return false; + } + invoke!((), receiver, selector, Id => value); + true +} + +pub(super) fn send_void_two_ids(receiver: Id, name: &CStr, first: Id, second: Id) -> bool { + let Some(selector) = selector(name) else { + return false; + }; + if !can_send(receiver, selector) { + return false; + } + invoke!((), receiver, selector, Id => first, Id => second); + true +} + +pub(super) fn send_void_bool(receiver: Id, name: &CStr, value: bool) -> bool { + let Some(selector) = selector(name) else { + return false; + }; + if !can_send(receiver, selector) { + return false; + } + invoke!((), receiver, selector, i8 => i8::from(value)); + true +} + +pub(super) fn send_void_f64(receiver: Id, name: &CStr, value: f64) -> bool { + let Some(selector) = selector(name) else { + return false; + }; + if !can_send(receiver, selector) { + return false; + } + invoke!((), receiver, selector, f64 => value); + true +} + +pub(super) fn send_void_f32(receiver: Id, name: &CStr, value: f32) -> bool { + let Some(selector) = selector(name) else { + return false; + }; + if !can_send(receiver, selector) { + return false; + } + invoke!((), receiver, selector, f32 => value); + true +} + +pub(super) fn send_void_isize(receiver: Id, name: &CStr, value: isize) -> bool { + let Some(selector) = selector(name) else { + return false; + }; + if !can_send(receiver, selector) { + return false; + } + invoke!((), receiver, selector, isize => value); + true +} + +pub(super) fn send_void_usize(receiver: Id, name: &CStr, value: usize) -> bool { + let Some(selector) = selector(name) else { + return false; + }; + if !can_send(receiver, selector) { + return false; + } + invoke!((), receiver, selector, usize => value); + true +} + +pub(super) fn send_void_point(receiver: Id, name: &CStr, value: CGPoint) -> bool { + let Some(selector) = selector(name) else { + return false; + }; + if !can_send(receiver, selector) { + return false; + } + invoke!((), receiver, selector, CGPoint => value); + true +} + +pub(super) fn send_void_size(receiver: Id, name: &CStr, value: CGSize) -> bool { + let Some(selector) = selector(name) else { + return false; + }; + if !can_send(receiver, selector) { + return false; + } + invoke!((), receiver, selector, CGSize => value); + true +} + +pub(super) fn send_void_rect(receiver: Id, name: &CStr, value: CGRect) -> bool { + let Some(selector) = selector(name) else { + return false; + }; + if !can_send(receiver, selector) { + return false; + } + invoke!((), receiver, selector, CGRect => value); + true +} + +pub(super) fn send_void_rect_bool(receiver: Id, name: &CStr, rect: CGRect, value: bool) -> bool { + let Some(selector) = selector(name) else { + return false; + }; + if !can_send(receiver, selector) { + return false; + } + invoke!((), receiver, selector, CGRect => rect, i8 => i8::from(value)); + true +} + +pub(super) fn dispatch_main(context: *mut c_void, work: DispatchFn) -> bool { + let queue = dispatch_get_main_queue(); + if queue.is_null() { + return false; + } + // SAFETY: the caller owns context until work runs; libdispatch invokes work + // exactly once with that unchanged context on the main queue. + unsafe { dispatch_async_f(queue, context, work) }; + true +} + +// The C SDK defines dispatch_get_main_queue as this inline address operation. +fn dispatch_get_main_queue() -> Id { + (&raw const _dispatch_main_q).cast_mut().cast::() +} + +pub(super) fn display_frame_for_ax_point(point: (f64, f64)) -> Option { + if !point.0.is_finite() || !point.1.is_finite() { + return None; + } + let mut display = 0_u32; + let mut count = 0_u32; + // SAFETY: display and count are valid writable outputs for one display ID. + let error = unsafe { + CGGetDisplaysWithPoint(CGPoint::new(point.0, point.1), 1, &mut display, &mut count) + }; + // SAFETY: CGMainDisplayID takes no arguments and returns a display ID. + let main_display = unsafe { CGMainDisplayID() }; + if error != 0 || count == 0 { + display = main_display; + } + if display == 0 { + return None; + } + // SAFETY: display came from CoreGraphics and is valid for this bounds query. + let bounds = unsafe { CGDisplayBounds(display) }; + let ax = Frame { + x: bounds.origin.x, + y: bounds.origin.y, + w: bounds.size.width, + h: bounds.size.height, + }; + if main_display == 0 { + return None; + } + // SAFETY: main_display came from CoreGraphics and is valid for this bounds query. + let main_bounds = unsafe { CGDisplayBounds(main_display) }; + let appkit = Frame { + x: ax.x, + y: main_bounds.origin.y + main_bounds.size.height - (ax.y + ax.h), + w: ax.w, + h: ax.h, + }; + (ax.x.is_finite() + && ax.y.is_finite() + && ax.w.is_finite() + && ax.h.is_finite() + && appkit.x.is_finite() + && appkit.y.is_finite() + && appkit.w.is_finite() + && appkit.h.is_finite() + && ax.w > 0.0 + && ax.h > 0.0) + .then_some(DisplayGeometry { ax, appkit }) +} + +pub(super) fn status_window_level() -> isize { + const STATUS_WINDOW_LEVEL_KEY: i32 = 9; + // SAFETY: STATUS_WINDOW_LEVEL_KEY is a documented CGWindowLevelKey value. + unsafe { CGWindowLevelForKey(STATUS_WINDOW_LEVEL_KEY) as isize } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn libdispatch_main_queue_symbol_is_available() { + assert!(!dispatch_get_main_queue().is_null()); + } +} diff --git a/crates/computer-use-mcp/src/backend/macos/overlay/geometry.rs b/crates/computer-use-mcp/src/backend/macos/overlay/geometry.rs new file mode 100644 index 00000000..3e2aa5b9 --- /dev/null +++ b/crates/computer-use-mcp/src/backend/macos/overlay/geometry.rs @@ -0,0 +1,116 @@ +use crate::outline::Frame; + +pub(super) const BORDER_PADDING: f64 = 80.0; + +#[derive(Clone, Copy)] +pub(super) struct DisplayGeometry { + pub(super) ax: Frame, + pub(super) appkit: Frame, +} + +/// Converts an AX global point (y-down) into AppKit global coordinates +/// (y-up), using the display containing the point as the flip axis. +pub(super) fn ax_screen_to_appkit(point: (f64, f64), display: DisplayGeometry) -> (f64, f64) { + ( + display.appkit.x + (point.0 - display.ax.x), + display.appkit.y + display.ax.y + display.ax.h - point.1, + ) +} + +fn ax_frame_to_appkit(frame: Frame, display: DisplayGeometry) -> Frame { + let (left, bottom) = ax_screen_to_appkit((frame.x, frame.y + frame.h), display); + Frame { + x: left, + y: bottom, + w: frame.w, + h: frame.h, + } +} + +pub(super) fn border_frame(window_frame: Frame, display: DisplayGeometry) -> Frame { + let appkit = ax_frame_to_appkit(window_frame, display); + Frame { + x: appkit.x - BORDER_PADDING, + y: appkit.y - BORDER_PADDING, + w: appkit.w + BORDER_PADDING * 2.0, + h: appkit.h + BORDER_PADDING * 2.0, + } +} + +pub(super) fn is_finite_point(point: (f64, f64)) -> bool { + point.0.is_finite() && point.1.is_finite() +} + +pub(super) fn is_valid_frame(frame: Frame) -> bool { + frame.x.is_finite() + && frame.y.is_finite() + && frame.w.is_finite() + && frame.h.is_finite() + && (frame.x + frame.w).is_finite() + && (frame.y + frame.h).is_finite() + && (frame.w + BORDER_PADDING * 2.0).is_finite() + && (frame.h + BORDER_PADDING * 2.0).is_finite() + && frame.w > 0.0 + && frame.h > 0.0 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn flips_ax_geometry_and_expands_border_on_the_containing_display() { + let display = DisplayGeometry { + ax: Frame { + x: 0.0, + y: 0.0, + w: 1_440.0, + h: 900.0, + }, + appkit: Frame { + x: 0.0, + y: 0.0, + w: 1_440.0, + h: 900.0, + }, + }; + + assert_eq!(ax_screen_to_appkit((100.0, 250.0), display), (100.0, 650.0)); + assert_eq!( + border_frame( + Frame { + x: 100.0, + y: 200.0, + w: 500.0, + h: 400.0, + }, + display, + ), + Frame { + x: 20.0, + y: 220.0, + w: 660.0, + h: 560.0, + } + ); + + let offset_display = DisplayGeometry { + ax: Frame { + x: 1_440.0, + y: 100.0, + w: 1_920.0, + h: 1_080.0, + }, + appkit: Frame { + x: 1_440.0, + y: -280.0, + w: 1_920.0, + h: 1_080.0, + }, + }; + assert_eq!( + ax_screen_to_appkit((1_500.0, 200.0), offset_display), + (1_500.0, 700.0) + ); + } +} diff --git a/crates/computer-use-mcp/src/backend/macos/overlay/mod.rs b/crates/computer-use-mcp/src/backend/macos/overlay/mod.rs new file mode 100644 index 00000000..ae889733 --- /dev/null +++ b/crates/computer-use-mcp/src/backend/macos/overlay/mod.rs @@ -0,0 +1,210 @@ +#![allow(dead_code)] + +mod border; +mod cursor; +mod ffi; +mod geometry; + +use std::cell::RefCell; +use std::ffi::c_void; +use std::sync::atomic::{AtomicBool, Ordering}; + +use border::BorderUi; +use cursor::CursorUi; + +use self::ffi::{class, dispatch_main, send_id, send_void}; +use self::geometry::{is_finite_point, is_valid_frame}; +use crate::outline::Frame; + +static ENABLED: AtomicBool = AtomicBool::new(false); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum OverlayActionKind { + Click, + Scroll, + Drag, + Keyboard, + Move, +} + +pub(crate) fn set_enabled(on: bool) { + let was_enabled = ENABLED.swap(on, Ordering::AcqRel); + if was_enabled && !on { + enqueue(UiCommand::Clear); + } +} + +pub(crate) fn show_action( + kind: OverlayActionKind, + ax_screen_point: (f64, f64), + window_frame: Frame, +) { + if !ENABLED.load(Ordering::Acquire) || !is_finite_point(ax_screen_point) { + return; + } + enqueue(UiCommand::ShowAction { + kind, + point: ax_screen_point, + window_frame, + }); +} + +pub(crate) fn show_drag(from: (f64, f64), to: (f64, f64), window_frame: Frame) { + if !ENABLED.load(Ordering::Acquire) || !is_finite_point(from) || !is_finite_point(to) { + return; + } + enqueue(UiCommand::ShowDrag { + from, + to, + window_frame, + }); +} + +pub(crate) fn highlight_window(window_frame: Frame) { + if !ENABLED.load(Ordering::Acquire) || !is_valid_frame(window_frame) { + return; + } + enqueue(UiCommand::Highlight(window_frame)); +} + +pub(crate) fn clear() { + if ENABLED.load(Ordering::Acquire) { + enqueue(UiCommand::Clear); + } +} + +enum UiCommand { + ShowAction { + kind: OverlayActionKind, + point: (f64, f64), + window_frame: Frame, + }, + ShowDrag { + from: (f64, f64), + to: (f64, f64), + window_frame: Frame, + }, + Highlight(Frame), + Clear, +} + +struct OverlayState { + cursor: Option, + border: Option, +} + +impl OverlayState { + fn show_action(&mut self, kind: OverlayActionKind, point: (f64, f64), window_frame: Frame) { + if is_valid_frame(window_frame) { + self.highlight(window_frame); + } + let Some(display) = ffi::display_frame_for_ax_point(point) else { + return; + }; + if self.cursor.is_none() { + self.cursor = CursorUi::new(); + } + if let Some(cursor) = self.cursor.as_mut() { + cursor.show(kind, point, display); + } + } + + fn show_drag(&mut self, from: (f64, f64), to: (f64, f64), window_frame: Frame) { + if is_valid_frame(window_frame) { + self.highlight(window_frame); + } + let Some(from_display) = ffi::display_frame_for_ax_point(from) else { + return; + }; + let to_display = ffi::display_frame_for_ax_point(to).unwrap_or(from_display); + if self.cursor.is_none() { + self.cursor = CursorUi::new(); + } + if let Some(cursor) = self.cursor.as_mut() { + cursor.show_drag(from, to, from_display, to_display); + } + } + + fn highlight(&mut self, window_frame: Frame) { + let Some(display) = ffi::display_frame_for_ax_point(window_frame.center()) else { + return; + }; + if self.border.is_none() { + self.border = BorderUi::new(); + } + if let Some(border) = self.border.as_mut() { + border.show(window_frame, display); + } + } + + fn clear(&mut self) { + if let Some(cursor) = self.cursor.as_mut() { + cursor.hide(); + } + if let Some(border) = self.border.as_mut() { + border.hide(); + } + } +} + +// This thread-local is intentionally read only by `run_command`, which is +// exclusively submitted to the process main queue. Objective-C window and +// layer pointers therefore never cross back into background-thread UI code. +thread_local! { + static MAIN_STATE: RefCell = const { + RefCell::new(OverlayState { + cursor: None, + border: None, + }) + }; +} + +fn enqueue(command: UiCommand) { + let context = Box::into_raw(Box::new(command)).cast::(); + if !dispatch_main(context, run_command) { + // SAFETY: context came from Box::into_raw above and dispatch rejected it, + // so no callback can own or free it. + drop(unsafe { Box::from_raw(context.cast::()) }); + } +} + +// SAFETY: libdispatch calls this only with the Box created by enqueue. +unsafe extern "C" fn run_command(context: *mut c_void) { + if context.is_null() { + return; + } + // SAFETY: enqueue passes exactly one Box to a callback that + // libdispatch invokes exactly once. + let command = unsafe { Box::from_raw(context.cast::()) }; + let should_run = matches!(*command, UiCommand::Clear) || ENABLED.load(Ordering::Acquire); + if !should_run { + return; + } + + let pool = class(c"NSAutoreleasePool").and_then(|pool| send_id(pool, c"new")); + // sharedApplication initializes AppKit if needed; it does not activate the app. + let _ = + class(c"NSApplication").and_then(|application| send_id(application, c"sharedApplication")); + let _ = MAIN_STATE.try_with(|state| { + let Ok(mut state) = state.try_borrow_mut() else { + return; + }; + match *command { + UiCommand::ShowAction { + kind, + point, + window_frame, + } => state.show_action(kind, point, window_frame), + UiCommand::ShowDrag { + from, + to, + window_frame, + } => state.show_drag(from, to, window_frame), + UiCommand::Highlight(window_frame) => state.highlight(window_frame), + UiCommand::Clear => state.clear(), + } + }); + if let Some(pool) = pool { + let _ = send_void(pool, c"drain"); + } +} From dca30384324469bb7cdff701f74983b368e28d8a Mon Sep 17 00:00:00 2001 From: Tryanks Date: Tue, 1 Sep 2026 03:54:43 +0800 Subject: [PATCH 3/4] feat(computer-use): per-pid scheduling lanes + harness annotations --- crates/computer-use-mcp/src/state.rs | 451 ++++++++++++++++++++++++++- crates/computer-use-mcp/src/tools.rs | 82 ++++- 2 files changed, 520 insertions(+), 13 deletions(-) diff --git a/crates/computer-use-mcp/src/state.rs b/crates/computer-use-mcp/src/state.rs index a1629854..268a6431 100644 --- a/crates/computer-use-mcp/src/state.rs +++ b/crates/computer-use-mcp/src/state.rs @@ -1,18 +1,28 @@ //! Bounded immutable observations and independently bounded output pages. -use std::collections::{HashMap, VecDeque}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::fmt; use std::sync::{Arc, Mutex, OnceLock}; use crate::backend::RootInfo; use crate::outline::{ - MAX_MODEL_LINES, PAGE_BYTES, PREVIEW_BYTES, UiNode, assign_refs, assign_refs_from_previous, - output_exceeds_limit, safe_prefix, + MAX_MODEL_BYTES, MAX_MODEL_LINES, PAGE_BYTES, PREVIEW_BYTES, UiNode, assign_refs, + assign_refs_from_previous, canonical_role, output_exceeds_limit, safe_prefix, }; pub const OBSERVATION_CAPACITY: usize = 8; pub const OUTPUT_CAPACITY: usize = 32; +const RECENT_ACTION_CAPACITY: usize = 8; +const CANDIDATE_TARGET_CAPACITY: usize = 64; +const DELTA_ENTRY_CAPACITY: usize = 16; +const STABLE_LABEL_CAPACITY: usize = 3_000; +const STABLE_LABEL_MAX_BYTES: usize = 512; +const DISPLAY_LABEL_MAX_CHARS: usize = 160; +const DISPLAY_LABEL_MAX_BYTES: usize = 200; +const ROLE_MAX_BYTES: usize = 48; +const ACTION_DESCRIPTION_MAX_BYTES: usize = 256; + #[derive(Debug)] pub struct Observation { pub state_id: String, @@ -20,6 +30,21 @@ pub struct Observation { pub root_epoch: u64, pub tree: UiNode, pub screenshot_png: Option>, + pub harness_annotation: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct StableLabel { + key: String, + display: String, +} + +#[derive(Debug, Default)] +struct HarnessHistory { + observation_sequence: u64, + initial_labels: Option>, + previous_labels: Option>, + recent_actions: VecDeque, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -109,6 +134,7 @@ pub struct StateStore { observations: HashMap>, observation_lru: VecDeque, root_epochs: HashMap, + harness_histories: HashMap, outputs: HashMap, output_lru: VecDeque, } @@ -123,6 +149,7 @@ impl Default for StateStore { observations: HashMap::new(), observation_lru: VecDeque::new(), root_epochs: HashMap::new(), + harness_histories: HashMap::new(), outputs: HashMap::new(), output_lru: VecDeque::new(), } @@ -150,7 +177,9 @@ impl StateStore { assign_refs(&mut tree); } - let epoch = self.root_epochs.entry(identity).or_default(); + let harness_annotation = self.record_harness_observation(&root, &tree); + + let epoch = self.root_epochs.entry(identity.clone()).or_default(); *epoch += 1; let state_id = format!("S{}", self.next_state); self.next_state += 1; @@ -160,18 +189,68 @@ impl StateStore { root_epoch: *epoch, tree, screenshot_png, + harness_annotation, }); self.observations .insert(state_id.clone(), Arc::clone(&observation)); touch(&mut self.observation_lru, &state_id); while self.observations.len() > self.observation_capacity { - if let Some(evicted) = self.observation_lru.pop_front() { - self.observations.remove(&evicted); + if let Some(evicted) = self.observation_lru.pop_front() + && let Some(observation) = self.observations.remove(&evicted) + { + self.drop_harness_history_if_root_evicted(&observation.root.identity()); } } observation } + pub fn record_actions( + &mut self, + root: &RootInfo, + descriptions: impl IntoIterator, + ) { + let history = self.harness_histories.entry(root.identity()).or_default(); + for description in descriptions { + history + .recent_actions + .push_back(truncate_plain(&description, ACTION_DESCRIPTION_MAX_BYTES)); + while history.recent_actions.len() > RECENT_ACTION_CAPACITY { + history.recent_actions.pop_front(); + } + } + } + + fn record_harness_observation(&mut self, root: &RootInfo, tree: &UiNode) -> String { + let current_labels = stable_labels(tree); + let candidate_targets = candidate_target_lines(tree); + let history = self.harness_histories.entry(root.identity()).or_default(); + history.observation_sequence = history.observation_sequence.saturating_add(1); + let initial_labels = history + .initial_labels + .get_or_insert_with(|| current_labels.clone()); + let annotation = harness_annotation( + root, + history.observation_sequence, + initial_labels, + history.previous_labels.as_deref(), + &history.recent_actions, + &candidate_targets, + ¤t_labels, + ); + history.previous_labels = Some(current_labels); + annotation + } + + fn drop_harness_history_if_root_evicted(&mut self, identity: &str) { + let root_remains = self + .observations + .values() + .any(|observation| observation.root.identity() == identity); + if !root_remains { + self.harness_histories.remove(identity); + } + } + pub fn get(&mut self, state_id: &str) -> Result, StateError> { let observation = self .observations @@ -291,6 +370,241 @@ impl StateStore { } } +pub(crate) fn harness_action_description( + action_name: &str, + target_ref: Option<&str>, + tree: &UiNode, +) -> String { + let mut description = action_name.to_string(); + if let Some(ref_id) = target_ref { + description.push(' '); + description.push_str(&truncate_plain(ref_id, 32)); + if let Some(node) = tree.find(ref_id) + && let Some(label) = candidate_label(node) + { + description.push_str(" \""); + description.push_str(&display_label(&label)); + description.push('"'); + } + } + truncate_plain(&description, ACTION_DESCRIPTION_MAX_BYTES) +} + +fn harness_annotation( + root: &RootInfo, + observation_sequence: u64, + initial_labels: &[StableLabel], + previous_labels: Option<&[StableLabel]>, + recent_actions: &VecDeque, + candidate_targets: &[String], + current_labels: &[StableLabel], +) -> String { + let mut lines = vec![ + "".to_string(), + format!("observation_sequence: {observation_sequence}"), + format!("root: pid={} window_id={}", root.pid, root.window_id), + "".to_string(), + ]; + if recent_actions.is_empty() { + lines.push("none".to_string()); + } else { + lines.extend(recent_actions.iter().map(|action| format!("- {action}"))); + } + lines.push("".to_string()); + lines.push("".to_string()); + if candidate_targets.is_empty() { + lines.push("none".to_string()); + } else { + lines.extend(candidate_targets.iter().cloned()); + } + lines.push("".to_string()); + lines.push("".to_string()); + lines.extend(harness_delta_lines(current_labels, previous_labels)); + lines.push("".to_string()); + lines.push("".to_string()); + lines.extend(harness_delta_lines(current_labels, Some(initial_labels))); + lines.push("".to_string()); + lines.push("".to_string()); + let annotation = lines.join("\n"); + debug_assert!(annotation.len() <= MAX_MODEL_BYTES); + annotation +} + +fn stable_labels(root: &UiNode) -> Vec { + fn visit(node: &UiNode, seen: &mut HashSet, labels: &mut Vec) { + if labels.len() >= STABLE_LABEL_CAPACITY { + return; + } + if let Some(display) = stable_label(node) { + let key = truncate_plain(&display.to_lowercase(), STABLE_LABEL_MAX_BYTES); + if seen.insert(key.clone()) { + labels.push(StableLabel { key, display }); + } + } + for child in &node.children { + visit(child, seen, labels); + if labels.len() >= STABLE_LABEL_CAPACITY { + break; + } + } + } + + let mut seen = HashSet::new(); + let mut labels = Vec::new(); + visit(root, &mut seen, &mut labels); + labels +} + +fn stable_label(node: &UiNode) -> Option { + choose_label(&node.title, &node.description, None) +} + +fn candidate_label(node: &UiNode) -> Option { + choose_label(&node.title, &node.description, Some(&node.value)) +} + +fn choose_label(title: &str, description: &str, value: Option<&str>) -> Option { + let title = normalize_label(title); + let description = normalize_label(description); + let value = value.map(normalize_label).unwrap_or_default(); + let description_is_richer = !description.is_empty() + && (title.is_empty() + || description.chars().count() > title.chars().count().saturating_add(8) + || description.to_lowercase().contains(&title.to_lowercase())); + let selected = if description_is_richer { + description + } else if !title.is_empty() { + title + } else if !description.is_empty() { + description + } else { + value + }; + (!selected.is_empty()).then_some(selected) +} + +fn normalize_label(value: &str) -> String { + let normalized = value.split_whitespace().collect::>().join(" "); + truncate_plain(&normalized, STABLE_LABEL_MAX_BYTES) +} + +fn candidate_target_lines(root: &UiNode) -> Vec { + fn visit(node: &UiNode, lines: &mut Vec) { + if lines.len() >= CANDIDATE_TARGET_CAPACITY { + return; + } + if node.is_interactive() { + let ref_id = truncate_plain(&node.ref_id, 32); + let role = truncate_plain(&canonical_role(&node.role), ROLE_MAX_BYTES); + let label = candidate_label(node) + .map(|label| display_label(&label)) + .unwrap_or_default(); + lines.push(format!("- {ref_id} {role} \"{label}\"")); + } + for child in &node.children { + visit(child, lines); + if lines.len() >= CANDIDATE_TARGET_CAPACITY { + break; + } + } + } + + let mut lines = Vec::new(); + visit(root, &mut lines); + lines +} + +fn harness_delta_lines(current: &[StableLabel], baseline: Option<&[StableLabel]>) -> Vec { + let Some(baseline) = baseline else { + return vec!["initial observation for this root.".to_string()]; + }; + let current_keys: HashSet<_> = current.iter().map(|label| label.key.as_str()).collect(); + let baseline_keys: HashSet<_> = baseline.iter().map(|label| label.key.as_str()).collect(); + let added: Vec<_> = current + .iter() + .filter(|label| !baseline_keys.contains(label.key.as_str())) + .collect(); + let removed: Vec<_> = baseline + .iter() + .filter(|label| !current_keys.contains(label.key.as_str())) + .collect(); + if added.is_empty() && removed.is_empty() { + return vec!["no stable label changes.".to_string()]; + } + + let mut lines = Vec::new(); + lines.extend( + added + .iter() + .take(DELTA_ENTRY_CAPACITY) + .map(|label| format!("+ \"{}\"", display_label(&label.display))), + ); + if added.len() > DELTA_ENTRY_CAPACITY { + lines.push(format!("+ ... {} more", added.len() - DELTA_ENTRY_CAPACITY)); + } + lines.extend( + removed + .iter() + .take(DELTA_ENTRY_CAPACITY) + .map(|label| format!("- \"{}\"", display_label(&label.display))), + ); + if removed.len() > DELTA_ENTRY_CAPACITY { + lines.push(format!( + "- ... {} more", + removed.len() - DELTA_ENTRY_CAPACITY + )); + } + lines +} + +fn display_label(value: &str) -> String { + let mut output = String::new(); + let mut chars = value.chars().peekable(); + let mut truncated = false; + for (count, ch) in chars.by_ref().enumerate() { + if count >= DISPLAY_LABEL_MAX_CHARS { + truncated = true; + break; + } + let escaped = match ch { + '\\' => "\\\\".to_string(), + '"' => "\\\"".to_string(), + other if other.is_control() => " ".to_string(), + other => other.to_string(), + }; + if output.len() + escaped.len() > DISPLAY_LABEL_MAX_BYTES.saturating_sub('…'.len_utf8()) { + truncated = true; + break; + } + output.push_str(&escaped); + } + if chars.peek().is_some() { + truncated = true; + } + if truncated { + output.push('…'); + } + output +} + +fn truncate_plain(value: &str, max_bytes: usize) -> String { + if value.len() <= max_bytes { + return value.to_string(); + } + let ellipsis = '…'; + let mut end = max_bytes + .saturating_sub(ellipsis.len_utf8()) + .min(value.len()); + while !value.is_char_boundary(end) { + end = end.saturating_sub(1); + } + let mut output = value[..end].to_string(); + if max_bytes >= ellipsis.len_utf8() { + output.push(ellipsis); + } + output +} + fn page( output_ref: &str, owner_state: Option, @@ -373,6 +687,16 @@ mod tests { assert!(matches!(store.get("S2"), Err(StateError::Evicted(_)))); } + #[test] + fn evicting_the_last_observation_for_a_root_drops_its_harness_history() { + let mut store = StateStore::default(); + for window_id in 1..=OBSERVATION_CAPACITY as u32 + 1 { + store.insert_observation(root(window_id), tree(&format!("Window {window_id}")), None); + } + assert!(!store.harness_histories.contains_key(&root(1).identity())); + assert_eq!(store.harness_histories.len(), OBSERVATION_CAPACITY); + } + #[test] fn newer_root_epoch_rejects_actions_from_old_state() { let mut store = StateStore::default(); @@ -421,4 +745,119 @@ mod tests { .unwrap(); assert_eq!(repeated, repeated_again); } + + #[test] + fn harness_delta_reports_added_removed_and_unchanged_labels() { + let entry = |display: &str| StableLabel { + key: display.to_lowercase(), + display: display.to_string(), + }; + let baseline = vec![entry("Kept"), entry("Removed")]; + let current = vec![entry("Kept"), entry("Added")]; + let changed = harness_delta_lines(¤t, Some(&baseline)); + assert!(changed.iter().any(|line| line == "+ \"Added\"")); + assert!(changed.iter().any(|line| line == "- \"Removed\"")); + + let unchanged = harness_delta_lines(¤t, Some(¤t)); + assert_eq!(unchanged, ["no stable label changes."]); + + let initial = harness_delta_lines(¤t, None); + assert_eq!(initial, ["initial observation for this root."]); + } + + #[test] + fn recent_actions_keep_only_the_latest_bounded_window_in_order() { + let mut store = StateStore::default(); + let root = root(1); + store.insert_observation(root.clone(), tree("one"), None); + let total = RECENT_ACTION_CAPACITY + 3; + store.record_actions(&root, (0..total).map(|index| format!("press @e{index}"))); + + let recent = &store + .harness_histories + .get(&root.identity()) + .unwrap() + .recent_actions; + assert_eq!(recent.len(), RECENT_ACTION_CAPACITY); + assert_eq!( + recent.front().unwrap(), + &format!("press @e{}", total - RECENT_ACTION_CAPACITY) + ); + assert_eq!(recent.back().unwrap(), &format!("press @e{}", total - 1)); + } + + #[test] + fn candidate_targets_include_only_interactive_nodes_with_assigned_refs() { + let mut tree = UiNode { + role: "window".into(), + title: "Test".into(), + enabled: true, + children: vec![ + UiNode { + role: "button".into(), + title: "Save".into(), + enabled: true, + ..UiNode::default() + }, + UiNode { + role: "group".into(), + description: "Action group".into(), + actions: vec!["press".into()], + enabled: true, + ..UiNode::default() + }, + UiNode { + role: "static_text".into(), + title: "Read only".into(), + enabled: true, + ..UiNode::default() + }, + ], + ..UiNode::default() + }; + assign_refs(&mut tree); + let button_ref = tree.children[0].ref_id.clone(); + let action_ref = tree.children[1].ref_id.clone(); + let read_only_ref = tree.children[2].ref_id.clone(); + + let candidates = candidate_target_lines(&tree); + assert_eq!(candidates.len(), 2); + assert!( + candidates + .iter() + .any(|line| line == &format!("- {button_ref} button \"Save\"")) + ); + assert!( + candidates + .iter() + .any(|line| { line == &format!("- {action_ref} group \"Action group\"") }) + ); + assert!(candidates.iter().all(|line| !line.contains(&read_only_ref))); + } + + #[test] + fn observation_sequence_increments_independently_per_root() { + let mut store = StateStore::default(); + let first_root = root(1); + let other_root = root(2); + let first = store.insert_observation(first_root.clone(), tree("one"), None); + let second = store.insert_observation(first_root.clone(), tree("two"), None); + let other = store.insert_observation(other_root, tree("other"), None); + + assert!(first.harness_annotation.contains("observation_sequence: 1")); + assert!( + second + .harness_annotation + .contains("observation_sequence: 2") + ); + assert!(other.harness_annotation.contains("observation_sequence: 1")); + assert_eq!( + store + .harness_histories + .get(&first_root.identity()) + .unwrap() + .observation_sequence, + 2 + ); + } } diff --git a/crates/computer-use-mcp/src/tools.rs b/crates/computer-use-mcp/src/tools.rs index 9eece001..c17c766d 100644 --- a/crates/computer-use-mcp/src/tools.rs +++ b/crates/computer-use-mcp/src/tools.rs @@ -302,7 +302,7 @@ pub fn service() -> Service { mod dispatch { use super::*; use std::collections::HashMap; - use std::sync::{Mutex, OnceLock}; + use std::sync::{Arc, Mutex, OnceLock}; use std::time::{Duration, Instant}; use base64::Engine as _; @@ -356,14 +356,34 @@ mod dispatch { } static ROOTS: OnceLock> = OnceLock::new(); - static OBSERVATION_TRANSACTION: OnceLock> = OnceLock::new(); + + #[derive(Default)] + struct ObservationLanes { + by_pid: Mutex>>>, + } + + impl ObservationLanes { + fn lane(&self, pid: u32) -> Arc> { + Arc::clone( + self.by_pid + .lock() + .unwrap() + .entry(pid) + .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))), + ) + } + } + + static OBSERVATION_LANES: OnceLock = OnceLock::new(); fn roots() -> &'static Mutex { ROOTS.get_or_init(|| Mutex::new(RootRegistry::default())) } - fn observation_transaction() -> &'static tokio::sync::Mutex<()> { - OBSERVATION_TRANSACTION.get_or_init(|| tokio::sync::Mutex::new(())) + fn observation_lane(pid: u32) -> Arc> { + OBSERVATION_LANES + .get_or_init(ObservationLanes::default) + .lane(pid) } pub(super) async fn find_roots(params: FindRootsParams) -> CallToolResult { @@ -419,11 +439,12 @@ mod dispatch { { return result; } - let _transaction = observation_transaction().lock().await; let root = match resolve_root(params.root.as_deref()) { Ok(root) => root, Err(result) => return *result, }; + let lane = observation_lane(root.pid); + let _transaction = lane.lock().await; let capture = capture_policy(config.image_mode, params.mode, &permissions); let request = ObserveRequest { semantic: !matches!(params.mode, Some(ObserveMode::Visual)), @@ -529,7 +550,12 @@ mod dispatch { if params.actions.is_empty() { return tool_error("act_ui requires at least one action"); } - let _transaction = observation_transaction().lock().await; + let pid = match crate::state::global().lock().unwrap().get(¶ms.state_id) { + Ok(observation) => observation.root.pid, + Err(error) => return tool_error(&error.to_string()), + }; + let lane = observation_lane(pid); + let _transaction = lane.lock().await; let previous = match crate::state::global() .lock() .unwrap() @@ -539,6 +565,7 @@ mod dispatch { Err(error) => return tool_error(&error.to_string()), }; let mut step_results = Vec::new(); + let mut action_descriptions = Vec::new(); let mut stopped_at = None; let mut activation = "none"; for (index, action) in params.actions.iter().enumerate() { @@ -548,6 +575,11 @@ mod dispatch { Err(error) => ActionResult::didnt(error, Delivery::None), }; let didnt = result.outcome == ActionOutcome::Didnt; + action_descriptions.push(crate::state::harness_action_description( + action_name(action.action), + action.r#ref.as_deref(), + &previous.tree, + )); activation = match (activation, result.delivery) { (_, Delivery::ForegroundHid) => "foreground", ("none", Delivery::BackgroundPid) => "background", @@ -565,6 +597,10 @@ mod dispatch { break; } } + crate::state::global() + .lock() + .unwrap() + .record_actions(&previous.root, action_descriptions); let expectation_preexisting = params .expect @@ -619,6 +655,8 @@ mod dispatch { } else { text.push_str(&diff.text); } + text.push_str("\n\n"); + text.push_str(&successor.harness_annotation); bounded_success(Some(&successor.state_id), text, Vec::new()) } @@ -684,7 +722,12 @@ mod dispatch { if let Some(result) = permission_gate(permissions, true, false) { return result; } - let _transaction = observation_transaction().lock().await; + let pid = match crate::state::global().lock().unwrap().get(¶ms.state_id) { + Ok(observation) => observation.root.pid, + Err(error) => return tool_error(&error.to_string()), + }; + let lane = observation_lane(pid); + let _transaction = lane.lock().await; let previous = match crate::state::global() .lock() .unwrap() @@ -890,6 +933,8 @@ mod dispatch { } text.push('\n'); text.push_str(&outline::render_folded(&observation.tree)); + text.push_str("\n\n"); + text.push_str(&observation.harness_annotation); let extra = screenshot_for_response .map(|screenshot| { ContentBlock::image( @@ -1093,6 +1138,29 @@ mod dispatch { CallToolResult::error(vec![ContentBlock::text(message)]) } + #[cfg(test)] + mod scheduling_tests { + use super::*; + + #[test] + fn per_pid_lanes_allow_other_pids_and_serialize_the_same_pid() { + let lanes = ObservationLanes::default(); + let first_pid = lanes.lane(1001); + let same_pid = lanes.lane(1001); + let other_pid = lanes.lane(2002); + assert!(Arc::ptr_eq(&first_pid, &same_pid)); + assert!(!Arc::ptr_eq(&first_pid, &other_pid)); + + let first_guard = first_pid.try_lock().unwrap(); + let other_guard = other_pid.try_lock().unwrap(); + assert!(same_pid.try_lock().is_err()); + + drop(first_guard); + let same_pid_guard = same_pid.try_lock().unwrap(); + drop((same_pid_guard, other_guard)); + } + } + #[cfg(all(test, target_os = "macos"))] mod tests { use super::*; From 33ffbb013bb6e718defb71207fca0817c03d8aaa Mon Sep 17 00:00:00 2001 From: Tryanks Date: Tue, 1 Sep 2026 04:02:13 +0800 Subject: [PATCH 4/4] feat(settings): expose agent-cursor and foreground-fallback toggles (Computer Use) --- crates/core/src/settings.rs | 8 ++++++++ crates/protocol/src/tests.rs | 2 ++ crates/ui/src/settings_page.rs | 37 ++++++++++++++++++++++++++++++++++ crates/ui/src/store/intents.rs | 6 ++++++ locales/en.yml | 6 ++++++ locales/zh-CN.yml | 6 ++++++ 6 files changed, 65 insertions(+) diff --git a/crates/core/src/settings.rs b/crates/core/src/settings.rs index 9bb8ea35..6aa1d202 100644 --- a/crates/core/src/settings.rs +++ b/crates/core/src/settings.rs @@ -635,6 +635,8 @@ pub enum SettingsPatch { ComputerUseEnabled(bool), ComputerUseImageMode(ImageMode), ComputerUseAllowInput(bool), + ComputerUseAllowForegroundFallback(bool), + ComputerUseShowAgentCursor(bool), BrowserEnabled(bool), BrowserHomeUrl(Option), BrowserAllowEvaluate(bool), @@ -894,6 +896,12 @@ impl Settings { SettingsPatch::ComputerUseEnabled(value) => self.computer_use.enabled = value, SettingsPatch::ComputerUseImageMode(value) => self.computer_use.image_mode = value, SettingsPatch::ComputerUseAllowInput(value) => self.computer_use.allow_input = value, + SettingsPatch::ComputerUseAllowForegroundFallback(value) => { + self.computer_use.allow_foreground_fallback = value + } + SettingsPatch::ComputerUseShowAgentCursor(value) => { + self.computer_use.show_agent_cursor = value + } SettingsPatch::BrowserEnabled(value) => self.browser.enabled = value, SettingsPatch::BrowserHomeUrl(value) => self.browser.home_url = value, SettingsPatch::BrowserAllowEvaluate(value) => self.browser.allow_evaluate = value, diff --git a/crates/protocol/src/tests.rs b/crates/protocol/src/tests.rs index 1c351f3b..56a44888 100644 --- a/crates/protocol/src/tests.rs +++ b/crates/protocol/src/tests.rs @@ -255,6 +255,8 @@ fn settings_patches_round_trip() { SettingsPatch::ComputerUseEnabled(true), SettingsPatch::ComputerUseImageMode(ImageMode::Always), SettingsPatch::ComputerUseAllowInput(false), + SettingsPatch::ComputerUseAllowForegroundFallback(true), + SettingsPatch::ComputerUseShowAgentCursor(false), SettingsPatch::BrowserEnabled(false), SettingsPatch::BrowserHomeUrl(Some("https://example.com".into())), SettingsPatch::BrowserAllowEvaluate(false), diff --git a/crates/ui/src/settings_page.rs b/crates/ui/src/settings_page.rs index d345fe31..7be4959e 100644 --- a/crates/ui/src/settings_page.rs +++ b/crates/ui/src/settings_page.rs @@ -1366,6 +1366,25 @@ impl SettingsPage { this.dispatch_settings(|store| store.set_computer_use_allow_input(true), cx) }, ); + let allow_foreground_fallback_reset = self.reset_action( + "reset-cu-allow-foreground-fallback", + settings.computer_use.allow_foreground_fallback, + cx, + |this, _, cx| { + this.dispatch_settings( + |store| store.set_computer_use_allow_foreground_fallback(false), + cx, + ) + }, + ); + let show_agent_cursor_reset = self.reset_action( + "reset-cu-show-agent-cursor", + !settings.computer_use.show_agent_cursor, + cx, + |this, _, cx| { + this.dispatch_settings(|store| store.set_computer_use_show_agent_cursor(true), cx) + }, + ); let rows = vec![ self.toggle_row( "cu-enabled", @@ -1386,6 +1405,24 @@ impl SettingsPage { cx, WorkspaceStore::set_computer_use_allow_input, ), + self.toggle_row( + "cu-allow-foreground-fallback", + crate::tr!("computer_use.allow_foreground_fallback.title"), + crate::tr!("computer_use.allow_foreground_fallback.description"), + settings.computer_use.allow_foreground_fallback, + allow_foreground_fallback_reset, + cx, + WorkspaceStore::set_computer_use_allow_foreground_fallback, + ), + self.toggle_row( + "cu-show-agent-cursor", + crate::tr!("computer_use.show_agent_cursor.title"), + crate::tr!("computer_use.show_agent_cursor.description"), + settings.computer_use.show_agent_cursor, + show_agent_cursor_reset, + cx, + WorkspaceStore::set_computer_use_show_agent_cursor, + ), ]; v_flex() .gap(px(24.)) diff --git a/crates/ui/src/store/intents.rs b/crates/ui/src/store/intents.rs index 378ebeb5..709c9bda 100644 --- a/crates/ui/src/store/intents.rs +++ b/crates/ui/src/store/intents.rs @@ -116,6 +116,12 @@ impl WorkspaceStore { pub fn set_computer_use_allow_input(&mut self, value: bool) { self.patch_settings(SettingsPatch::ComputerUseAllowInput(value)); } + pub fn set_computer_use_allow_foreground_fallback(&mut self, value: bool) { + self.patch_settings(SettingsPatch::ComputerUseAllowForegroundFallback(value)); + } + pub fn set_computer_use_show_agent_cursor(&mut self, value: bool) { + self.patch_settings(SettingsPatch::ComputerUseShowAgentCursor(value)); + } pub fn set_browser_enabled(&mut self, value: bool) { self.patch_settings(SettingsPatch::BrowserEnabled(value)); } diff --git a/locales/en.yml b/locales/en.yml index 4692292c..ff0fc072 100644 --- a/locales/en.yml +++ b/locales/en.yml @@ -267,6 +267,12 @@ computer_use: allow_input: title: "Allow input actions" description: "When off, the tools are observe-only: clicks, typing, and other actions are rejected." + allow_foreground_fallback: + title: "Foreground keyboard fallback" + description: "If background delivery can't start, retry keyboard input by briefly focusing the target window. Off by default; pointer actions never fall back to the foreground." + show_agent_cursor: + title: "Show agent cursor" + description: "Display a floating cursor and window highlight showing where the agent is acting. The real pointer is never moved." permissions_section: "SYSTEM PERMISSIONS" unsupported: "Computer use is only available on macOS." browser: diff --git a/locales/zh-CN.yml b/locales/zh-CN.yml index 3fe6f7ed..634d012b 100644 --- a/locales/zh-CN.yml +++ b/locales/zh-CN.yml @@ -267,6 +267,12 @@ computer_use: allow_input: title: "允许输入操作" description: "关闭后工具仅用于观察：点击、输入等操作都会被拒绝。" + allow_foreground_fallback: + title: "前台键盘回退" + description: "当后台投递无法启动时，通过短暂聚焦目标窗口重试键盘输入。默认关闭；指针操作绝不回退到前台。" + show_agent_cursor: + title: "显示 agent 光标" + description: "显示一个浮动光标与窗口高亮，指示 agent 正在操作的位置。真实指针不会移动。" permissions_section: "系统权限" unsupported: "电脑操作仅在 macOS 上可用。" browser: