feat(computer-use): raw-image fallback for text-sparse AX trees

crates/computer-use-mcp/src/backend.rs

@@ -70,6 +70,16 @@ pub enum CapturePolicy {
  IfSparse,
 }
 
+impl CapturePolicy {
+ pub fn should_capture(self, text_sparse: bool) -> bool {
+ match self {
+ Self::Never => false,
+ Self::Always => true,
+ Self::IfSparse => text_sparse,
+ }
+ }
+}
+
 #[derive(Debug, Clone, Copy)]
 pub struct ObserveRequest {
  pub semantic: bool,
@@ -80,6 +90,7 @@ pub struct ObserveRequest {
 pub struct RootObservation {
  pub root: RootInfo,
  pub tree: UiNode,
+ pub text_sparse: bool,
  pub screenshot_png: Option<Vec<u8>>,
 }
 

crates/computer-use-mcp/src/backend/macos/mod.rs

@@ -17,10 +17,10 @@ use core_graphics::window::{
 };
 
 use super::{
- ActionKind, ActionRequest, ActionResult, BackendError, BackendErrorCode, CapturePolicy,
-ObserveRequest, RootFilters, RootInfo, RootObservation,
+ ActionKind, ActionRequest, ActionResult, BackendError, BackendErrorCode, ObserveRequest,
+ RootFilters, RootInfo, RootObservation,
 };
-use crate::outline::{UiNode, interactive_count};
+use crate::outline::{UiNode, is_text_sparse};
 
 pub(super) struct MacosBackend;
 
@@ -108,17 +108,15 @@ impl MacosBackend {
  ..UiNode::default()
  }
  };
- let should_capture = match request.capture {
- CapturePolicy::Never => false,
- CapturePolicy::Always => true,
- CapturePolicy::IfSparse => interactive_count(&tree) <= 3,
- };
+ let text_sparse = is_text_sparse(&tree);
+ let should_capture = request.capture.should_capture(text_sparse);
  let screenshot_png = should_capture
  .then(|| capture::capture_window(root))
  .transpose()?;
  Ok(RootObservation {
  root: root.clone(),
  tree,
+ text_sparse,
  screenshot_png,
  })
  }

crates/computer-use-mcp/src/outline.rs

@@ -9,6 +9,8 @@ pub const MAX_MODEL_LINES: usize = 2_000;
 pub const PREVIEW_BYTES: usize = 16 * 1024;
 pub const PAGE_BYTES: usize = 16 * 1024;
 pub const SEARCH_LIMIT: usize = 20;
+pub const TEXT_SPARSE_DESCENDANT_THRESHOLD: usize = 3;
+pub const TEXT_SPARSE_MIN_FRAME_AREA: f64 = 20_000.0;
 
 const FOLDED_DEPTH: usize = 7;
 const FOLDED_LINES: usize = 500;
@@ -125,6 +127,30 @@ pub fn interactive_count(root: &UiNode) -> usize {
  count
 }
 
+/// A large accessibility tree is text-sparse when fewer than three of its
+/// descendants expose a title, value, or description. The root window's own
+/// title is deliberately excluded because it does not describe the contents.
+pub fn is_text_sparse(root: &UiNode) -> bool {
+ let area = root.frame.w * root.frame.h;
+ root.frame.has_area()
+ && area.is_finite()
+ && area >= TEXT_SPARSE_MIN_FRAME_AREA
+ && text_bearing_descendant_count(root) < TEXT_SPARSE_DESCENDANT_THRESHOLD
+}
+
+fn text_bearing_descendant_count(root: &UiNode) -> usize {
+ root.children
+ .iter()
+ .map(|child| usize::from(node_has_text(child)) + text_bearing_descendant_count(child))
+ .sum()
+}
+
+fn node_has_text(node: &UiNode) -> bool {
+ [&node.title, &node.value, &node.description]
+ .into_iter()
+ .any(|text| !text.trim().is_empty())
+}
+
 pub fn assign_refs(root: &mut UiNode) {
  let mut next = 1_u64;
  walk_mut(root, &mut |node, _| {
@@ -721,6 +747,49 @@ mod tests {
  assert!(!diff.use_full_view);
  }
 
+ fn framed_tree(children: Vec<UiNode>) -> UiNode {
+ UiNode {
+ frame: Frame {
+ x: 20.0,
+ y: 30.0,
+ w: 800.0,
+ h: 600.0,
+ },
+ children,
+ ..node("window", "Root window title", Vec::new())
+ }
+ }
+
+ #[test]
+ fn large_tree_with_fewer_than_three_text_descendants_is_sparse() {
+ let tree = framed_tree(vec![
+ node("button", "Menu", Vec::new()),
+ node("static_text", "Score", Vec::new()),
+ ]);
+
+ assert!(is_text_sparse(&tree));
+ }
+
+ #[test]
+ fn large_tree_with_three_text_descendants_is_adequate() {
+ let tree = framed_tree(vec![
+ node("button", "Menu", Vec::new()),
+ node("static_text", "Score", Vec::new()),
+ node("static_text", "Ready", Vec::new()),
+ ]);
+
+ assert!(!is_text_sparse(&tree));
+ }
+
+ #[test]
+ fn small_tree_is_not_sparse_even_without_text_descendants() {
+ let mut tree = framed_tree(Vec::new());
+ tree.frame.w = 100.0;
+ tree.frame.h = 100.0;
+
+ assert!(!is_text_sparse(&tree));
+ }
+
  #[test]
  fn safe_prefix_ends_on_utf8_and_line_boundaries() {
  let text = "a\nβ\ncharlie";

crates/computer-use-mcp/src/tools.rs

@@ -405,15 +405,15 @@ mod dispatch {
 
  pub(super) async fn observe_ui(params: ObserveUiParams) -> CallToolResult {
  let permissions = permissions();
+ let config = crate::config::get();
  let needs_accessibility = !matches!(params.mode, Some(ObserveMode::Visual));
- let needs_screen_recording =
- matches!(params.mode, Some(ObserveMode::Visual | ObserveMode::Fused));
+ let needs_screen_recording = config.image_mode != crate::config::ImageMode::Never
+&& matches!(params.mode, Some(ObserveMode::Visual | ObserveMode::Fused));
  if let Some(result) =
  permission_gate(permissions, needs_accessibility, needs_screen_recording)
  {
  return result;
  }
- let config = crate::config::get();
  if config.image_mode == crate::config::ImageMode::Always
  && let Some(result) = permission_gate(permissions, false, true)
  {
@@ -858,6 +858,9 @@ mod dispatch {
  count_nodes(&observation.tree),
  outline::interactive_count(&observation.tree)
  );
+ if observed.text_sparse {
+ text.push_str("\ntext_sparse: true");
+ }
  if let Some(warning) = warning {
  text.push_str("\nwarning: ");
  text.push_str(warning);
@@ -1066,6 +1069,89 @@ mod dispatch {
  fn tool_error(message: &str) -> CallToolResult {
  CallToolResult::error(vec![ContentBlock::text(message)])
  }
+
+ #[cfg(all(test, target_os = "macos"))]
+ mod tests {
+ use super::*;
+ use crate::outline::Frame;
+
+ fn sparse_observation(screenshot_png: Option<Vec<u8>>) -> RootObservation {
+ RootObservation {
+ root: RootInfo {
+ ref_id: "@r1".into(),
+ app_name: "Canvas App".into(),
+ title: "Canvas".into(),
+ frame: Frame {
+ x: 0.0,
+ y: 0.0,
+ w: 800.0,
+ h: 600.0,
+ },
+ ..RootInfo::default()
+ },
+ tree: UiNode {
+ role: "window".into(),
+ title: "Canvas".into(),
+ frame: Frame {
+ x: 0.0,
+ y: 0.0,
+ w: 800.0,
+ h: 600.0,
+ },
+ ..UiNode::default()
+ },
+ text_sparse: true,
+ screenshot_png,
+ }
+ }
+
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn auto_with_permission_returns_sparse_flag_and_image() {
+ let permissions = crate::permissions::PermissionStatus {
+ accessibility: true,
+ screen_recording: true,
+ };
+ let policy = capture_policy(
+ crate::config::ImageMode::Auto,
+ Some(ObserveMode::Semantic),
+ &permissions,
+ );
+ assert_eq!(policy, CapturePolicy::IfSparse);
+ assert!(policy.should_capture(true));
+
+ let result =
+ save_observation(sparse_observation(Some(vec![0x89, b'P', b'N', b'G'])), None);
+ assert!(matches!(
+ result.content.as_slice(),
+ [ContentBlock::Text(text), ContentBlock::Image(image)]
+ if text.text.contains("text_sparse: true")
+ && image.mime_type == "image/png"
+ ));
+ }
+
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn never_returns_sparse_flag_without_image() {
+ let permissions = crate::permissions::PermissionStatus {
+ accessibility: true,
+ screen_recording: true,
+ };
+ let policy = capture_policy(
+ crate::config::ImageMode::Never,
+ Some(ObserveMode::Visual),
+ &permissions,
+ );
+ assert_eq!(policy, CapturePolicy::Never);
+ assert!(!policy.should_capture(true));
+
+ let result = save_observation(sparse_observation(None), None);
+ assert!(matches!(
+ result.content.as_slice(),
+ [ContentBlock::Text(text)] if text.text.contains("text_sparse: true")
+ ));
+ }
+ }
 }
 
 #[cfg(test)]

docs/computer-use.md

@@ -34,10 +34,12 @@ Core contract, inherited from pi-computer-use:
 - **Bounded output.** Model-visible text is capped; oversized results return a preview plus a
  continuation ref for `read_text`.
 
-Deliberate v1 deviations from pi-computer-use (documented so later work can close them):
-no OCR/`pictureOnly` nodes, no CDP browser roots (browser automation stays on the
+Deliberate deviations from pi-computer-use (documented so later work can close them): no
+Windows/UIAutomation backend, no CDP browser roots (browser automation stays on the
 `tcode_preview` server and the embedded WebView), no separate helper app (see below), and a
-simplified successor-diff heuristic.
+simplified successor-diff heuristic. The earlier no-OCR/`pictureOnly`-node deviation is resolved
+by raw-image pass-through for text-sparse accessibility trees. By maintainer decision this
+fallback does not run OCR or synthesize text nodes; the model reads the attached pixels directly.
 
 ## Architecture
 
@@ -61,6 +63,25 @@ Unlike pi-computer-use, tcode needs **no helper app**: tcode is itself a signed
 Accessibility and Screen Recording grants attach directly to tcode. That removes helper
 install/signing/attribution handling entirely.
 
+## Text-sparse image fallback
+
+An observed window of at least 20,000 square screen points is considered text-sparse when fewer
+than three accessibility descendants expose a title, value, or description. The root window
+title is excluded. A sparse observation includes `text_sparse: true` so the agent knows the AX
+outline does not adequately describe the window.
+
+Image mode controls the raw screenshot attachment through the same capture path as other
+observations:
+
+- `auto` attaches one window screenshot only when the sparse rule triggers and Screen Recording
+ permission is available. Without permission, it returns the plain sparse tree and a warning.
+- `always` attaches one screenshot to every observation; sparse observations still include the
+ marker.
+- `never` never captures or attaches an image; sparse observations still include the marker.
+
+The window is captured at most once per observation. The fallback is intentionally OCR-free and
+does not add `pictureOnly` or other synthesized nodes.
+
 ## macOS permissions
 
 | Permission | Needed for | Check | Request |
@@ -102,5 +123,6 @@ own "Quit & Reopen" dialog. tcode therefore treats any permission flow as a pote
  grant permissions inside the VM, then inspect permission status via SSH.
 - CI (macOS/Linux/Windows) builds the platform fallback paths and runs the platform-neutral unit
  tests:
- outline folding and search ranking, state-store eviction and staleness, tool schemas,
- settings serde round-trips, and MCP registration wiring for all three provider paths.
+ outline folding, search ranking, text-sparse fallback decisions and observation shape,
+ state-store eviction and staleness, tool schemas, settings serde round-trips, and MCP
+ registration wiring for all three provider paths.

crates/computer-use-mcp/src/backend.rs

@@ -70,6 +70,16 @@ pub enum CapturePolicy {
  IfSparse,
 }
 
+impl CapturePolicy {
+ pub fn should_capture(self, text_sparse: bool) -> bool {
+ match self {
+ Self::Never => false,
+ Self::Always => true,
+ Self::IfSparse => text_sparse,
+ }
+ }
+}
+
 #[derive(Debug, Clone, Copy)]
 pub struct ObserveRequest {
  pub semantic: bool,
@@ -80,6 +90,7 @@ pub struct ObserveRequest {
 pub struct RootObservation {
  pub root: RootInfo,
  pub tree: UiNode,
+ pub text_sparse: bool,
  pub screenshot_png: Option<Vec<u8>>,
 }
 

crates/computer-use-mcp/src/backend/macos/mod.rs

@@ -17,10 +17,10 @@ use core_graphics::window::{
 };
 
 use super::{
- ActionKind, ActionRequest, ActionResult, BackendError, BackendErrorCode, CapturePolicy,
-ObserveRequest, RootFilters, RootInfo, RootObservation,
+ ActionKind, ActionRequest, ActionResult, BackendError, BackendErrorCode, ObserveRequest,
+ RootFilters, RootInfo, RootObservation,
 };
-use crate::outline::{UiNode, interactive_count};
+use crate::outline::{UiNode, is_text_sparse};
 
 pub(super) struct MacosBackend;
 
@@ -108,17 +108,15 @@ impl MacosBackend {
  ..UiNode::default()
  }
  };
- let should_capture = match request.capture {
- CapturePolicy::Never => false,
- CapturePolicy::Always => true,
- CapturePolicy::IfSparse => interactive_count(&tree) <= 3,
- };
+ let text_sparse = is_text_sparse(&tree);
+ let should_capture = request.capture.should_capture(text_sparse);
  let screenshot_png = should_capture
  .then(|| capture::capture_window(root))
  .transpose()?;
  Ok(RootObservation {
  root: root.clone(),
  tree,
+ text_sparse,
  screenshot_png,
  })
  }

crates/computer-use-mcp/src/outline.rs

@@ -9,6 +9,8 @@ pub const MAX_MODEL_LINES: usize = 2_000;
 pub const PREVIEW_BYTES: usize = 16 * 1024;
 pub const PAGE_BYTES: usize = 16 * 1024;
 pub const SEARCH_LIMIT: usize = 20;
+pub const TEXT_SPARSE_DESCENDANT_THRESHOLD: usize = 3;
+pub const TEXT_SPARSE_MIN_FRAME_AREA: f64 = 20_000.0;
 
 const FOLDED_DEPTH: usize = 7;
 const FOLDED_LINES: usize = 500;
@@ -125,6 +127,30 @@ pub fn interactive_count(root: &UiNode) -> usize {
  count
 }
 
+/// A large accessibility tree is text-sparse when fewer than three of its
+/// descendants expose a title, value, or description. The root window's own
+/// title is deliberately excluded because it does not describe the contents.
+pub fn is_text_sparse(root: &UiNode) -> bool {
+ let area = root.frame.w * root.frame.h;
+ root.frame.has_area()
+ && area.is_finite()
+ && area >= TEXT_SPARSE_MIN_FRAME_AREA
+ && text_bearing_descendant_count(root) < TEXT_SPARSE_DESCENDANT_THRESHOLD
+}
+
+fn text_bearing_descendant_count(root: &UiNode) -> usize {
+ root.children
+ .iter()
+ .map(|child| usize::from(node_has_text(child)) + text_bearing_descendant_count(child))
+ .sum()
+}
+
+fn node_has_text(node: &UiNode) -> bool {
+ [&node.title, &node.value, &node.description]
+ .into_iter()
+ .any(|text| !text.trim().is_empty())
+}
+
 pub fn assign_refs(root: &mut UiNode) {
  let mut next = 1_u64;
  walk_mut(root, &mut |node, _| {
@@ -721,6 +747,49 @@ mod tests {
  assert!(!diff.use_full_view);
  }
 
+ fn framed_tree(children: Vec<UiNode>) -> UiNode {
+ UiNode {
+ frame: Frame {
+ x: 20.0,
+ y: 30.0,
+ w: 800.0,
+ h: 600.0,
+ },
+ children,
+ ..node("window", "Root window title", Vec::new())
+ }
+ }
+
+ #[test]
+ fn large_tree_with_fewer_than_three_text_descendants_is_sparse() {
+ let tree = framed_tree(vec![
+ node("button", "Menu", Vec::new()),
+ node("static_text", "Score", Vec::new()),
+ ]);
+
+ assert!(is_text_sparse(&tree));
+ }
+
+ #[test]
+ fn large_tree_with_three_text_descendants_is_adequate() {
+ let tree = framed_tree(vec![
+ node("button", "Menu", Vec::new()),
+ node("static_text", "Score", Vec::new()),
+ node("static_text", "Ready", Vec::new()),
+ ]);
+
+ assert!(!is_text_sparse(&tree));
+ }
+
+ #[test]
+ fn small_tree_is_not_sparse_even_without_text_descendants() {
+ let mut tree = framed_tree(Vec::new());
+ tree.frame.w = 100.0;
+ tree.frame.h = 100.0;
+
+ assert!(!is_text_sparse(&tree));
+ }
+
  #[test]
  fn safe_prefix_ends_on_utf8_and_line_boundaries() {
  let text = "a\nβ\ncharlie";

crates/computer-use-mcp/src/tools.rs

@@ -405,15 +405,15 @@ mod dispatch {
 
  pub(super) async fn observe_ui(params: ObserveUiParams) -> CallToolResult {
  let permissions = permissions();
+ let config = crate::config::get();
  let needs_accessibility = !matches!(params.mode, Some(ObserveMode::Visual));
- let needs_screen_recording =
- matches!(params.mode, Some(ObserveMode::Visual | ObserveMode::Fused));
+ let needs_screen_recording = config.image_mode != crate::config::ImageMode::Never
+&& matches!(params.mode, Some(ObserveMode::Visual | ObserveMode::Fused));
  if let Some(result) =
  permission_gate(permissions, needs_accessibility, needs_screen_recording)
  {
  return result;
  }
- let config = crate::config::get();
  if config.image_mode == crate::config::ImageMode::Always
  && let Some(result) = permission_gate(permissions, false, true)
  {
@@ -858,6 +858,9 @@ mod dispatch {
  count_nodes(&observation.tree),
  outline::interactive_count(&observation.tree)
  );
+ if observed.text_sparse {
+ text.push_str("\ntext_sparse: true");
+ }
  if let Some(warning) = warning {
  text.push_str("\nwarning: ");
  text.push_str(warning);
@@ -1066,6 +1069,89 @@ mod dispatch {
  fn tool_error(message: &str) -> CallToolResult {
  CallToolResult::error(vec![ContentBlock::text(message)])
  }
+
+ #[cfg(all(test, target_os = "macos"))]
+ mod tests {
+ use super::*;
+ use crate::outline::Frame;
+
+ fn sparse_observation(screenshot_png: Option<Vec<u8>>) -> RootObservation {
+ RootObservation {
+ root: RootInfo {
+ ref_id: "@r1".into(),
+ app_name: "Canvas App".into(),
+ title: "Canvas".into(),
+ frame: Frame {
+ x: 0.0,
+ y: 0.0,
+ w: 800.0,
+ h: 600.0,
+ },
+ ..RootInfo::default()
+ },
+ tree: UiNode {
+ role: "window".into(),
+ title: "Canvas".into(),
+ frame: Frame {
+ x: 0.0,
+ y: 0.0,
+ w: 800.0,
+ h: 600.0,
+ },
+ ..UiNode::default()
+ },
+ text_sparse: true,
+ screenshot_png,
+ }
+ }
+
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn auto_with_permission_returns_sparse_flag_and_image() {
+ let permissions = crate::permissions::PermissionStatus {
+ accessibility: true,
+ screen_recording: true,
+ };
+ let policy = capture_policy(
+ crate::config::ImageMode::Auto,
+ Some(ObserveMode::Semantic),
+ &permissions,
+ );
+ assert_eq!(policy, CapturePolicy::IfSparse);
+ assert!(policy.should_capture(true));
+
+ let result =
+ save_observation(sparse_observation(Some(vec![0x89, b'P', b'N', b'G'])), None);
+ assert!(matches!(
+ result.content.as_slice(),
+ [ContentBlock::Text(text), ContentBlock::Image(image)]
+ if text.text.contains("text_sparse: true")
+ && image.mime_type == "image/png"
+ ));
+ }
+
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn never_returns_sparse_flag_without_image() {
+ let permissions = crate::permissions::PermissionStatus {
+ accessibility: true,
+ screen_recording: true,
+ };
+ let policy = capture_policy(
+ crate::config::ImageMode::Never,
+ Some(ObserveMode::Visual),
+ &permissions,
+ );
+ assert_eq!(policy, CapturePolicy::Never);
+ assert!(!policy.should_capture(true));
+
+ let result = save_observation(sparse_observation(None), None);
+ assert!(matches!(
+ result.content.as_slice(),
+ [ContentBlock::Text(text)] if text.text.contains("text_sparse: true")
+ ));
+ }
+ }
 }
 
 #[cfg(test)]

docs/computer-use.md

@@ -34,10 +34,12 @@ Core contract, inherited from pi-computer-use:
 - **Bounded output.** Model-visible text is capped; oversized results return a preview plus a
  continuation ref for `read_text`.
 
-Deliberate v1 deviations from pi-computer-use (documented so later work can close them):
-no OCR/`pictureOnly` nodes, no CDP browser roots (browser automation stays on the
+Deliberate deviations from pi-computer-use (documented so later work can close them): no
+Windows/UIAutomation backend, no CDP browser roots (browser automation stays on the
 `tcode_preview` server and the embedded WebView), no separate helper app (see below), and a
-simplified successor-diff heuristic.
+simplified successor-diff heuristic. The earlier no-OCR/`pictureOnly`-node deviation is resolved
+by raw-image pass-through for text-sparse accessibility trees. By maintainer decision this
+fallback does not run OCR or synthesize text nodes; the model reads the attached pixels directly.
 
 ## Architecture
 
@@ -61,6 +63,25 @@ Unlike pi-computer-use, tcode needs **no helper app**: tcode is itself a signed
 Accessibility and Screen Recording grants attach directly to tcode. That removes helper
 install/signing/attribution handling entirely.
 
+## Text-sparse image fallback
+
+An observed window of at least 20,000 square screen points is considered text-sparse when fewer
+than three accessibility descendants expose a title, value, or description. The root window
+title is excluded. A sparse observation includes `text_sparse: true` so the agent knows the AX
+outline does not adequately describe the window.
+
+Image mode controls the raw screenshot attachment through the same capture path as other
+observations:
+
+- `auto` attaches one window screenshot only when the sparse rule triggers and Screen Recording
+ permission is available. Without permission, it returns the plain sparse tree and a warning.
+- `always` attaches one screenshot to every observation; sparse observations still include the
+ marker.
+- `never` never captures or attaches an image; sparse observations still include the marker.
+
+The window is captured at most once per observation. The fallback is intentionally OCR-free and
+does not add `pictureOnly` or other synthesized nodes.
+
 ## macOS permissions
 
 | Permission | Needed for | Check | Request |
@@ -102,5 +123,6 @@ own "Quit & Reopen" dialog. tcode therefore treats any permission flow as a pote
  grant permissions inside the VM, then inspect permission status via SSH.
 - CI (macOS/Linux/Windows) builds the platform fallback paths and runs the platform-neutral unit
  tests:
- outline folding and search ranking, state-store eviction and staleness, tool schemas,
- settings serde round-trips, and MCP registration wiring for all three provider paths.
+ outline folding, search ranking, text-sparse fallback decisions and observation shape,
+ state-store eviction and staleness, tool schemas, settings serde round-trips, and MCP
+ registration wiring for all three provider paths.

crates/computer-use-mcp/src/backend.rs

@@ -70,6 +70,16 @@ pub enum CapturePolicy {
  IfSparse,
 }
 
+impl CapturePolicy {
+ pub fn should_capture(self, text_sparse: bool) -> bool {
+ match self {
+ Self::Never => false,
+ Self::Always => true,
+ Self::IfSparse => text_sparse,
+ }
+ }
+}
+
 #[derive(Debug, Clone, Copy)]
 pub struct ObserveRequest {
  pub semantic: bool,
@@ -80,6 +90,7 @@ pub struct ObserveRequest {
 pub struct RootObservation {
  pub root: RootInfo,
  pub tree: UiNode,
+ pub text_sparse: bool,
  pub screenshot_png: Option<Vec<u8>>,
 }
 

crates/computer-use-mcp/src/backend/macos/mod.rs

@@ -17,10 +17,10 @@ use core_graphics::window::{
 };
 
 use super::{
- ActionKind, ActionRequest, ActionResult, BackendError, BackendErrorCode, CapturePolicy,
-ObserveRequest, RootFilters, RootInfo, RootObservation,
+ ActionKind, ActionRequest, ActionResult, BackendError, BackendErrorCode, ObserveRequest,
+ RootFilters, RootInfo, RootObservation,
 };
-use crate::outline::{UiNode, interactive_count};
+use crate::outline::{UiNode, is_text_sparse};
 
 pub(super) struct MacosBackend;
 
@@ -108,17 +108,15 @@ impl MacosBackend {
  ..UiNode::default()
  }
  };
- let should_capture = match request.capture {
- CapturePolicy::Never => false,
- CapturePolicy::Always => true,
- CapturePolicy::IfSparse => interactive_count(&tree) <= 3,
- };
+ let text_sparse = is_text_sparse(&tree);
+ let should_capture = request.capture.should_capture(text_sparse);
  let screenshot_png = should_capture
  .then(|| capture::capture_window(root))
  .transpose()?;
  Ok(RootObservation {
  root: root.clone(),
  tree,
+ text_sparse,
  screenshot_png,
  })
  }

crates/computer-use-mcp/src/outline.rs

@@ -9,6 +9,8 @@ pub const MAX_MODEL_LINES: usize = 2_000;
 pub const PREVIEW_BYTES: usize = 16 * 1024;
 pub const PAGE_BYTES: usize = 16 * 1024;
 pub const SEARCH_LIMIT: usize = 20;
+pub const TEXT_SPARSE_DESCENDANT_THRESHOLD: usize = 3;
+pub const TEXT_SPARSE_MIN_FRAME_AREA: f64 = 20_000.0;
 
 const FOLDED_DEPTH: usize = 7;
 const FOLDED_LINES: usize = 500;
@@ -125,6 +127,30 @@ pub fn interactive_count(root: &UiNode) -> usize {
  count
 }
 
+/// A large accessibility tree is text-sparse when fewer than three of its
+/// descendants expose a title, value, or description. The root window's own
+/// title is deliberately excluded because it does not describe the contents.
+pub fn is_text_sparse(root: &UiNode) -> bool {
+ let area = root.frame.w * root.frame.h;
+ root.frame.has_area()
+ && area.is_finite()
+ && area >= TEXT_SPARSE_MIN_FRAME_AREA
+ && text_bearing_descendant_count(root) < TEXT_SPARSE_DESCENDANT_THRESHOLD
+}
+
+fn text_bearing_descendant_count(root: &UiNode) -> usize {
+ root.children
+ .iter()
+ .map(|child| usize::from(node_has_text(child)) + text_bearing_descendant_count(child))
+ .sum()
+}
+
+fn node_has_text(node: &UiNode) -> bool {
+ [&node.title, &node.value, &node.description]
+ .into_iter()
+ .any(|text| !text.trim().is_empty())
+}
+
 pub fn assign_refs(root: &mut UiNode) {
  let mut next = 1_u64;
  walk_mut(root, &mut |node, _| {
@@ -721,6 +747,49 @@ mod tests {
  assert!(!diff.use_full_view);
  }
 
+ fn framed_tree(children: Vec<UiNode>) -> UiNode {
+ UiNode {
+ frame: Frame {
+ x: 20.0,
+ y: 30.0,
+ w: 800.0,
+ h: 600.0,
+ },
+ children,
+ ..node("window", "Root window title", Vec::new())
+ }
+ }
+
+ #[test]
+ fn large_tree_with_fewer_than_three_text_descendants_is_sparse() {
+ let tree = framed_tree(vec![
+ node("button", "Menu", Vec::new()),
+ node("static_text", "Score", Vec::new()),
+ ]);
+
+ assert!(is_text_sparse(&tree));
+ }
+
+ #[test]
+ fn large_tree_with_three_text_descendants_is_adequate() {
+ let tree = framed_tree(vec![
+ node("button", "Menu", Vec::new()),
+ node("static_text", "Score", Vec::new()),
+ node("static_text", "Ready", Vec::new()),
+ ]);
+
+ assert!(!is_text_sparse(&tree));
+ }
+
+ #[test]
+ fn small_tree_is_not_sparse_even_without_text_descendants() {
+ let mut tree = framed_tree(Vec::new());
+ tree.frame.w = 100.0;
+ tree.frame.h = 100.0;
+
+ assert!(!is_text_sparse(&tree));
+ }
+
  #[test]
  fn safe_prefix_ends_on_utf8_and_line_boundaries() {
  let text = "a\nβ\ncharlie";

crates/computer-use-mcp/src/tools.rs

@@ -405,15 +405,15 @@ mod dispatch {
 
  pub(super) async fn observe_ui(params: ObserveUiParams) -> CallToolResult {
  let permissions = permissions();
+ let config = crate::config::get();
  let needs_accessibility = !matches!(params.mode, Some(ObserveMode::Visual));
- let needs_screen_recording =
- matches!(params.mode, Some(ObserveMode::Visual | ObserveMode::Fused));
+ let needs_screen_recording = config.image_mode != crate::config::ImageMode::Never
+&& matches!(params.mode, Some(ObserveMode::Visual | ObserveMode::Fused));
  if let Some(result) =
  permission_gate(permissions, needs_accessibility, needs_screen_recording)
  {
  return result;
  }
- let config = crate::config::get();
  if config.image_mode == crate::config::ImageMode::Always
  && let Some(result) = permission_gate(permissions, false, true)
  {
@@ -858,6 +858,9 @@ mod dispatch {
  count_nodes(&observation.tree),
  outline::interactive_count(&observation.tree)
  );
+ if observed.text_sparse {
+ text.push_str("\ntext_sparse: true");
+ }
  if let Some(warning) = warning {
  text.push_str("\nwarning: ");
  text.push_str(warning);
@@ -1066,6 +1069,89 @@ mod dispatch {
  fn tool_error(message: &str) -> CallToolResult {
  CallToolResult::error(vec![ContentBlock::text(message)])
  }
+
+ #[cfg(all(test, target_os = "macos"))]
+ mod tests {
+ use super::*;
+ use crate::outline::Frame;
+
+ fn sparse_observation(screenshot_png: Option<Vec<u8>>) -> RootObservation {
+ RootObservation {
+ root: RootInfo {
+ ref_id: "@r1".into(),
+ app_name: "Canvas App".into(),
+ title: "Canvas".into(),
+ frame: Frame {
+ x: 0.0,
+ y: 0.0,
+ w: 800.0,
+ h: 600.0,
+ },
+ ..RootInfo::default()
+ },
+ tree: UiNode {
+ role: "window".into(),
+ title: "Canvas".into(),
+ frame: Frame {
+ x: 0.0,
+ y: 0.0,
+ w: 800.0,
+ h: 600.0,
+ },
+ ..UiNode::default()
+ },
+ text_sparse: true,
+ screenshot_png,
+ }
+ }
+
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn auto_with_permission_returns_sparse_flag_and_image() {
+ let permissions = crate::permissions::PermissionStatus {
+ accessibility: true,
+ screen_recording: true,
+ };
+ let policy = capture_policy(
+ crate::config::ImageMode::Auto,
+ Some(ObserveMode::Semantic),
+ &permissions,
+ );
+ assert_eq!(policy, CapturePolicy::IfSparse);
+ assert!(policy.should_capture(true));
+
+ let result =
+ save_observation(sparse_observation(Some(vec![0x89, b'P', b'N', b'G'])), None);
+ assert!(matches!(
+ result.content.as_slice(),
+ [ContentBlock::Text(text), ContentBlock::Image(image)]
+ if text.text.contains("text_sparse: true")
+ && image.mime_type == "image/png"
+ ));
+ }
+
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn never_returns_sparse_flag_without_image() {
+ let permissions = crate::permissions::PermissionStatus {
+ accessibility: true,
+ screen_recording: true,
+ };
+ let policy = capture_policy(
+ crate::config::ImageMode::Never,
+ Some(ObserveMode::Visual),
+ &permissions,
+ );
+ assert_eq!(policy, CapturePolicy::Never);
+ assert!(!policy.should_capture(true));
+
+ let result = save_observation(sparse_observation(None), None);
+ assert!(matches!(
+ result.content.as_slice(),
+ [ContentBlock::Text(text)] if text.text.contains("text_sparse: true")
+ ));
+ }
+ }
 }
 
 #[cfg(test)]

docs/computer-use.md

@@ -34,10 +34,12 @@ Core contract, inherited from pi-computer-use:
 - **Bounded output.** Model-visible text is capped; oversized results return a preview plus a
  continuation ref for `read_text`.
 
-Deliberate v1 deviations from pi-computer-use (documented so later work can close them):
-no OCR/`pictureOnly` nodes, no CDP browser roots (browser automation stays on the
+Deliberate deviations from pi-computer-use (documented so later work can close them): no
+Windows/UIAutomation backend, no CDP browser roots (browser automation stays on the
 `tcode_preview` server and the embedded WebView), no separate helper app (see below), and a
-simplified successor-diff heuristic.
+simplified successor-diff heuristic. The earlier no-OCR/`pictureOnly`-node deviation is resolved
+by raw-image pass-through for text-sparse accessibility trees. By maintainer decision this
+fallback does not run OCR or synthesize text nodes; the model reads the attached pixels directly.
 
 ## Architecture
 
@@ -61,6 +63,25 @@ Unlike pi-computer-use, tcode needs **no helper app**: tcode is itself a signed
 Accessibility and Screen Recording grants attach directly to tcode. That removes helper
 install/signing/attribution handling entirely.
 
+## Text-sparse image fallback
+
+An observed window of at least 20,000 square screen points is considered text-sparse when fewer
+than three accessibility descendants expose a title, value, or description. The root window
+title is excluded. A sparse observation includes `text_sparse: true` so the agent knows the AX
+outline does not adequately describe the window.
+
+Image mode controls the raw screenshot attachment through the same capture path as other
+observations:
+
+- `auto` attaches one window screenshot only when the sparse rule triggers and Screen Recording
+ permission is available. Without permission, it returns the plain sparse tree and a warning.
+- `always` attaches one screenshot to every observation; sparse observations still include the
+ marker.
+- `never` never captures or attaches an image; sparse observations still include the marker.
+
+The window is captured at most once per observation. The fallback is intentionally OCR-free and
+does not add `pictureOnly` or other synthesized nodes.
+
 ## macOS permissions
 
 | Permission | Needed for | Check | Request |
@@ -102,5 +123,6 @@ own "Quit & Reopen" dialog. tcode therefore treats any permission flow as a pote
  grant permissions inside the VM, then inspect permission status via SSH.
 - CI (macOS/Linux/Windows) builds the platform fallback paths and runs the platform-neutral unit
  tests:
- outline folding and search ranking, state-store eviction and staleness, tool schemas,
- settings serde round-trips, and MCP registration wiring for all three provider paths.
+ outline folding, search ranking, text-sparse fallback decisions and observation shape,
+ state-store eviction and staleness, tool schemas, settings serde round-trips, and MCP
+ registration wiring for all three provider paths.

crates/computer-use-mcp/src/backend.rs

@@ -70,6 +70,16 @@ pub enum CapturePolicy {
  IfSparse,
 }
 
+impl CapturePolicy {
+ pub fn should_capture(self, text_sparse: bool) -> bool {
+ match self {
+ Self::Never => false,
+ Self::Always => true,
+ Self::IfSparse => text_sparse,
+ }
+ }
+}
+
 #[derive(Debug, Clone, Copy)]
 pub struct ObserveRequest {
  pub semantic: bool,
@@ -80,6 +90,7 @@ pub struct ObserveRequest {
 pub struct RootObservation {
  pub root: RootInfo,
  pub tree: UiNode,
+ pub text_sparse: bool,
  pub screenshot_png: Option<Vec<u8>>,
 }
 

crates/computer-use-mcp/src/backend/macos/mod.rs

@@ -17,10 +17,10 @@ use core_graphics::window::{
 };
 
 use super::{
- ActionKind, ActionRequest, ActionResult, BackendError, BackendErrorCode, CapturePolicy,
-ObserveRequest, RootFilters, RootInfo, RootObservation,
+ ActionKind, ActionRequest, ActionResult, BackendError, BackendErrorCode, ObserveRequest,
+ RootFilters, RootInfo, RootObservation,
 };
-use crate::outline::{UiNode, interactive_count};
+use crate::outline::{UiNode, is_text_sparse};
 
 pub(super) struct MacosBackend;
 
@@ -108,17 +108,15 @@ impl MacosBackend {
  ..UiNode::default()
  }
  };
- let should_capture = match request.capture {
- CapturePolicy::Never => false,
- CapturePolicy::Always => true,
- CapturePolicy::IfSparse => interactive_count(&tree) <= 3,
- };
+ let text_sparse = is_text_sparse(&tree);
+ let should_capture = request.capture.should_capture(text_sparse);
  let screenshot_png = should_capture
  .then(|| capture::capture_window(root))
  .transpose()?;
  Ok(RootObservation {
  root: root.clone(),
  tree,
+ text_sparse,
  screenshot_png,
  })
  }

crates/computer-use-mcp/src/outline.rs

@@ -9,6 +9,8 @@ pub const MAX_MODEL_LINES: usize = 2_000;
 pub const PREVIEW_BYTES: usize = 16 * 1024;
 pub const PAGE_BYTES: usize = 16 * 1024;
 pub const SEARCH_LIMIT: usize = 20;
+pub const TEXT_SPARSE_DESCENDANT_THRESHOLD: usize = 3;
+pub const TEXT_SPARSE_MIN_FRAME_AREA: f64 = 20_000.0;
 
 const FOLDED_DEPTH: usize = 7;
 const FOLDED_LINES: usize = 500;
@@ -125,6 +127,30 @@ pub fn interactive_count(root: &UiNode) -> usize {
  count
 }
 
+/// A large accessibility tree is text-sparse when fewer than three of its
+/// descendants expose a title, value, or description. The root window's own
+/// title is deliberately excluded because it does not describe the contents.
+pub fn is_text_sparse(root: &UiNode) -> bool {
+ let area = root.frame.w * root.frame.h;
+ root.frame.has_area()
+ && area.is_finite()
+ && area >= TEXT_SPARSE_MIN_FRAME_AREA
+ && text_bearing_descendant_count(root) < TEXT_SPARSE_DESCENDANT_THRESHOLD
+}
+
+fn text_bearing_descendant_count(root: &UiNode) -> usize {
+ root.children
+ .iter()
+ .map(|child| usize::from(node_has_text(child)) + text_bearing_descendant_count(child))
+ .sum()
+}
+
+fn node_has_text(node: &UiNode) -> bool {
+ [&node.title, &node.value, &node.description]
+ .into_iter()
+ .any(|text| !text.trim().is_empty())
+}
+
 pub fn assign_refs(root: &mut UiNode) {
  let mut next = 1_u64;
  walk_mut(root, &mut |node, _| {
@@ -721,6 +747,49 @@ mod tests {
  assert!(!diff.use_full_view);
  }
 
+ fn framed_tree(children: Vec<UiNode>) -> UiNode {
+ UiNode {
+ frame: Frame {
+ x: 20.0,
+ y: 30.0,
+ w: 800.0,
+ h: 600.0,
+ },
+ children,
+ ..node("window", "Root window title", Vec::new())
+ }
+ }
+
+ #[test]
+ fn large_tree_with_fewer_than_three_text_descendants_is_sparse() {
+ let tree = framed_tree(vec![
+ node("button", "Menu", Vec::new()),
+ node("static_text", "Score", Vec::new()),
+ ]);
+
+ assert!(is_text_sparse(&tree));
+ }
+
+ #[test]
+ fn large_tree_with_three_text_descendants_is_adequate() {
+ let tree = framed_tree(vec![
+ node("button", "Menu", Vec::new()),
+ node("static_text", "Score", Vec::new()),
+ node("static_text", "Ready", Vec::new()),
+ ]);
+
+ assert!(!is_text_sparse(&tree));
+ }
+
+ #[test]
+ fn small_tree_is_not_sparse_even_without_text_descendants() {
+ let mut tree = framed_tree(Vec::new());
+ tree.frame.w = 100.0;
+ tree.frame.h = 100.0;
+
+ assert!(!is_text_sparse(&tree));
+ }
+
  #[test]
  fn safe_prefix_ends_on_utf8_and_line_boundaries() {
  let text = "a\nβ\ncharlie";

crates/computer-use-mcp/src/tools.rs

@@ -405,15 +405,15 @@ mod dispatch {
 
  pub(super) async fn observe_ui(params: ObserveUiParams) -> CallToolResult {
  let permissions = permissions();
+ let config = crate::config::get();
  let needs_accessibility = !matches!(params.mode, Some(ObserveMode::Visual));
- let needs_screen_recording =
- matches!(params.mode, Some(ObserveMode::Visual | ObserveMode::Fused));
+ let needs_screen_recording = config.image_mode != crate::config::ImageMode::Never
+&& matches!(params.mode, Some(ObserveMode::Visual | ObserveMode::Fused));
  if let Some(result) =
  permission_gate(permissions, needs_accessibility, needs_screen_recording)
  {
  return result;
  }
- let config = crate::config::get();
  if config.image_mode == crate::config::ImageMode::Always
  && let Some(result) = permission_gate(permissions, false, true)
  {
@@ -858,6 +858,9 @@ mod dispatch {
  count_nodes(&observation.tree),
  outline::interactive_count(&observation.tree)
  );
+ if observed.text_sparse {
+ text.push_str("\ntext_sparse: true");
+ }
  if let Some(warning) = warning {
  text.push_str("\nwarning: ");
  text.push_str(warning);
@@ -1066,6 +1069,89 @@ mod dispatch {
  fn tool_error(message: &str) -> CallToolResult {
  CallToolResult::error(vec![ContentBlock::text(message)])
  }
+
+ #[cfg(all(test, target_os = "macos"))]
+ mod tests {
+ use super::*;
+ use crate::outline::Frame;
+
+ fn sparse_observation(screenshot_png: Option<Vec<u8>>) -> RootObservation {
+ RootObservation {
+ root: RootInfo {
+ ref_id: "@r1".into(),
+ app_name: "Canvas App".into(),
+ title: "Canvas".into(),
+ frame: Frame {
+ x: 0.0,
+ y: 0.0,
+ w: 800.0,
+ h: 600.0,
+ },
+ ..RootInfo::default()
+ },
+ tree: UiNode {
+ role: "window".into(),
+ title: "Canvas".into(),
+ frame: Frame {
+ x: 0.0,
+ y: 0.0,
+ w: 800.0,
+ h: 600.0,
+ },
+ ..UiNode::default()
+ },
+ text_sparse: true,
+ screenshot_png,
+ }
+ }
+
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn auto_with_permission_returns_sparse_flag_and_image() {
+ let permissions = crate::permissions::PermissionStatus {
+ accessibility: true,
+ screen_recording: true,
+ };
+ let policy = capture_policy(
+ crate::config::ImageMode::Auto,
+ Some(ObserveMode::Semantic),
+ &permissions,
+ );
+ assert_eq!(policy, CapturePolicy::IfSparse);
+ assert!(policy.should_capture(true));
+
+ let result =
+ save_observation(sparse_observation(Some(vec![0x89, b'P', b'N', b'G'])), None);
+ assert!(matches!(
+ result.content.as_slice(),
+ [ContentBlock::Text(text), ContentBlock::Image(image)]
+ if text.text.contains("text_sparse: true")
+ && image.mime_type == "image/png"
+ ));
+ }
+
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn never_returns_sparse_flag_without_image() {
+ let permissions = crate::permissions::PermissionStatus {
+ accessibility: true,
+ screen_recording: true,
+ };
+ let policy = capture_policy(
+ crate::config::ImageMode::Never,
+ Some(ObserveMode::Visual),
+ &permissions,
+ );
+ assert_eq!(policy, CapturePolicy::Never);
+ assert!(!policy.should_capture(true));
+
+ let result = save_observation(sparse_observation(None), None);
+ assert!(matches!(
+ result.content.as_slice(),
+ [ContentBlock::Text(text)] if text.text.contains("text_sparse: true")
+ ));
+ }
+ }
 }
 
 #[cfg(test)]

docs/computer-use.md

@@ -34,10 +34,12 @@ Core contract, inherited from pi-computer-use:
 - **Bounded output.** Model-visible text is capped; oversized results return a preview plus a
  continuation ref for `read_text`.
 
-Deliberate v1 deviations from pi-computer-use (documented so later work can close them):
-no OCR/`pictureOnly` nodes, no CDP browser roots (browser automation stays on the
+Deliberate deviations from pi-computer-use (documented so later work can close them): no
+Windows/UIAutomation backend, no CDP browser roots (browser automation stays on the
 `tcode_preview` server and the embedded WebView), no separate helper app (see below), and a
-simplified successor-diff heuristic.
+simplified successor-diff heuristic. The earlier no-OCR/`pictureOnly`-node deviation is resolved
+by raw-image pass-through for text-sparse accessibility trees. By maintainer decision this
+fallback does not run OCR or synthesize text nodes; the model reads the attached pixels directly.
 
 ## Architecture
 
@@ -61,6 +63,25 @@ Unlike pi-computer-use, tcode needs **no helper app**: tcode is itself a signed
 Accessibility and Screen Recording grants attach directly to tcode. That removes helper
 install/signing/attribution handling entirely.
 
+## Text-sparse image fallback
+
+An observed window of at least 20,000 square screen points is considered text-sparse when fewer
+than three accessibility descendants expose a title, value, or description. The root window
+title is excluded. A sparse observation includes `text_sparse: true` so the agent knows the AX
+outline does not adequately describe the window.
+
+Image mode controls the raw screenshot attachment through the same capture path as other
+observations:
+
+- `auto` attaches one window screenshot only when the sparse rule triggers and Screen Recording
+ permission is available. Without permission, it returns the plain sparse tree and a warning.
+- `always` attaches one screenshot to every observation; sparse observations still include the
+ marker.
+- `never` never captures or attaches an image; sparse observations still include the marker.
+
+The window is captured at most once per observation. The fallback is intentionally OCR-free and
+does not add `pictureOnly` or other synthesized nodes.
+
 ## macOS permissions
 
 | Permission | Needed for | Check | Request |
@@ -102,5 +123,6 @@ own "Quit & Reopen" dialog. tcode therefore treats any permission flow as a pote
  grant permissions inside the VM, then inspect permission status via SSH.
 - CI (macOS/Linux/Windows) builds the platform fallback paths and runs the platform-neutral unit
  tests:
- outline folding and search ranking, state-store eviction and staleness, tool schemas,
- settings serde round-trips, and MCP registration wiring for all three provider paths.
+ outline folding, search ranking, text-sparse fallback decisions and observation shape,
+ state-store eviction and staleness, tool schemas, settings serde round-trips, and MCP
+ registration wiring for all three provider paths.

crates/computer-use-mcp/src/backend.rs

@@ -70,6 +70,16 @@ pub enum CapturePolicy {
  IfSparse,
 }
 
+impl CapturePolicy {
+ pub fn should_capture(self, text_sparse: bool) -> bool {
+ match self {
+ Self::Never => false,
+ Self::Always => true,
+ Self::IfSparse => text_sparse,
+ }
+ }
+}
+
 #[derive(Debug, Clone, Copy)]
 pub struct ObserveRequest {
  pub semantic: bool,
@@ -80,6 +90,7 @@ pub struct ObserveRequest {
 pub struct RootObservation {
  pub root: RootInfo,
  pub tree: UiNode,
+ pub text_sparse: bool,
  pub screenshot_png: Option<Vec<u8>>,
 }
 

crates/computer-use-mcp/src/backend/macos/mod.rs

@@ -17,10 +17,10 @@ use core_graphics::window::{
 };
 
 use super::{
- ActionKind, ActionRequest, ActionResult, BackendError, BackendErrorCode, CapturePolicy,
-ObserveRequest, RootFilters, RootInfo, RootObservation,
+ ActionKind, ActionRequest, ActionResult, BackendError, BackendErrorCode, ObserveRequest,
+ RootFilters, RootInfo, RootObservation,
 };
-use crate::outline::{UiNode, interactive_count};
+use crate::outline::{UiNode, is_text_sparse};
 
 pub(super) struct MacosBackend;
 
@@ -108,17 +108,15 @@ impl MacosBackend {
  ..UiNode::default()
  }
  };
- let should_capture = match request.capture {
- CapturePolicy::Never => false,
- CapturePolicy::Always => true,
- CapturePolicy::IfSparse => interactive_count(&tree) <= 3,
- };
+ let text_sparse = is_text_sparse(&tree);
+ let should_capture = request.capture.should_capture(text_sparse);
  let screenshot_png = should_capture
  .then(|| capture::capture_window(root))
  .transpose()?;
  Ok(RootObservation {
  root: root.clone(),
  tree,
+ text_sparse,
  screenshot_png,
  })
  }

crates/computer-use-mcp/src/outline.rs

@@ -9,6 +9,8 @@ pub const MAX_MODEL_LINES: usize = 2_000;
 pub const PREVIEW_BYTES: usize = 16 * 1024;
 pub const PAGE_BYTES: usize = 16 * 1024;
 pub const SEARCH_LIMIT: usize = 20;
+pub const TEXT_SPARSE_DESCENDANT_THRESHOLD: usize = 3;
+pub const TEXT_SPARSE_MIN_FRAME_AREA: f64 = 20_000.0;
 
 const FOLDED_DEPTH: usize = 7;
 const FOLDED_LINES: usize = 500;
@@ -125,6 +127,30 @@ pub fn interactive_count(root: &UiNode) -> usize {
  count
 }
 
+/// A large accessibility tree is text-sparse when fewer than three of its
+/// descendants expose a title, value, or description. The root window's own
+/// title is deliberately excluded because it does not describe the contents.
+pub fn is_text_sparse(root: &UiNode) -> bool {
+ let area = root.frame.w * root.frame.h;
+ root.frame.has_area()
+ && area.is_finite()
+ && area >= TEXT_SPARSE_MIN_FRAME_AREA
+ && text_bearing_descendant_count(root) < TEXT_SPARSE_DESCENDANT_THRESHOLD
+}
+
+fn text_bearing_descendant_count(root: &UiNode) -> usize {
+ root.children
+ .iter()
+ .map(|child| usize::from(node_has_text(child)) + text_bearing_descendant_count(child))
+ .sum()
+}
+
+fn node_has_text(node: &UiNode) -> bool {
+ [&node.title, &node.value, &node.description]
+ .into_iter()
+ .any(|text| !text.trim().is_empty())
+}
+
 pub fn assign_refs(root: &mut UiNode) {
  let mut next = 1_u64;
  walk_mut(root, &mut |node, _| {
@@ -721,6 +747,49 @@ mod tests {
  assert!(!diff.use_full_view);
  }
 
+ fn framed_tree(children: Vec<UiNode>) -> UiNode {
+ UiNode {
+ frame: Frame {
+ x: 20.0,
+ y: 30.0,
+ w: 800.0,
+ h: 600.0,
+ },
+ children,
+ ..node("window", "Root window title", Vec::new())
+ }
+ }
+
+ #[test]
+ fn large_tree_with_fewer_than_three_text_descendants_is_sparse() {
+ let tree = framed_tree(vec![
+ node("button", "Menu", Vec::new()),
+ node("static_text", "Score", Vec::new()),
+ ]);
+
+ assert!(is_text_sparse(&tree));
+ }
+
+ #[test]
+ fn large_tree_with_three_text_descendants_is_adequate() {
+ let tree = framed_tree(vec![
+ node("button", "Menu", Vec::new()),
+ node("static_text", "Score", Vec::new()),
+ node("static_text", "Ready", Vec::new()),
+ ]);
+
+ assert!(!is_text_sparse(&tree));
+ }
+
+ #[test]
+ fn small_tree_is_not_sparse_even_without_text_descendants() {
+ let mut tree = framed_tree(Vec::new());
+ tree.frame.w = 100.0;
+ tree.frame.h = 100.0;
+
+ assert!(!is_text_sparse(&tree));
+ }
+
  #[test]
  fn safe_prefix_ends_on_utf8_and_line_boundaries() {
  let text = "a\nβ\ncharlie";

crates/computer-use-mcp/src/tools.rs

@@ -405,15 +405,15 @@ mod dispatch {
 
  pub(super) async fn observe_ui(params: ObserveUiParams) -> CallToolResult {
  let permissions = permissions();
+ let config = crate::config::get();
  let needs_accessibility = !matches!(params.mode, Some(ObserveMode::Visual));
- let needs_screen_recording =
- matches!(params.mode, Some(ObserveMode::Visual | ObserveMode::Fused));
+ let needs_screen_recording = config.image_mode != crate::config::ImageMode::Never
+&& matches!(params.mode, Some(ObserveMode::Visual | ObserveMode::Fused));
  if let Some(result) =
  permission_gate(permissions, needs_accessibility, needs_screen_recording)
  {
  return result;
  }
- let config = crate::config::get();
  if config.image_mode == crate::config::ImageMode::Always
  && let Some(result) = permission_gate(permissions, false, true)
  {
@@ -858,6 +858,9 @@ mod dispatch {
  count_nodes(&observation.tree),
  outline::interactive_count(&observation.tree)
  );
+ if observed.text_sparse {
+ text.push_str("\ntext_sparse: true");
+ }
  if let Some(warning) = warning {
  text.push_str("\nwarning: ");
  text.push_str(warning);
@@ -1066,6 +1069,89 @@ mod dispatch {
  fn tool_error(message: &str) -> CallToolResult {
  CallToolResult::error(vec![ContentBlock::text(message)])
  }
+
+ #[cfg(all(test, target_os = "macos"))]
+ mod tests {
+ use super::*;
+ use crate::outline::Frame;
+
+ fn sparse_observation(screenshot_png: Option<Vec<u8>>) -> RootObservation {
+ RootObservation {
+ root: RootInfo {
+ ref_id: "@r1".into(),
+ app_name: "Canvas App".into(),
+ title: "Canvas".into(),
+ frame: Frame {
+ x: 0.0,
+ y: 0.0,
+ w: 800.0,
+ h: 600.0,
+ },
+ ..RootInfo::default()
+ },
+ tree: UiNode {
+ role: "window".into(),
+ title: "Canvas".into(),
+ frame: Frame {
+ x: 0.0,
+ y: 0.0,
+ w: 800.0,
+ h: 600.0,
+ },
+ ..UiNode::default()
+ },
+ text_sparse: true,
+ screenshot_png,
+ }
+ }
+
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn auto_with_permission_returns_sparse_flag_and_image() {
+ let permissions = crate::permissions::PermissionStatus {
+ accessibility: true,
+ screen_recording: true,
+ };
+ let policy = capture_policy(
+ crate::config::ImageMode::Auto,
+ Some(ObserveMode::Semantic),
+ &permissions,
+ );
+ assert_eq!(policy, CapturePolicy::IfSparse);
+ assert!(policy.should_capture(true));
+
+ let result =
+ save_observation(sparse_observation(Some(vec![0x89, b'P', b'N', b'G'])), None);
+ assert!(matches!(
+ result.content.as_slice(),
+ [ContentBlock::Text(text), ContentBlock::Image(image)]
+ if text.text.contains("text_sparse: true")
+ && image.mime_type == "image/png"
+ ));
+ }
+
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn never_returns_sparse_flag_without_image() {
+ let permissions = crate::permissions::PermissionStatus {
+ accessibility: true,
+ screen_recording: true,
+ };
+ let policy = capture_policy(
+ crate::config::ImageMode::Never,
+ Some(ObserveMode::Visual),
+ &permissions,
+ );
+ assert_eq!(policy, CapturePolicy::Never);
+ assert!(!policy.should_capture(true));
+
+ let result = save_observation(sparse_observation(None), None);
+ assert!(matches!(
+ result.content.as_slice(),
+ [ContentBlock::Text(text)] if text.text.contains("text_sparse: true")
+ ));
+ }
+ }
 }
 
 #[cfg(test)]

docs/computer-use.md

@@ -34,10 +34,12 @@ Core contract, inherited from pi-computer-use:
 - **Bounded output.** Model-visible text is capped; oversized results return a preview plus a
  continuation ref for `read_text`.
 
-Deliberate v1 deviations from pi-computer-use (documented so later work can close them):
-no OCR/`pictureOnly` nodes, no CDP browser roots (browser automation stays on the
+Deliberate deviations from pi-computer-use (documented so later work can close them): no
+Windows/UIAutomation backend, no CDP browser roots (browser automation stays on the
 `tcode_preview` server and the embedded WebView), no separate helper app (see below), and a
-simplified successor-diff heuristic.
+simplified successor-diff heuristic. The earlier no-OCR/`pictureOnly`-node deviation is resolved
+by raw-image pass-through for text-sparse accessibility trees. By maintainer decision this
+fallback does not run OCR or synthesize text nodes; the model reads the attached pixels directly.
 
 ## Architecture
 
@@ -61,6 +63,25 @@ Unlike pi-computer-use, tcode needs **no helper app**: tcode is itself a signed
 Accessibility and Screen Recording grants attach directly to tcode. That removes helper
 install/signing/attribution handling entirely.
 
+## Text-sparse image fallback
+
+An observed window of at least 20,000 square screen points is considered text-sparse when fewer
+than three accessibility descendants expose a title, value, or description. The root window
+title is excluded. A sparse observation includes `text_sparse: true` so the agent knows the AX
+outline does not adequately describe the window.
+
+Image mode controls the raw screenshot attachment through the same capture path as other
+observations:
+
+- `auto` attaches one window screenshot only when the sparse rule triggers and Screen Recording
+ permission is available. Without permission, it returns the plain sparse tree and a warning.
+- `always` attaches one screenshot to every observation; sparse observations still include the
+ marker.
+- `never` never captures or attaches an image; sparse observations still include the marker.
+
+The window is captured at most once per observation. The fallback is intentionally OCR-free and
+does not add `pictureOnly` or other synthesized nodes.
+
 ## macOS permissions
 
 | Permission | Needed for | Check | Request |
@@ -102,5 +123,6 @@ own "Quit & Reopen" dialog. tcode therefore treats any permission flow as a pote
  grant permissions inside the VM, then inspect permission status via SSH.
 - CI (macOS/Linux/Windows) builds the platform fallback paths and runs the platform-neutral unit
  tests:
- outline folding and search ranking, state-store eviction and staleness, tool schemas,
- settings serde round-trips, and MCP registration wiring for all three provider paths.
+ outline folding, search ranking, text-sparse fallback decisions and observation shape,
+ state-store eviction and staleness, tool schemas, settings serde round-trips, and MCP
+ registration wiring for all three provider paths.

crates/computer-use-mcp/src/backend.rs

@@ -70,6 +70,16 @@ pub enum CapturePolicy {
  IfSparse,
 }
 
+impl CapturePolicy {
+ pub fn should_capture(self, text_sparse: bool) -> bool {
+ match self {
+ Self::Never => false,
+ Self::Always => true,
+ Self::IfSparse => text_sparse,
+ }
+ }
+}
+
 #[derive(Debug, Clone, Copy)]
 pub struct ObserveRequest {
  pub semantic: bool,
@@ -80,6 +90,7 @@ pub struct ObserveRequest {
 pub struct RootObservation {
  pub root: RootInfo,
  pub tree: UiNode,
+ pub text_sparse: bool,
  pub screenshot_png: Option<Vec<u8>>,
 }
 

crates/computer-use-mcp/src/backend/macos/mod.rs

@@ -17,10 +17,10 @@ use core_graphics::window::{
 };
 
 use super::{
- ActionKind, ActionRequest, ActionResult, BackendError, BackendErrorCode, CapturePolicy,
-ObserveRequest, RootFilters, RootInfo, RootObservation,
+ ActionKind, ActionRequest, ActionResult, BackendError, BackendErrorCode, ObserveRequest,
+ RootFilters, RootInfo, RootObservation,
 };
-use crate::outline::{UiNode, interactive_count};
+use crate::outline::{UiNode, is_text_sparse};
 
 pub(super) struct MacosBackend;
 
@@ -108,17 +108,15 @@ impl MacosBackend {
  ..UiNode::default()
  }
  };
- let should_capture = match request.capture {
- CapturePolicy::Never => false,
- CapturePolicy::Always => true,
- CapturePolicy::IfSparse => interactive_count(&tree) <= 3,
- };
+ let text_sparse = is_text_sparse(&tree);
+ let should_capture = request.capture.should_capture(text_sparse);
  let screenshot_png = should_capture
  .then(|| capture::capture_window(root))
  .transpose()?;
  Ok(RootObservation {
  root: root.clone(),
  tree,
+ text_sparse,
  screenshot_png,
  })
  }

crates/computer-use-mcp/src/outline.rs

@@ -9,6 +9,8 @@ pub const MAX_MODEL_LINES: usize = 2_000;
 pub const PREVIEW_BYTES: usize = 16 * 1024;
 pub const PAGE_BYTES: usize = 16 * 1024;
 pub const SEARCH_LIMIT: usize = 20;
+pub const TEXT_SPARSE_DESCENDANT_THRESHOLD: usize = 3;
+pub const TEXT_SPARSE_MIN_FRAME_AREA: f64 = 20_000.0;
 
 const FOLDED_DEPTH: usize = 7;
 const FOLDED_LINES: usize = 500;
@@ -125,6 +127,30 @@ pub fn interactive_count(root: &UiNode) -> usize {
  count
 }
 
+/// A large accessibility tree is text-sparse when fewer than three of its
+/// descendants expose a title, value, or description. The root window's own
+/// title is deliberately excluded because it does not describe the contents.
+pub fn is_text_sparse(root: &UiNode) -> bool {
+ let area = root.frame.w * root.frame.h;
+ root.frame.has_area()
+ && area.is_finite()
+ && area >= TEXT_SPARSE_MIN_FRAME_AREA
+ && text_bearing_descendant_count(root) < TEXT_SPARSE_DESCENDANT_THRESHOLD
+}
+
+fn text_bearing_descendant_count(root: &UiNode) -> usize {
+ root.children
+ .iter()
+ .map(|child| usize::from(node_has_text(child)) + text_bearing_descendant_count(child))
+ .sum()
+}
+
+fn node_has_text(node: &UiNode) -> bool {
+ [&node.title, &node.value, &node.description]
+ .into_iter()
+ .any(|text| !text.trim().is_empty())
+}
+
 pub fn assign_refs(root: &mut UiNode) {
  let mut next = 1_u64;
  walk_mut(root, &mut |node, _| {
@@ -721,6 +747,49 @@ mod tests {
  assert!(!diff.use_full_view);
  }
 
+ fn framed_tree(children: Vec<UiNode>) -> UiNode {
+ UiNode {
+ frame: Frame {
+ x: 20.0,
+ y: 30.0,
+ w: 800.0,
+ h: 600.0,
+ },
+ children,
+ ..node("window", "Root window title", Vec::new())
+ }
+ }
+
+ #[test]
+ fn large_tree_with_fewer_than_three_text_descendants_is_sparse() {
+ let tree = framed_tree(vec![
+ node("button", "Menu", Vec::new()),
+ node("static_text", "Score", Vec::new()),
+ ]);
+
+ assert!(is_text_sparse(&tree));
+ }
+
+ #[test]
+ fn large_tree_with_three_text_descendants_is_adequate() {
+ let tree = framed_tree(vec![
+ node("button", "Menu", Vec::new()),
+ node("static_text", "Score", Vec::new()),
+ node("static_text", "Ready", Vec::new()),
+ ]);
+
+ assert!(!is_text_sparse(&tree));
+ }
+
+ #[test]
+ fn small_tree_is_not_sparse_even_without_text_descendants() {
+ let mut tree = framed_tree(Vec::new());
+ tree.frame.w = 100.0;
+ tree.frame.h = 100.0;
+
+ assert!(!is_text_sparse(&tree));
+ }
+
  #[test]
  fn safe_prefix_ends_on_utf8_and_line_boundaries() {
  let text = "a\nβ\ncharlie";

crates/computer-use-mcp/src/tools.rs

@@ -405,15 +405,15 @@ mod dispatch {
 
  pub(super) async fn observe_ui(params: ObserveUiParams) -> CallToolResult {
  let permissions = permissions();
+ let config = crate::config::get();
  let needs_accessibility = !matches!(params.mode, Some(ObserveMode::Visual));
- let needs_screen_recording =
- matches!(params.mode, Some(ObserveMode::Visual | ObserveMode::Fused));
+ let needs_screen_recording = config.image_mode != crate::config::ImageMode::Never
+&& matches!(params.mode, Some(ObserveMode::Visual | ObserveMode::Fused));
  if let Some(result) =
  permission_gate(permissions, needs_accessibility, needs_screen_recording)
  {
  return result;
  }
- let config = crate::config::get();
  if config.image_mode == crate::config::ImageMode::Always
  && let Some(result) = permission_gate(permissions, false, true)
  {
@@ -858,6 +858,9 @@ mod dispatch {
  count_nodes(&observation.tree),
  outline::interactive_count(&observation.tree)
  );
+ if observed.text_sparse {
+ text.push_str("\ntext_sparse: true");
+ }
  if let Some(warning) = warning {
  text.push_str("\nwarning: ");
  text.push_str(warning);
@@ -1066,6 +1069,89 @@ mod dispatch {
  fn tool_error(message: &str) -> CallToolResult {
  CallToolResult::error(vec![ContentBlock::text(message)])
  }
+
+ #[cfg(all(test, target_os = "macos"))]
+ mod tests {
+ use super::*;
+ use crate::outline::Frame;
+
+ fn sparse_observation(screenshot_png: Option<Vec<u8>>) -> RootObservation {
+ RootObservation {
+ root: RootInfo {
+ ref_id: "@r1".into(),
+ app_name: "Canvas App".into(),
+ title: "Canvas".into(),
+ frame: Frame {
+ x: 0.0,
+ y: 0.0,
+ w: 800.0,
+ h: 600.0,
+ },
+ ..RootInfo::default()
+ },
+ tree: UiNode {
+ role: "window".into(),
+ title: "Canvas".into(),
+ frame: Frame {
+ x: 0.0,
+ y: 0.0,
+ w: 800.0,
+ h: 600.0,
+ },
+ ..UiNode::default()
+ },
+ text_sparse: true,
+ screenshot_png,
+ }
+ }
+
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn auto_with_permission_returns_sparse_flag_and_image() {
+ let permissions = crate::permissions::PermissionStatus {
+ accessibility: true,
+ screen_recording: true,
+ };
+ let policy = capture_policy(
+ crate::config::ImageMode::Auto,
+ Some(ObserveMode::Semantic),
+ &permissions,
+ );
+ assert_eq!(policy, CapturePolicy::IfSparse);
+ assert!(policy.should_capture(true));
+
+ let result =
+ save_observation(sparse_observation(Some(vec![0x89, b'P', b'N', b'G'])), None);
+ assert!(matches!(
+ result.content.as_slice(),
+ [ContentBlock::Text(text), ContentBlock::Image(image)]
+ if text.text.contains("text_sparse: true")
+ && image.mime_type == "image/png"
+ ));
+ }
+
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn never_returns_sparse_flag_without_image() {
+ let permissions = crate::permissions::PermissionStatus {
+ accessibility: true,
+ screen_recording: true,
+ };
+ let policy = capture_policy(
+ crate::config::ImageMode::Never,
+ Some(ObserveMode::Visual),
+ &permissions,
+ );
+ assert_eq!(policy, CapturePolicy::Never);
+ assert!(!policy.should_capture(true));
+
+ let result = save_observation(sparse_observation(None), None);
+ assert!(matches!(
+ result.content.as_slice(),
+ [ContentBlock::Text(text)] if text.text.contains("text_sparse: true")
+ ));
+ }
+ }
 }
 
 #[cfg(test)]

docs/computer-use.md

@@ -34,10 +34,12 @@ Core contract, inherited from pi-computer-use:
 - **Bounded output.** Model-visible text is capped; oversized results return a preview plus a
  continuation ref for `read_text`.
 
-Deliberate v1 deviations from pi-computer-use (documented so later work can close them):
-no OCR/`pictureOnly` nodes, no CDP browser roots (browser automation stays on the
+Deliberate deviations from pi-computer-use (documented so later work can close them): no
+Windows/UIAutomation backend, no CDP browser roots (browser automation stays on the
 `tcode_preview` server and the embedded WebView), no separate helper app (see below), and a
-simplified successor-diff heuristic.
+simplified successor-diff heuristic. The earlier no-OCR/`pictureOnly`-node deviation is resolved
+by raw-image pass-through for text-sparse accessibility trees. By maintainer decision this
+fallback does not run OCR or synthesize text nodes; the model reads the attached pixels directly.
 
 ## Architecture
 
@@ -61,6 +63,25 @@ Unlike pi-computer-use, tcode needs **no helper app**: tcode is itself a signed
 Accessibility and Screen Recording grants attach directly to tcode. That removes helper
 install/signing/attribution handling entirely.
 
+## Text-sparse image fallback
+
+An observed window of at least 20,000 square screen points is considered text-sparse when fewer
+than three accessibility descendants expose a title, value, or description. The root window
+title is excluded. A sparse observation includes `text_sparse: true` so the agent knows the AX
+outline does not adequately describe the window.
+
+Image mode controls the raw screenshot attachment through the same capture path as other
+observations:
+
+- `auto` attaches one window screenshot only when the sparse rule triggers and Screen Recording
+ permission is available. Without permission, it returns the plain sparse tree and a warning.
+- `always` attaches one screenshot to every observation; sparse observations still include the
+ marker.
+- `never` never captures or attaches an image; sparse observations still include the marker.
+
+The window is captured at most once per observation. The fallback is intentionally OCR-free and
+does not add `pictureOnly` or other synthesized nodes.
+
 ## macOS permissions
 
 | Permission | Needed for | Check | Request |
@@ -102,5 +123,6 @@ own "Quit & Reopen" dialog. tcode therefore treats any permission flow as a pote
  grant permissions inside the VM, then inspect permission status via SSH.
 - CI (macOS/Linux/Windows) builds the platform fallback paths and runs the platform-neutral unit
  tests:
- outline folding and search ranking, state-store eviction and staleness, tool schemas,
- settings serde round-trips, and MCP registration wiring for all three provider paths.
+ outline folding, search ranking, text-sparse fallback decisions and observation shape,
+ state-store eviction and staleness, tool schemas, settings serde round-trips, and MCP
+ registration wiring for all three provider paths.

crates/computer-use-mcp/src/backend.rs

@@ -70,6 +70,16 @@ pub enum CapturePolicy {
  IfSparse,
 }
 
+impl CapturePolicy {
+ pub fn should_capture(self, text_sparse: bool) -> bool {
+ match self {
+ Self::Never => false,
+ Self::Always => true,
+ Self::IfSparse => text_sparse,
+ }
+ }
+}
+
 #[derive(Debug, Clone, Copy)]
 pub struct ObserveRequest {
  pub semantic: bool,
@@ -80,6 +90,7 @@ pub struct ObserveRequest {
 pub struct RootObservation {
  pub root: RootInfo,
  pub tree: UiNode,
+ pub text_sparse: bool,
  pub screenshot_png: Option<Vec<u8>>,
 }
 

crates/computer-use-mcp/src/backend/macos/mod.rs

@@ -17,10 +17,10 @@ use core_graphics::window::{
 };
 
 use super::{
- ActionKind, ActionRequest, ActionResult, BackendError, BackendErrorCode, CapturePolicy,
-ObserveRequest, RootFilters, RootInfo, RootObservation,
+ ActionKind, ActionRequest, ActionResult, BackendError, BackendErrorCode, ObserveRequest,
+ RootFilters, RootInfo, RootObservation,
 };
-use crate::outline::{UiNode, interactive_count};
+use crate::outline::{UiNode, is_text_sparse};
 
 pub(super) struct MacosBackend;
 
@@ -108,17 +108,15 @@ impl MacosBackend {
  ..UiNode::default()
  }
  };
- let should_capture = match request.capture {
- CapturePolicy::Never => false,
- CapturePolicy::Always => true,
- CapturePolicy::IfSparse => interactive_count(&tree) <= 3,
- };
+ let text_sparse = is_text_sparse(&tree);
+ let should_capture = request.capture.should_capture(text_sparse);
  let screenshot_png = should_capture
  .then(|| capture::capture_window(root))
  .transpose()?;
  Ok(RootObservation {
  root: root.clone(),
  tree,
+ text_sparse,
  screenshot_png,
  })
  }

crates/computer-use-mcp/src/outline.rs

@@ -9,6 +9,8 @@ pub const MAX_MODEL_LINES: usize = 2_000;
 pub const PREVIEW_BYTES: usize = 16 * 1024;
 pub const PAGE_BYTES: usize = 16 * 1024;
 pub const SEARCH_LIMIT: usize = 20;
+pub const TEXT_SPARSE_DESCENDANT_THRESHOLD: usize = 3;
+pub const TEXT_SPARSE_MIN_FRAME_AREA: f64 = 20_000.0;
 
 const FOLDED_DEPTH: usize = 7;
 const FOLDED_LINES: usize = 500;
@@ -125,6 +127,30 @@ pub fn interactive_count(root: &UiNode) -> usize {
  count
 }
 
+/// A large accessibility tree is text-sparse when fewer than three of its
+/// descendants expose a title, value, or description. The root window's own
+/// title is deliberately excluded because it does not describe the contents.
+pub fn is_text_sparse(root: &UiNode) -> bool {
+ let area = root.frame.w * root.frame.h;
+ root.frame.has_area()
+ && area.is_finite()
+ && area >= TEXT_SPARSE_MIN_FRAME_AREA
+ && text_bearing_descendant_count(root) < TEXT_SPARSE_DESCENDANT_THRESHOLD
+}
+
+fn text_bearing_descendant_count(root: &UiNode) -> usize {
+ root.children
+ .iter()
+ .map(|child| usize::from(node_has_text(child)) + text_bearing_descendant_count(child))
+ .sum()
+}
+
+fn node_has_text(node: &UiNode) -> bool {
+ [&node.title, &node.value, &node.description]
+ .into_iter()
+ .any(|text| !text.trim().is_empty())
+}
+
 pub fn assign_refs(root: &mut UiNode) {
  let mut next = 1_u64;
  walk_mut(root, &mut |node, _| {
@@ -721,6 +747,49 @@ mod tests {
  assert!(!diff.use_full_view);
  }
 
+ fn framed_tree(children: Vec<UiNode>) -> UiNode {
+ UiNode {
+ frame: Frame {
+ x: 20.0,
+ y: 30.0,
+ w: 800.0,
+ h: 600.0,
+ },
+ children,
+ ..node("window", "Root window title", Vec::new())
+ }
+ }
+
+ #[test]
+ fn large_tree_with_fewer_than_three_text_descendants_is_sparse() {
+ let tree = framed_tree(vec![
+ node("button", "Menu", Vec::new()),
+ node("static_text", "Score", Vec::new()),
+ ]);
+
+ assert!(is_text_sparse(&tree));
+ }
+
+ #[test]
+ fn large_tree_with_three_text_descendants_is_adequate() {
+ let tree = framed_tree(vec![
+ node("button", "Menu", Vec::new()),
+ node("static_text", "Score", Vec::new()),
+ node("static_text", "Ready", Vec::new()),
+ ]);
+
+ assert!(!is_text_sparse(&tree));
+ }
+
+ #[test]
+ fn small_tree_is_not_sparse_even_without_text_descendants() {
+ let mut tree = framed_tree(Vec::new());
+ tree.frame.w = 100.0;
+ tree.frame.h = 100.0;
+
+ assert!(!is_text_sparse(&tree));
+ }
+
  #[test]
  fn safe_prefix_ends_on_utf8_and_line_boundaries() {
  let text = "a\nβ\ncharlie";

crates/computer-use-mcp/src/tools.rs

@@ -405,15 +405,15 @@ mod dispatch {
 
  pub(super) async fn observe_ui(params: ObserveUiParams) -> CallToolResult {
  let permissions = permissions();
+ let config = crate::config::get();
  let needs_accessibility = !matches!(params.mode, Some(ObserveMode::Visual));
- let needs_screen_recording =
- matches!(params.mode, Some(ObserveMode::Visual | ObserveMode::Fused));
+ let needs_screen_recording = config.image_mode != crate::config::ImageMode::Never
+&& matches!(params.mode, Some(ObserveMode::Visual | ObserveMode::Fused));
  if let Some(result) =
  permission_gate(permissions, needs_accessibility, needs_screen_recording)
  {
  return result;
  }
- let config = crate::config::get();
  if config.image_mode == crate::config::ImageMode::Always
  && let Some(result) = permission_gate(permissions, false, true)
  {
@@ -858,6 +858,9 @@ mod dispatch {
  count_nodes(&observation.tree),
  outline::interactive_count(&observation.tree)
  );
+ if observed.text_sparse {
+ text.push_str("\ntext_sparse: true");
+ }
  if let Some(warning) = warning {
  text.push_str("\nwarning: ");
  text.push_str(warning);
@@ -1066,6 +1069,89 @@ mod dispatch {
  fn tool_error(message: &str) -> CallToolResult {
  CallToolResult::error(vec![ContentBlock::text(message)])
  }
+
+ #[cfg(all(test, target_os = "macos"))]
+ mod tests {
+ use super::*;
+ use crate::outline::Frame;
+
+ fn sparse_observation(screenshot_png: Option<Vec<u8>>) -> RootObservation {
+ RootObservation {
+ root: RootInfo {
+ ref_id: "@r1".into(),
+ app_name: "Canvas App".into(),
+ title: "Canvas".into(),
+ frame: Frame {
+ x: 0.0,
+ y: 0.0,
+ w: 800.0,
+ h: 600.0,
+ },
+ ..RootInfo::default()
+ },
+ tree: UiNode {
+ role: "window".into(),
+ title: "Canvas".into(),
+ frame: Frame {
+ x: 0.0,
+ y: 0.0,
+ w: 800.0,
+ h: 600.0,
+ },
+ ..UiNode::default()
+ },
+ text_sparse: true,
+ screenshot_png,
+ }
+ }
+
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn auto_with_permission_returns_sparse_flag_and_image() {
+ let permissions = crate::permissions::PermissionStatus {
+ accessibility: true,
+ screen_recording: true,
+ };
+ let policy = capture_policy(
+ crate::config::ImageMode::Auto,
+ Some(ObserveMode::Semantic),
+ &permissions,
+ );
+ assert_eq!(policy, CapturePolicy::IfSparse);
+ assert!(policy.should_capture(true));
+
+ let result =
+ save_observation(sparse_observation(Some(vec![0x89, b'P', b'N', b'G'])), None);
+ assert!(matches!(
+ result.content.as_slice(),
+ [ContentBlock::Text(text), ContentBlock::Image(image)]
+ if text.text.contains("text_sparse: true")
+ && image.mime_type == "image/png"
+ ));
+ }
+
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn never_returns_sparse_flag_without_image() {
+ let permissions = crate::permissions::PermissionStatus {
+ accessibility: true,
+ screen_recording: true,
+ };
+ let policy = capture_policy(
+ crate::config::ImageMode::Never,
+ Some(ObserveMode::Visual),
+ &permissions,
+ );
+ assert_eq!(policy, CapturePolicy::Never);
+ assert!(!policy.should_capture(true));
+
+ let result = save_observation(sparse_observation(None), None);
+ assert!(matches!(
+ result.content.as_slice(),
+ [ContentBlock::Text(text)] if text.text.contains("text_sparse: true")
+ ));
+ }
+ }
 }
 
 #[cfg(test)]

docs/computer-use.md

@@ -34,10 +34,12 @@ Core contract, inherited from pi-computer-use:
 - **Bounded output.** Model-visible text is capped; oversized results return a preview plus a
  continuation ref for `read_text`.
 
-Deliberate v1 deviations from pi-computer-use (documented so later work can close them):
-no OCR/`pictureOnly` nodes, no CDP browser roots (browser automation stays on the
+Deliberate deviations from pi-computer-use (documented so later work can close them): no
+Windows/UIAutomation backend, no CDP browser roots (browser automation stays on the
 `tcode_preview` server and the embedded WebView), no separate helper app (see below), and a
-simplified successor-diff heuristic.
+simplified successor-diff heuristic. The earlier no-OCR/`pictureOnly`-node deviation is resolved
+by raw-image pass-through for text-sparse accessibility trees. By maintainer decision this
+fallback does not run OCR or synthesize text nodes; the model reads the attached pixels directly.
 
 ## Architecture
 
@@ -61,6 +63,25 @@ Unlike pi-computer-use, tcode needs **no helper app**: tcode is itself a signed
 Accessibility and Screen Recording grants attach directly to tcode. That removes helper
 install/signing/attribution handling entirely.
 
+## Text-sparse image fallback
+
+An observed window of at least 20,000 square screen points is considered text-sparse when fewer
+than three accessibility descendants expose a title, value, or description. The root window
+title is excluded. A sparse observation includes `text_sparse: true` so the agent knows the AX
+outline does not adequately describe the window.
+
+Image mode controls the raw screenshot attachment through the same capture path as other
+observations:
+
+- `auto` attaches one window screenshot only when the sparse rule triggers and Screen Recording
+ permission is available. Without permission, it returns the plain sparse tree and a warning.
+- `always` attaches one screenshot to every observation; sparse observations still include the
+ marker.
+- `never` never captures or attaches an image; sparse observations still include the marker.
+
+The window is captured at most once per observation. The fallback is intentionally OCR-free and
+does not add `pictureOnly` or other synthesized nodes.
+
 ## macOS permissions
 
 | Permission | Needed for | Check | Request |
@@ -102,5 +123,6 @@ own "Quit & Reopen" dialog. tcode therefore treats any permission flow as a pote
  grant permissions inside the VM, then inspect permission status via SSH.
 - CI (macOS/Linux/Windows) builds the platform fallback paths and runs the platform-neutral unit
  tests:
- outline folding and search ranking, state-store eviction and staleness, tool schemas,
- settings serde round-trips, and MCP registration wiring for all three provider paths.
+ outline folding, search ranking, text-sparse fallback decisions and observation shape,
+ state-store eviction and staleness, tool schemas, settings serde round-trips, and MCP
+ registration wiring for all three provider paths.

crates/computer-use-mcp/src/backend.rs

@@ -70,6 +70,16 @@ pub enum CapturePolicy {
  IfSparse,
 }
 
+impl CapturePolicy {
+ pub fn should_capture(self, text_sparse: bool) -> bool {
+ match self {
+ Self::Never => false,
+ Self::Always => true,
+ Self::IfSparse => text_sparse,
+ }
+ }
+}
+
 #[derive(Debug, Clone, Copy)]
 pub struct ObserveRequest {
  pub semantic: bool,
@@ -80,6 +90,7 @@ pub struct ObserveRequest {
 pub struct RootObservation {
  pub root: RootInfo,
  pub tree: UiNode,
+ pub text_sparse: bool,
  pub screenshot_png: Option<Vec<u8>>,
 }
 

crates/computer-use-mcp/src/backend/macos/mod.rs

@@ -17,10 +17,10 @@ use core_graphics::window::{
 };
 
 use super::{
- ActionKind, ActionRequest, ActionResult, BackendError, BackendErrorCode, CapturePolicy,
-ObserveRequest, RootFilters, RootInfo, RootObservation,
+ ActionKind, ActionRequest, ActionResult, BackendError, BackendErrorCode, ObserveRequest,
+ RootFilters, RootInfo, RootObservation,
 };
-use crate::outline::{UiNode, interactive_count};
+use crate::outline::{UiNode, is_text_sparse};
 
 pub(super) struct MacosBackend;
 
@@ -108,17 +108,15 @@ impl MacosBackend {
  ..UiNode::default()
  }
  };
- let should_capture = match request.capture {
- CapturePolicy::Never => false,
- CapturePolicy::Always => true,
- CapturePolicy::IfSparse => interactive_count(&tree) <= 3,
- };
+ let text_sparse = is_text_sparse(&tree);
+ let should_capture = request.capture.should_capture(text_sparse);
  let screenshot_png = should_capture
  .then(|| capture::capture_window(root))
  .transpose()?;
  Ok(RootObservation {
  root: root.clone(),
  tree,
+ text_sparse,
  screenshot_png,
  })
  }

crates/computer-use-mcp/src/outline.rs

@@ -9,6 +9,8 @@ pub const MAX_MODEL_LINES: usize = 2_000;
 pub const PREVIEW_BYTES: usize = 16 * 1024;
 pub const PAGE_BYTES: usize = 16 * 1024;
 pub const SEARCH_LIMIT: usize = 20;
+pub const TEXT_SPARSE_DESCENDANT_THRESHOLD: usize = 3;
+pub const TEXT_SPARSE_MIN_FRAME_AREA: f64 = 20_000.0;
 
 const FOLDED_DEPTH: usize = 7;
 const FOLDED_LINES: usize = 500;
@@ -125,6 +127,30 @@ pub fn interactive_count(root: &UiNode) -> usize {
  count
 }
 
+/// A large accessibility tree is text-sparse when fewer than three of its
+/// descendants expose a title, value, or description. The root window's own
+/// title is deliberately excluded because it does not describe the contents.
+pub fn is_text_sparse(root: &UiNode) -> bool {
+ let area = root.frame.w * root.frame.h;
+ root.frame.has_area()
+ && area.is_finite()
+ && area >= TEXT_SPARSE_MIN_FRAME_AREA
+ && text_bearing_descendant_count(root) < TEXT_SPARSE_DESCENDANT_THRESHOLD
+}
+
+fn text_bearing_descendant_count(root: &UiNode) -> usize {
+ root.children
+ .iter()
+ .map(|child| usize::from(node_has_text(child)) + text_bearing_descendant_count(child))
+ .sum()
+}
+
+fn node_has_text(node: &UiNode) -> bool {
+ [&node.title, &node.value, &node.description]
+ .into_iter()
+ .any(|text| !text.trim().is_empty())
+}
+
 pub fn assign_refs(root: &mut UiNode) {
  let mut next = 1_u64;
  walk_mut(root, &mut |node, _| {
@@ -721,6 +747,49 @@ mod tests {
  assert!(!diff.use_full_view);
  }
 
+ fn framed_tree(children: Vec<UiNode>) -> UiNode {
+ UiNode {
+ frame: Frame {
+ x: 20.0,
+ y: 30.0,
+ w: 800.0,
+ h: 600.0,
+ },
+ children,
+ ..node("window", "Root window title", Vec::new())
+ }
+ }
+
+ #[test]
+ fn large_tree_with_fewer_than_three_text_descendants_is_sparse() {
+ let tree = framed_tree(vec![
+ node("button", "Menu", Vec::new()),
+ node("static_text", "Score", Vec::new()),
+ ]);
+
+ assert!(is_text_sparse(&tree));
+ }
+
+ #[test]
+ fn large_tree_with_three_text_descendants_is_adequate() {
+ let tree = framed_tree(vec![
+ node("button", "Menu", Vec::new()),
+ node("static_text", "Score", Vec::new()),
+ node("static_text", "Ready", Vec::new()),
+ ]);
+
+ assert!(!is_text_sparse(&tree));
+ }
+
+ #[test]
+ fn small_tree_is_not_sparse_even_without_text_descendants() {
+ let mut tree = framed_tree(Vec::new());
+ tree.frame.w = 100.0;
+ tree.frame.h = 100.0;
+
+ assert!(!is_text_sparse(&tree));
+ }
+
  #[test]
  fn safe_prefix_ends_on_utf8_and_line_boundaries() {
  let text = "a\nβ\ncharlie";

crates/computer-use-mcp/src/tools.rs

@@ -405,15 +405,15 @@ mod dispatch {
 
  pub(super) async fn observe_ui(params: ObserveUiParams) -> CallToolResult {
  let permissions = permissions();
+ let config = crate::config::get();
  let needs_accessibility = !matches!(params.mode, Some(ObserveMode::Visual));
- let needs_screen_recording =
- matches!(params.mode, Some(ObserveMode::Visual | ObserveMode::Fused));
+ let needs_screen_recording = config.image_mode != crate::config::ImageMode::Never
+&& matches!(params.mode, Some(ObserveMode::Visual | ObserveMode::Fused));
  if let Some(result) =
  permission_gate(permissions, needs_accessibility, needs_screen_recording)
  {
  return result;
  }
- let config = crate::config::get();
  if config.image_mode == crate::config::ImageMode::Always
  && let Some(result) = permission_gate(permissions, false, true)
  {
@@ -858,6 +858,9 @@ mod dispatch {
  count_nodes(&observation.tree),
  outline::interactive_count(&observation.tree)
  );
+ if observed.text_sparse {
+ text.push_str("\ntext_sparse: true");
+ }
  if let Some(warning) = warning {
  text.push_str("\nwarning: ");
  text.push_str(warning);
@@ -1066,6 +1069,89 @@ mod dispatch {
  fn tool_error(message: &str) -> CallToolResult {
  CallToolResult::error(vec![ContentBlock::text(message)])
  }
+
+ #[cfg(all(test, target_os = "macos"))]
+ mod tests {
+ use super::*;
+ use crate::outline::Frame;
+
+ fn sparse_observation(screenshot_png: Option<Vec<u8>>) -> RootObservation {
+ RootObservation {
+ root: RootInfo {
+ ref_id: "@r1".into(),
+ app_name: "Canvas App".into(),
+ title: "Canvas".into(),
+ frame: Frame {
+ x: 0.0,
+ y: 0.0,
+ w: 800.0,
+ h: 600.0,
+ },
+ ..RootInfo::default()
+ },
+ tree: UiNode {
+ role: "window".into(),
+ title: "Canvas".into(),
+ frame: Frame {
+ x: 0.0,
+ y: 0.0,
+ w: 800.0,
+ h: 600.0,
+ },
+ ..UiNode::default()
+ },
+ text_sparse: true,
+ screenshot_png,
+ }
+ }
+
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn auto_with_permission_returns_sparse_flag_and_image() {
+ let permissions = crate::permissions::PermissionStatus {
+ accessibility: true,
+ screen_recording: true,
+ };
+ let policy = capture_policy(
+ crate::config::ImageMode::Auto,
+ Some(ObserveMode::Semantic),
+ &permissions,
+ );
+ assert_eq!(policy, CapturePolicy::IfSparse);
+ assert!(policy.should_capture(true));
+
+ let result =
+ save_observation(sparse_observation(Some(vec![0x89, b'P', b'N', b'G'])), None);
+ assert!(matches!(
+ result.content.as_slice(),
+ [ContentBlock::Text(text), ContentBlock::Image(image)]
+ if text.text.contains("text_sparse: true")
+ && image.mime_type == "image/png"
+ ));
+ }
+
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn never_returns_sparse_flag_without_image() {
+ let permissions = crate::permissions::PermissionStatus {
+ accessibility: true,
+ screen_recording: true,
+ };
+ let policy = capture_policy(
+ crate::config::ImageMode::Never,
+ Some(ObserveMode::Visual),
+ &permissions,
+ );
+ assert_eq!(policy, CapturePolicy::Never);
+ assert!(!policy.should_capture(true));
+
+ let result = save_observation(sparse_observation(None), None);
+ assert!(matches!(
+ result.content.as_slice(),
+ [ContentBlock::Text(text)] if text.text.contains("text_sparse: true")
+ ));
+ }
+ }
 }
 
 #[cfg(test)]

docs/computer-use.md

@@ -34,10 +34,12 @@ Core contract, inherited from pi-computer-use:
 - **Bounded output.** Model-visible text is capped; oversized results return a preview plus a
  continuation ref for `read_text`.
 
-Deliberate v1 deviations from pi-computer-use (documented so later work can close them):
-no OCR/`pictureOnly` nodes, no CDP browser roots (browser automation stays on the
+Deliberate deviations from pi-computer-use (documented so later work can close them): no
+Windows/UIAutomation backend, no CDP browser roots (browser automation stays on the
 `tcode_preview` server and the embedded WebView), no separate helper app (see below), and a
-simplified successor-diff heuristic.
+simplified successor-diff heuristic. The earlier no-OCR/`pictureOnly`-node deviation is resolved
+by raw-image pass-through for text-sparse accessibility trees. By maintainer decision this
+fallback does not run OCR or synthesize text nodes; the model reads the attached pixels directly.
 
 ## Architecture
 
@@ -61,6 +63,25 @@ Unlike pi-computer-use, tcode needs **no helper app**: tcode is itself a signed
 Accessibility and Screen Recording grants attach directly to tcode. That removes helper
 install/signing/attribution handling entirely.
 
+## Text-sparse image fallback
+
+An observed window of at least 20,000 square screen points is considered text-sparse when fewer
+than three accessibility descendants expose a title, value, or description. The root window
+title is excluded. A sparse observation includes `text_sparse: true` so the agent knows the AX
+outline does not adequately describe the window.
+
+Image mode controls the raw screenshot attachment through the same capture path as other
+observations:
+
+- `auto` attaches one window screenshot only when the sparse rule triggers and Screen Recording
+ permission is available. Without permission, it returns the plain sparse tree and a warning.
+- `always` attaches one screenshot to every observation; sparse observations still include the
+ marker.
+- `never` never captures or attaches an image; sparse observations still include the marker.
+
+The window is captured at most once per observation. The fallback is intentionally OCR-free and
+does not add `pictureOnly` or other synthesized nodes.
+
 ## macOS permissions
 
 | Permission | Needed for | Check | Request |
@@ -102,5 +123,6 @@ own "Quit & Reopen" dialog. tcode therefore treats any permission flow as a pote
  grant permissions inside the VM, then inspect permission status via SSH.
 - CI (macOS/Linux/Windows) builds the platform fallback paths and runs the platform-neutral unit
  tests:
- outline folding and search ranking, state-store eviction and staleness, tool schemas,
- settings serde round-trips, and MCP registration wiring for all three provider paths.
+ outline folding, search ranking, text-sparse fallback decisions and observation shape,
+ state-store eviction and staleness, tool schemas, settings serde round-trips, and MCP
+ registration wiring for all three provider paths.