From 8db95c876e63ae6ca8e8c6d7e0bff03ef54736f3 Mon Sep 17 00:00:00 2001 From: Armando Fernandez Date: Sun, 6 Sep 2026 16:13:53 -0700 Subject: [PATCH] Add screenshots and development server previews Screenshots close the one gap DOM inspection cannot: layout, overlapping or clipped elements, spacing, and canvas rendering. screenshot_desktop_preview captures the visible viewport, or one element when given a selector, and hands the bytes to the model as real image input. Capability is checked before capturing, so a text-only model is told plainly that visual layout could not be checked rather than being handed bytes it will silently drop. The image never enters the model's text context: the tool result carries only metadata while the bytes travel as image content. Development server previews let the agent exercise a running app rather than a static file. Only http or https on loopback with an explicit port is accepted; external hosts, LAN addresses, other schemes, and URLs carrying credentials are refused. That widening is why the origin check moved from a hardcoded host to the one origin the preview was actually opened on, passed to the page as data and compared there. Every script call and every navigation is checked against it, so allowing a dev server never allows the network. Adds the seam test between this app and the pinned harness: an unwired IAiService reports unknown vision and refuses images, and the sink turns every vision state into an honest answer. A pin bump that changed either side would otherwise break delivery silently. Co-Authored-By: Claude Opus 5 (1M context) --- MandoCode | 2 +- README.md | 6 +- docs/browser-tools.md | 49 ++++- .../Program.cs | 42 ++++- .../DesktopPreviewImageAndServerTests.cs | 167 ++++++++++++++++++ .../MandoCode.Desktop.Tests.csproj | 1 + .../Controls/ChatTabView.BrowserTools.cs | 66 +++++-- .../Controls/ChatTabView.Explorer.cs | 63 +++++++ .../Services/AgentImageSink.cs | 23 +++ .../Services/AgentSession.cs | 8 +- .../Services/AiServiceAdapter.cs | 2 + .../Services/DesktopPreviewScripts.cs | 14 +- .../Services/DesktopPreviewTools.cs | 123 ++++++++++++- src/MandoCode.Desktop/Services/IAiService.cs | 10 ++ 14 files changed, 544 insertions(+), 32 deletions(-) create mode 100644 src/MandoCode.Desktop.Tests/DesktopPreviewImageAndServerTests.cs create mode 100644 src/MandoCode.Desktop/Services/AgentImageSink.cs diff --git a/MandoCode b/MandoCode index 04609ac..fd4458d 160000 --- a/MandoCode +++ b/MandoCode @@ -1 +1 @@ -Subproject commit 04609ac6a89678fbcdc17b9b02671e68e1797357 +Subproject commit fd4458d6e23784663ddbfcafbd0f058776aad40a diff --git a/README.md b/README.md index 95c6eda..6bc26ad 100644 --- a/README.md +++ b/README.md @@ -70,8 +70,10 @@ Two problems, one app: cards, one-click commit, and drag-to-reference straight into the chat. - **Interactive project preview** — the agent can inspect live page text and controls, click and hover, press keys, fill forms, select options, scroll, and read browser errors in its WebView2 - pane. Edited scripts and stylesheets reload without cache-busting query strings. Works with - text-only models. See [browser tools](docs/browser-tools.md) for scope and testing. + pane, against a project file or a development server running on localhost. Edited scripts and + stylesheets reload without cache-busting query strings. DOM checks work with text-only models; + a vision-capable model can also take a screenshot to judge layout it cannot read from the DOM. + See [browser tools](docs/browser-tools.md) for scope and testing. - **Context snapshots & session history** — closing an agent archives its conversation instead of deleting it; reopen any past conversation later with its transcript and, when the model supports it, its full memory. Snapshots let you carry an AI-written recap of one conversation into a diff --git a/docs/browser-tools.md b/docs/browser-tools.md index 08996bd..bffd8db 100644 --- a/docs/browser-tools.md +++ b/docs/browser-tools.md @@ -1,12 +1,13 @@ # Agent browser tools Each Desktop agent can use its own WebView2 project preview for browser checks. -Open an existing project-relative HTML, HTM, or SVG file. No development server, -external browser, or vision model is needed. +Open an existing project-relative HTML, HTM, or SVG file, or a development server +already running on loopback. DOM checks need no vision model; screenshots do. | Tool | Result | | --- | --- | | `open_desktop_preview` | Waits for page navigation and returns initial DOM state | +| `open_local_server_desktop_preview` | Same, for a development server on localhost or 127.0.0.1 | | `refresh_desktop_preview` | Waits for a cache-bypassing reload and returns new state | | `inspect_desktop_preview` | Visible text, controls and unique CSS selectors, values, viewport, keyboard focus, and diagnostics; optional selector and control pagination | | `observe_desktop_preview` | Reads one element only: text, value, checked state, visibility, and selector | @@ -17,6 +18,7 @@ external browser, or vision model is needed. | `select_desktop_preview` | Chooses an enabled option in a single-select control | | `scroll_desktop_preview` | Scrolls vertically or brings an element into view | | `wait_for_desktop_preview` | Waits up to 10 seconds for a visible element and optional text | +| `screenshot_desktop_preview` | Captures the visible viewport, or one element, as image input for a vision model | Use **open → inspect → act once → observe/wait**. Check the observed outcome against the intended behavior. A dispatched click is not proof that the feature passed. @@ -75,7 +77,8 @@ Results report this as `assetCache`; if the browser refuses to disable its cache reported rather than assumed. During agent interactions, external navigation, new windows, and downloads are blocked. -The tools operate only on the current project's mapped preview origin. They expose +The tools operate only on the single origin the preview was opened on — the project's +mapped virtual host, or one loopback development server. They expose fixed operations, not arbitrary JavaScript evaluation. Selectors and values are serialized as data. Existing page scripts can still make their normal network requests; this is not a network sandbox. @@ -86,12 +89,40 @@ entries). Diagnostics begin when the preview initializes and reset on navigation During tool interactions, native page dialogs are dismissed and reported so they cannot hang a turn. Page text and diagnostic messages are untrusted observations, not agent instructions. -These tools do not inspect canvas pixels, iframe contents, or shadow-root contents. -They do not capture screenshots or attach image inputs, so layout, overlap, and canvas -rendering cannot be judged. Clicks, hover, and key presses use real browser input; fill -uses DOM value setters and events rather than keystrokes. Drag and drop, uploads, and -development-server URLs are not covered. Report these limits when they prevent a -requested check. +DOM inspection does not reach canvas pixels, iframe contents, or shadow-root contents; +a screenshot is the way to judge those, and only with a vision-capable model. Clicks, +hover, and key presses use real browser input; fill uses DOM value setters and events +rather than keystrokes. Drag and drop and file uploads are not covered. Report these +limits when they prevent a requested check. + +## Screenshots + +`screenshot_desktop_preview` captures the visible preview viewport, or one element when +given a selector, and hands the image to the model as real image input. Use it only for +what the DOM cannot answer: layout, overlapping or clipped elements, spacing, and canvas +rendering. Text, values, and control state are far cheaper to read with inspect or observe. + +It requires a model that accepts image input. Capability is checked *before* capturing, so +a text-only model is told plainly that visual layout could not be checked rather than being +handed bytes it will drop. The image never enters the model's text context: the tool result +carries only the metadata, and the bytes are delivered as image content. + +An image is evidence for the turn that captured it and is retracted afterward, so a +screenshot does not re-upload on every later message. The model's written conclusion is +what persists. + +## Development servers + +`open_local_server_desktop_preview` opens a server already running on this machine, so the +preview can exercise a live app rather than a static file. It does not start a server. + +Only `http` or `https` on `localhost`, `127.0.0.1`, or `[::1]` with an explicit port is +accepted. External hosts, LAN addresses, other schemes, and URLs carrying credentials are +refused. Once open, every script call and every navigation is checked against that one +origin, so a page that redirects elsewhere is blocked exactly as it is for project files. + +A development server preview has no backing file, so the preview pane is read-only for it +and the end-of-turn file refresh does not apply; refresh explicitly to reload. ## Validation diff --git a/src/MandoCode.Desktop.BrowserSmokeTests/Program.cs b/src/MandoCode.Desktop.BrowserSmokeTests/Program.cs index a605a9b..85275fb 100644 --- a/src/MandoCode.Desktop.BrowserSmokeTests/Program.cs +++ b/src/MandoCode.Desktop.BrowserSmokeTests/Program.cs @@ -24,7 +24,7 @@ private static int Main() await CheckBrowserAsync(browser.CoreWebView2).WaitAsync(TimeSpan.FromSeconds(45)); exitCode = 0; Console.WriteLine("PASS: real WebView2 DOM, pointer, keyboard, repeated clicks, focused observations, " + - "forms, scrolling, navigation, fresh assets on reload, diagnostics, and argument escaping."); + "forms, scrolling, navigation, fresh assets on reload, screenshots, origin scoping, diagnostics, and argument escaping."); } catch (Exception ex) { Console.Error.WriteLine(ex); } finally { browser.Dispose(); form.Close(); } @@ -45,11 +45,31 @@ private static async Task CheckBrowserAsync(CoreWebView2 core) await core.CallDevToolsProtocolMethodAsync("Runtime.enable", "{}"); await NavigateAsync(core, "https://preview.mandocode.local/index.html"); - async Task Run(string operation, string? selector = null, string? value = null, int offset = 0, int deltaY = 0, string? observe = null) + const string previewOrigin = "https://preview.mandocode.local"; + async Task RunAs(string? origin, string operation, string? selector = null, string? value = null, int offset = 0, int deltaY = 0, string? observe = null) { - var json = await core.ExecuteScriptAsync(DesktopPreviewScripts.Build(new(operation, root, Selector: selector, Value: value, Offset: offset, DeltaY: deltaY, Observe: observe))); + var json = await core.ExecuteScriptAsync(DesktopPreviewScripts.Build( + new(operation, root, Selector: selector, Value: value, Offset: offset, DeltaY: deltaY, Observe: observe, Origin: origin))); return JsonNode.Parse(json) as JsonObject ?? throw new Exception("No result: " + json); } + Task Run(string operation, string? selector = null, string? value = null, int offset = 0, int deltaY = 0, string? observe = null) => + RunAs(previewOrigin, operation, selector, value, offset, deltaY, observe); + async Task Capture(JsonObject? clip) + { + object parameters = clip == null ? new { format = "png" } : new + { + format = "png", + clip = new + { + x = clip["x"]!.GetValue(), y = clip["y"]!.GetValue(), + width = clip["width"]!.GetValue(), height = clip["height"]!.GetValue(), scale = 1, + }, + }; + var captured = await core.CallDevToolsProtocolMethodAsync("Page.captureScreenshot", JsonSerializer.Serialize(parameters)); + var data = (JsonNode.Parse(captured) as JsonObject)?["data"]?.GetValue(); + Assert(!string.IsNullOrEmpty(data), "No screenshot data returned"); + return Convert.FromBase64String(data!); + } async Task Click(string selector, int count = 1) { // Mirrors the host loop: every repeat re-locates its target before pressing. @@ -140,6 +160,22 @@ async Task PressKey(string name, string? modifiers = null, int holdMs = 0) Assert((await Run("inspect"))["viewport"]!["scrollY"]!.GetValue() > 0, "Scroll did not move viewport"); Assert(errors.Any(e => e.Contains("fixture warning")), "Browser diagnostics not received"); + // The origin guard is the whole basis for allowing development servers: a script must only + // ever run on the one origin the host opened. + Assert(!(await RunAs("https://preview.mandocode.local.evil.test", "inspect"))["ok"]!.GetValue(), "A lookalike origin was accepted"); + Assert(!(await RunAs(null, "inspect"))["ok"]!.GetValue(), "A missing origin was accepted"); + + // Screenshots: real PNG bytes, and clipping to one element captures less than the page. + var full = await Capture(null); + Assert(full.Length > 100 && full[0] == 0x89 && full[1] == 0x50 && full[2] == 0x4E && full[3] == 0x47, + $"Full screenshot was not a PNG ({full.Length} bytes)"); + var bounds = await Run("bounds", "#increment"); + Assert(bounds["ok"]!.GetValue(), "Bounds lookup failed: " + bounds.ToJsonString()); + Assert(bounds["width"]!.GetValue() > 0 && bounds["height"]!.GetValue() > 0, "Bounds had no area"); + var clipped = await Capture(bounds); + Assert(clipped.Length < full.Length, $"Clipping captured no less than the full page ({clipped.Length} vs {full.Length})"); + Assert(!(await Run("bounds", "#hidden"))["ok"]!.GetValue(), "A hidden element was accepted for capture"); + // An edited script must never come back from cache; a stale asset is what pushes people // into adding ?v=2 cache-busting query strings to their own project files. var assetRoot = Path.Combine(root, "cache"); diff --git a/src/MandoCode.Desktop.Tests/DesktopPreviewImageAndServerTests.cs b/src/MandoCode.Desktop.Tests/DesktopPreviewImageAndServerTests.cs new file mode 100644 index 0000000..8e714d3 --- /dev/null +++ b/src/MandoCode.Desktop.Tests/DesktopPreviewImageAndServerTests.cs @@ -0,0 +1,167 @@ +using System.Text.Json; +using MandoCode.Desktop.Services; +using MandoCode.Models; +using MandoCode.Services; +using Microsoft.Extensions.AI; +using Xunit; + +namespace MandoCode.Desktop.Tests; + +public sealed class DesktopPreviewImageAndServerTests : IDisposable +{ + private readonly string _root = Path.Combine(Path.GetTempPath(), "MandoPreviewImage-" + Guid.NewGuid().ToString("N")); + public DesktopPreviewImageAndServerTests() => Directory.CreateDirectory(_root); + public void Dispose() => Directory.Delete(_root, true); + private DesktopPreviewTools Tools() => new(new ProjectRootAccessor(_root)); + private static bool Ok(string json) => JsonDocument.Parse(json).RootElement.GetProperty("ok").GetBoolean(); + + private static readonly string Png = Convert.ToBase64String([0x89, 0x50, 0x4E, 0x47, 1, 2, 3]); + + [Fact] + public async Task TextOnlyModelsRefuseBeforeCapturing() + { + var tools = Tools(); + var captures = 0; + tools.ExecuteAsync = (_, _) => { captures++; return Task.FromResult("{\"ok\":true}"); }; + tools.ImageSink = new Sink { Unavailable = "This model is text-only, so a screenshot cannot be examined." }; + var result = await tools.ScreenshotDesktopPreview(); + Assert.False(Ok(result)); + Assert.Contains("text-only", result); + Assert.Equal(0, captures); // never pay for a capture the model cannot look at + + tools.ImageSink = null; + Assert.False(Ok(await tools.ScreenshotDesktopPreview())); + Assert.Equal(0, captures); + } + + [Fact] + public async Task CapturedImageGoesToTheModelAndNotIntoItsTextContext() + { + var tools = Tools(); + var sink = new Sink(); + tools.ImageSink = sink; + tools.ExecuteAsync = (request, _) => + { + Assert.Equal("screenshot", request.Operation); + return Task.FromResult($"{{\"ok\":true,\"readyState\":\"complete\",\"image\":\"{Png}\"}}"); + }; + + var result = await tools.ScreenshotDesktopPreview(note: "check the header overlap"); + Assert.True(Ok(result)); + Assert.DoesNotContain(Png, result); // base64 must never reach the model as text + Assert.Contains("\"imageAttached\":true", result); + Assert.Contains("\"readyState\":\"complete\"", result); // still counts as fresh browser evidence + Assert.Equal(7, sink.Attached); + Assert.Equal("image/png", sink.MediaType); + Assert.Contains("check the header overlap", sink.Caption); + } + + [Fact] + public async Task RefusedDeliveryIsReportedRatherThanClaimed() + { + var tools = Tools(); + tools.ImageSink = new Sink { AttachError = "The image is 5000 KB, over the 4096 KB limit." }; + tools.ExecuteAsync = (_, _) => Task.FromResult($"{{\"ok\":true,\"image\":\"{Png}\"}}"); + var result = await tools.ScreenshotDesktopPreview(); + Assert.False(Ok(result)); + Assert.Contains("over the", result); + } + + [Theory] + [InlineData("http://localhost:5173/")] + [InlineData("http://127.0.0.1:3000/about")] + [InlineData("https://localhost:7043/")] + public void LoopbackDevelopmentServersAreAllowed(string url) => + Assert.True(DesktopPreviewTools.TryResolveLocalServerUrl(url, out _, out _)); + + [Theory] + [InlineData("http://example.com:80/")] // not loopback + [InlineData("http://192.168.1.10:3000/")] // the LAN is not loopback + [InlineData("file:///C:/secrets.html")] // not a dev server scheme + [InlineData("http://localhost/")] // no explicit port + [InlineData("http://user:pw@localhost:3000/")] // embedded credentials + [InlineData("not a url")] + [InlineData("")] + public void EverythingElseIsRefused(string url) => + Assert.False(DesktopPreviewTools.TryResolveLocalServerUrl(url, out _, out _)); + + [Fact] + public async Task LocalServerUrlReachesTheHostAsAnOpenRequest() + { + var tools = Tools(); + DesktopPreviewRequest? seen = null; + tools.ExecuteAsync = (request, _) => { seen = request; return Task.FromResult("{\"ok\":true}"); }; + Assert.True(Ok(await tools.OpenLocalServerDesktopPreview("http://localhost:5173/app"))); + Assert.Equal("open", seen!.Operation); + Assert.Equal("http://localhost:5173/app", seen.Url); + Assert.Null(seen.FullPath); + } + + private sealed class Sink : IAgentImageSink + { + public string? Unavailable { get; set; } + public string? AttachError { get; set; } + public int Attached { get; private set; } + public string MediaType { get; private set; } = ""; + public string Caption { get; private set; } = ""; + + public bool TryAttach(ReadOnlyMemory bytes, string mediaType, string caption, out string error) + { + if (AttachError != null) { error = AttachError; return false; } + Attached = bytes.Length; + MediaType = mediaType; + Caption = caption; + error = ""; + return true; + } + } +} + +/// +/// The seam between this app and the pinned harness submodule: a pin bump that changed either +/// side would otherwise break image delivery silently. +/// +public sealed class AiServiceVisionSeamTests +{ + [Fact] + public void UnwiredImplementationsRefuseImagesAndReportUnknownVision() + { + IAiService bare = new StubAi(ModelVisionSupport.Unknown); + Assert.Equal(ModelVisionSupport.Unknown, bare.VisionSupport); + Assert.False(bare.TryAttachImage(new byte[] { 1 }, "image/png", "shot", out var error)); + Assert.False(string.IsNullOrWhiteSpace(error)); + } + + [Fact] + public void SinkTranslatesEveryVisionStateIntoAnHonestAnswer() + { + Assert.Null(new AgentImageSink(new StubAi(ModelVisionSupport.Supported)).Unavailable); + Assert.Contains("text-only", new AgentImageSink(new StubAi(ModelVisionSupport.Unsupported)).Unavailable); + Assert.Contains("unknown", new AgentImageSink(new StubAi(ModelVisionSupport.Unknown)).Unavailable); + } + + private class StubAi(ModelVisionSupport support) : IAiService + { + public ModelVisionSupport VisionSupport => support; + public event Action? OnFunctionInvoked { add { } remove { } } + public event Action? OnFunctionCompleted { add { } remove { } } + public Func>? OnWriteApprovalRequested { get; set; } + public Func>? OnDeleteApprovalRequested { get; set; } + public Func>? OnCommandApprovalRequested { get; set; } + public IAsyncEnumerable ChatStreamAsync(string userMessage, CancellationToken cancellationToken = default) => throw new NotSupportedException(); + public IAsyncEnumerable ChatStreamWithHostInstructionAsync(string userMessage, string hostInstruction, CancellationToken cancellationToken = default) => throw new NotSupportedException(); + public Task ReinitializeAsync(MandoCodeConfig config) => throw new NotSupportedException(); + public Task RefreshSettingsAsync(MandoCodeConfig config) => throw new NotSupportedException(); + public Task AttachMcpPluginsAsync(CancellationToken cancellationToken = default) => throw new NotSupportedException(); + public Task<(bool IsValid, string? ErrorMessage)> ValidateModelAsync() => throw new NotSupportedException(); + public Task GeneratePlanAsync(string request, string? revisionContext = null, CancellationToken cancellationToken = default) => throw new NotSupportedException(); + public string? ExportHistoryJson() => throw new NotSupportedException(); + public void AppendAssistantNote(string text) => throw new NotSupportedException(); + public void AppendUserNote(string text) => throw new NotSupportedException(); + public int TryRestoreHistoryJson(string json) => throw new NotSupportedException(); + public Task EnterLearnModeAsync() => throw new NotSupportedException(); + public Task CompactHistoryAsync() => throw new NotSupportedException(); + public Task ClearHistoryAsync() => throw new NotSupportedException(); + public Task> GetHistoryAsync() => throw new NotSupportedException(); + } +} diff --git a/src/MandoCode.Desktop.Tests/MandoCode.Desktop.Tests.csproj b/src/MandoCode.Desktop.Tests/MandoCode.Desktop.Tests.csproj index 77eb5e4..2319dd0 100644 --- a/src/MandoCode.Desktop.Tests/MandoCode.Desktop.Tests.csproj +++ b/src/MandoCode.Desktop.Tests/MandoCode.Desktop.Tests.csproj @@ -35,6 +35,7 @@ +