feature: deliver host-captured images to vision-capable models

docs/model-capabilities.md

@@ -24,3 +24,17 @@ Text/DOM browser tools need no vision capability. This change does not add brows
 inspection tools, screenshot capture, or image message delivery.
 
 Provider contract: https://docs.ollama.com/api-reference/show-model-details
+
+## Image delivery
+
+Detecting image support is separate from delivering an image. A host captures an image
+and calls `AIService.TryAttachImage`, which refuses unless the model reports vision
+support, the content is an image, and it is within `MaxImageInputBytes`.
+
+A tool result is text, so an image cannot ride back inside one. A queued image is added
+to history as a real user message after the current turn, and the turn is extended so the
+model looks at it — bounded by `MaxImageDeliveriesPerTurn`.
+
+Delivered images are retracted from history when the user turn ends. An image is evidence
+for the turn that captured it; keeping it would re-upload megabytes on every later request
+and crowd out the context the model needs. The model's written conclusion is what persists.

src/MandoCode/Services/Ai/AIService.cs

@@ -36,9 +36,90 @@ public class AIService
  public ModelVisionSupport VisionSupport => _visionIdentity == ModelIdentity
  ? _visionSupport : ModelVisionSupport.Unknown;
 
- // A host must also implement actual image delivery before exposing screenshot tools.
  public bool SupportsImageInput => VisionSupport == ModelVisionSupport.Supported;
 
+ /// <summary>Largest single host-supplied image, before base64 expansion.</summary>
+ public const int MaxImageInputBytes = 4 * 1024 * 1024;
+ /// <summary>How many times one user turn may be extended so the model can look at a new image.</summary>
+ public const int MaxImageDeliveriesPerTurn = 3;
+
+ private readonly List<AIContent> _pendingImages = [];
+ private int _imageDeliveriesThisTurn;
+
+ private Func<string, HttpResponseMessage>? _chatTransportForTests;
+
+ /// <summary>Test seam: answers model calls from the request body, so delivery can be asserted without a live daemon.</summary>
+ internal void SetChatClientFactoryForTests(Func<string, HttpResponseMessage> respond)
+ {
+ _chatTransportForTests = respond;
+ BuildAgent();
+ }
+
+ private sealed class StubTransport(Func<string, HttpResponseMessage> respond) : HttpMessageHandler
+ {
+ protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) =>
+ respond(request.Content == null ? "" : await request.Content.ReadAsStringAsync(cancellationToken));
+ }
+
+ /// <summary>
+ /// Queues a host-captured image for the model to actually look at. Delivery is deliberately
+ /// separate from capture: a tool result is text, so an image can only reach the model as
+ /// message content on a following turn. Refused outright unless the model accepts image
+ /// input, so a text-only model is never handed something it will silently ignore.
+ /// </summary>
+ public bool TryAttachImage(ReadOnlyMemory<byte> bytes, string mediaType, string caption, out string error)
+ {
+ if (!SupportsImageInput)
+ {
+ error = VisionSupport == ModelVisionSupport.Unknown
+ ? "This model's image support is unknown, so an image cannot be delivered. Use text and DOM observations."
+ : "This model is text-only and cannot receive images. Use text and DOM observations.";
+ return false;
+ }
+ if (bytes.IsEmpty) { error = "The captured image was empty."; return false; }
+ if (bytes.Length > MaxImageInputBytes)
+ {
+ error = $"The image is {bytes.Length / 1024} KB, over the {MaxImageInputBytes / 1024} KB limit.";
+ return false;
+ }
+ if (!mediaType.StartsWith("image/", StringComparison.OrdinalIgnoreCase))
+ {
+ error = "Only image content can be attached.";
+ return false;
+ }
+ lock (_pendingImages)
+ {
+ if (_pendingImages.Count >= 8) { error = "Too many images are already queued for this turn."; return false; }
+ if (!string.IsNullOrWhiteSpace(caption)) _pendingImages.Add(new TextContent(caption));
+ _pendingImages.Add(new DataContent(bytes, mediaType));
+ }
+ error = "";
+ return true;
+ }
+
+ /// <summary>
+ /// Moves queued images into history as a real user message so the model sees them on the next
+ /// turn. Returns the message so the caller can retract it afterward: an image is evidence for
+ /// the turn that asked for it, not permanent context that re-uploads on every later request.
+ /// </summary>
+ private async Task<ChatMessage?> TakePendingImageMessageAsync()
+ {
+ List<AIContent> contents;
+ lock (_pendingImages)
+ {
+ if (_pendingImages.Count == 0) return null;
+ contents = [.. _pendingImages];
+ _pendingImages.Clear();
+ }
+ contents.Insert(0, new TextContent(
+ "Host-captured image input follows. Describe only what is actually visible in it."));
+ var message = new ChatMessage(ChatRole.User, contents);
+ await _historyLock.WaitAsync();
+ try { _chatHistory.Add(message); }
+ finally { _historyLock.Release(); }
+ return message;
+ }
+
  // MAF agent — the live chat path (feat/agent-framework-migration).
  private AIAgent? _agent;
 
@@ -350,7 +431,8 @@ private void BuildAgent()
  // as a bogus transport error on slow local generations. The old client (like the old
  // agent it served) is left for GC rather than disposed — a rebuild can race a call still
  // in flight on the discarded agent.
- var ollamaHttpClient = new HttpClient(new NumCtxHttpHandler(EffectiveNumCtx))
+ var ollamaHttpClient = new HttpClient(
+ _chatTransportForTests is { } stub ? new StubTransport(stub) : new NumCtxHttpHandler(EffectiveNumCtx))
  {
  BaseAddress = new Uri(_config.OllamaEndpoint),
  Timeout = System.Threading.Timeout.InfiniteTimeSpan
@@ -765,6 +847,10 @@ public async IAsyncEnumerable<string> ChatStreamWithHostInstructionAsync(
  }
  finally { _historyLock.Release(); }
 
+ // Not cleared here: an image may be attached before the turn (a pasted screenshot) or
+ // during it (a preview capture). Both are real evidence and both must reach the model.
+ _imageDeliveriesThisTurn = 0;
+ var deliveredImages = new List<ChatMessage>();
  try
  {
  int continuations = 0;
@@ -773,6 +859,13 @@ public async IAsyncEnumerable<string> ChatStreamWithHostInstructionAsync(
  var (response, needsContinuation) = await RunOneChatTurnAsync(continuations, cancellationToken);
  yield return response;
 
+ // An image captured during the turn can only be looked at on a following one.
+ if (await TakePendingImageMessageAsync() is { } imageMessage)
+ {
+ deliveredImages.Add(imageMessage);
+ if (_imageDeliveriesThisTurn++ < MaxImageDeliveriesPerTurn) needsContinuation = true;
+ }
+
  if (!needsContinuation)
  break;
 
@@ -781,10 +874,15 @@ public async IAsyncEnumerable<string> ChatStreamWithHostInstructionAsync(
  }
  finally
  {
- if (hostMessage != null)
+ var transient = deliveredImages;
+ if (hostMessage != null) transient = [hostMessage, .. deliveredImages];
+ if (transient.Count > 0)
  {
  await _historyLock.WaitAsync(CancellationToken.None);
- try { _chatHistory.Remove(hostMessage); }
+ // Images are evidence for the turn that captured them. Leaving them in history
+ // would re-upload megabytes on every later request and crowd out the context the
+ // model needs; the model's written conclusion about the image is what persists.
+ try { foreach (var message in transient) _chatHistory.Remove(message); }
  finally { _historyLock.Release(); }
  }
  }

src/MandoCode/Services/Ai/PreviewToolPolicy.cs

@@ -9,7 +9,7 @@ public static bool IsLiveTool(string name) => name is
  "open_desktop_preview" or "refresh_desktop_preview" or "inspect_desktop_preview" or
  "observe_desktop_preview" or "click_desktop_preview" or "press_key_desktop_preview" or
  "hover_desktop_preview" or "fill_desktop_preview" or "select_desktop_preview" or
- "scroll_desktop_preview" or "wait_for_desktop_preview";
+ "scroll_desktop_preview" or "wait_for_desktop_preview" or "screenshot_desktop_preview";
 
  public static bool IsFailure(string name, string result) => IsLiveTool(name) && !HasSuccessfulResult(result);
 

tests/MandoCode.Tests/ImageInputDeliveryTests.cs

@@ -0,0 +1,98 @@
+using System.Net;
+using System.Text;
+using System.Text.Json;
+using MandoCode.Models;
+using MandoCode.Services;
+using Microsoft.Extensions.AI;
+using OllamaSharp;
+using Xunit;
+
+namespace MandoCode.Tests;
+
+public class ImageInputDeliveryTests
+{
+ private static readonly byte[] Png = [0x89, 0x50, 0x4E, 0x47, 1, 2, 3, 4];
+
+ [Fact]
+ public async Task TextOnlyAndUnknownModelsRefuseImages()
+ {
+ var ai = Create(new MandoCodeConfig());
+ Assert.False(ai.TryAttachImage(Png, "image/png", "shot", out var unknown));
+ Assert.Contains("unknown", unknown);
+
+ using var textOnly = Client("{\"capabilities\":[\"tools\"]}");
+ await ai.ValidateModelAsync(textOnly);
+ Assert.False(ai.TryAttachImage(Png, "image/png", "shot", out var refused));
+ Assert.Contains("text-only", refused);
+ }
+
+ [Fact]
+ public async Task VisionModelAcceptsBoundedImageContentOnly()
+ {
+ var ai = await VisionServiceAsync();
+ Assert.True(ai.TryAttachImage(Png, "image/png", "screenshot of the page", out _));
+ Assert.False(ai.TryAttachImage(ReadOnlyMemory<byte>.Empty, "image/png", "", out var empty));
+ Assert.Contains("empty", empty);
+ Assert.False(ai.TryAttachImage(new byte[AIService.MaxImageInputBytes + 1], "image/png", "", out var big));
+ Assert.Contains("limit", big);
+ Assert.False(ai.TryAttachImage(Png, "application/pdf", "", out var wrong));
+ Assert.Contains("image content", wrong);
+ }
+
+ [Fact]
+ public async Task AttachedImagesAreDeliveredThenRetractedFromHistory()
+ {
+ var ai = await VisionServiceAsync(streaming: false);
+ var seen = new List<string>();
+ ai.SetChatClientFactoryForTests(body => { seen.Add(body); return Reply("looked at it"); });
+
+ Assert.True(ai.TryAttachImage(Png, "image/png", "preview screenshot", out _));
+ await foreach (var _ in ai.ChatStreamAsync("check the page")) { }
+
+ // The model must actually receive the bytes...
+ var delivered = seen.Any(body => body.Contains(Convert.ToBase64String(Png)));
+ Assert.True(delivered, "the image never reached the model: " + string.Join("\n", seen));
+
+ // ...and the image must not linger in history to be re-uploaded on every later request.
+ var history = await ai.GetHistoryAsync();
+ Assert.DoesNotContain(history, message => message.Contents.OfType<DataContent>().Any());
+ }
+
+ private static async Task<AIService> VisionServiceAsync(bool streaming = true)
+ {
+ var config = new MandoCodeConfig();
+ if (!streaming) config.ResponseStreaming = "off";
+ var ai = Create(config);
+ using var vision = Client("{\"capabilities\":[\"vision\"]}");
+ await ai.ValidateModelAsync(vision);
+ Assert.True(ai.SupportsImageInput);
+ return ai;
+ }
+
+ private static AIService Create(MandoCodeConfig config)
+ {
+ var root = new ProjectRootAccessor(Path.GetTempPath());
+ return new AIService(root, config, new TokenTrackingService(), new PlanHandoff(),
+ new SkillLoader(config, root), new McpClientManager(config), new McpApprovalGate(config), new SpinnerService());
+ }
+
+ private static HttpResponseMessage Reply(string text) => new(HttpStatusCode.OK)
+ {
+ Content = new StringContent(JsonSerializer.Serialize(new
+ {
+ model = "m",
+ created_at = "2026-01-01T00:00:00Z",
+ message = new { role = "assistant", content = text },
+ done = true,
+ done_reason = "stop",
+ }), Encoding.UTF8, "application/json"),
+ };
+
+ private static HttpClient Client(string json) =>
+ new(new Handler(_ => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(json) })));
+
+ private sealed class Handler(Func<HttpRequestMessage, Task<HttpResponseMessage>> send) : HttpMessageHandler
+ {
+ protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => send(request);
+ }
+}

tests/MandoCode.Tests/OllamaImageWireFormatProbe.cs

@@ -0,0 +1,48 @@
+using System.Net;
+using System.Text;
+using System.Text.Json;
+using Microsoft.Extensions.AI;
+using OllamaSharp;
+using Xunit;
+
+namespace MandoCode.Tests;
+
+// Probe: does OllamaSharp put DataContent images on the wire where Ollama expects them?
+public sealed class OllamaImageWireFormatProbe
+{
+ [Fact]
+ public async Task ImageContentReachesTheOllamaImagesArray()
+ {
+ string? body = null;
+ var handler = new Capture(request =>
+ {
+ body = request.Content!.ReadAsStringAsync().Result;
+ return new HttpResponseMessage(HttpStatusCode.OK)
+ {
+ Content = new StringContent("{\"model\":\"m\",\"created_at\":\"2026-01-01T00:00:00Z\"," +
+ "\"message\":{\"role\":\"assistant\",\"content\":\"ok\"},\"done\":true}", Encoding.UTF8, "application/json")
+ };
+ });
+ using var http = new HttpClient(handler) { BaseAddress = new Uri("http://localhost:11434") };
+ IChatClient client = new OllamaApiClient(http, "llava");
+
+ var png = new byte[] { 0x89, 0x50, 0x4E, 0x47, 1, 2, 3, 4 };
+ var message = new ChatMessage(ChatRole.User, [
+ new TextContent("What is on screen?"),
+ new DataContent(png, "image/png"),
+ ]);
+ await client.GetResponseAsync([message]);
+
+ Assert.NotNull(body);
+ using var document = JsonDocument.Parse(body!);
+ var first = document.RootElement.GetProperty("messages")[0];
+ Assert.True(first.TryGetProperty("images", out var images), "no images array on the wire: " + body);
+ Assert.Equal(Convert.ToBase64String(png), images[0].GetString());
+ }
+
+ private sealed class Capture(Func<HttpRequestMessage, HttpResponseMessage> send) : HttpMessageHandler
+ {
+ protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) =>
+ Task.FromResult(send(request));
+ }
+}

tests/MandoCode.Tests/PreviewToolPolicyTests.cs

@@ -10,6 +10,7 @@ public class PreviewToolPolicyTests
  [Theory]
  [InlineData("inspect_desktop_preview")]
  [InlineData("observe_desktop_preview")]
+ [InlineData("screenshot_desktop_preview")]
  [InlineData("press_key_desktop_preview")]
  [InlineData("click_desktop_preview")]
  [InlineData("hover_desktop_preview")]
@@ -44,6 +45,7 @@ public async Task BrowserFailureIsReportedAsFailureInTheTranscript()
  [InlineData("inspect_desktop_preview", "{\"ok\":true,\"readyState\":\"complete\"}", true)]
  [InlineData("click_desktop_preview", "{\"ok\":true,\"readyState\":\"complete\"}", true)]
  [InlineData("observe_desktop_preview", "{\"ok\":true,\"readyState\":\"complete\"}", true)]
+ [InlineData("screenshot_desktop_preview", "{\"ok\":true,\"readyState\":\"complete\"}", true)]
  [InlineData("press_key_desktop_preview", "{\"ok\":true,\"readyState\":\"complete\"}", true)]
  [InlineData("wait_for_desktop_preview", "{\"ok\":true,\"readyState\":\"interactive\"}", true)]
  [InlineData("open_desktop_preview", "{\"ok\":true,\"readyState\":\"complete\"}", false)]