diff --git a/docs/model-capabilities.md b/docs/model-capabilities.md index dbbb3cc..af74dd0 100644 --- a/docs/model-capabilities.md +++ b/docs/model-capabilities.md @@ -24,3 +24,17 @@ Text/DOM browser tools need no vision capability. This change does not add brows inspection tools, screenshot capture, or image message delivery. Provider contract: https://docs.ollama.com/api-reference/show-model-details + +## Image delivery + +Detecting image support is separate from delivering an image. A host captures an image +and calls `AIService.TryAttachImage`, which refuses unless the model reports vision +support, the content is an image, and it is within `MaxImageInputBytes`. + +A tool result is text, so an image cannot ride back inside one. A queued image is added +to history as a real user message after the current turn, and the turn is extended so the +model looks at it — bounded by `MaxImageDeliveriesPerTurn`. + +Delivered images are retracted from history when the user turn ends. An image is evidence +for the turn that captured it; keeping it would re-upload megabytes on every later request +and crowd out the context the model needs. The model's written conclusion is what persists. diff --git a/src/MandoCode/Services/Ai/AIService.cs b/src/MandoCode/Services/Ai/AIService.cs index 7927a28..78a71c6 100644 --- a/src/MandoCode/Services/Ai/AIService.cs +++ b/src/MandoCode/Services/Ai/AIService.cs @@ -36,9 +36,90 @@ public class AIService public ModelVisionSupport VisionSupport => _visionIdentity == ModelIdentity ? _visionSupport : ModelVisionSupport.Unknown; - // A host must also implement actual image delivery before exposing screenshot tools. public bool SupportsImageInput => VisionSupport == ModelVisionSupport.Supported; + /// Largest single host-supplied image, before base64 expansion. + public const int MaxImageInputBytes = 4 * 1024 * 1024; + /// How many times one user turn may be extended so the model can look at a new image. + public const int MaxImageDeliveriesPerTurn = 3; + + private readonly List _pendingImages = []; + private int _imageDeliveriesThisTurn; + + private Func? _chatTransportForTests; + + /// Test seam: answers model calls from the request body, so delivery can be asserted without a live daemon. + internal void SetChatClientFactoryForTests(Func respond) + { + _chatTransportForTests = respond; + BuildAgent(); + } + + private sealed class StubTransport(Func respond) : HttpMessageHandler + { + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => + respond(request.Content == null ? "" : await request.Content.ReadAsStringAsync(cancellationToken)); + } + + /// + /// Queues a host-captured image for the model to actually look at. Delivery is deliberately + /// separate from capture: a tool result is text, so an image can only reach the model as + /// message content on a following turn. Refused outright unless the model accepts image + /// input, so a text-only model is never handed something it will silently ignore. + /// + public bool TryAttachImage(ReadOnlyMemory bytes, string mediaType, string caption, out string error) + { + if (!SupportsImageInput) + { + error = VisionSupport == ModelVisionSupport.Unknown + ? "This model's image support is unknown, so an image cannot be delivered. Use text and DOM observations." + : "This model is text-only and cannot receive images. Use text and DOM observations."; + return false; + } + if (bytes.IsEmpty) { error = "The captured image was empty."; return false; } + if (bytes.Length > MaxImageInputBytes) + { + error = $"The image is {bytes.Length / 1024} KB, over the {MaxImageInputBytes / 1024} KB limit."; + return false; + } + if (!mediaType.StartsWith("image/", StringComparison.OrdinalIgnoreCase)) + { + error = "Only image content can be attached."; + return false; + } + lock (_pendingImages) + { + if (_pendingImages.Count >= 8) { error = "Too many images are already queued for this turn."; return false; } + if (!string.IsNullOrWhiteSpace(caption)) _pendingImages.Add(new TextContent(caption)); + _pendingImages.Add(new DataContent(bytes, mediaType)); + } + error = ""; + return true; + } + + /// + /// Moves queued images into history as a real user message so the model sees them on the next + /// turn. Returns the message so the caller can retract it afterward: an image is evidence for + /// the turn that asked for it, not permanent context that re-uploads on every later request. + /// + private async Task TakePendingImageMessageAsync() + { + List contents; + lock (_pendingImages) + { + if (_pendingImages.Count == 0) return null; + contents = [.. _pendingImages]; + _pendingImages.Clear(); + } + contents.Insert(0, new TextContent( + "Host-captured image input follows. Describe only what is actually visible in it.")); + var message = new ChatMessage(ChatRole.User, contents); + await _historyLock.WaitAsync(); + try { _chatHistory.Add(message); } + finally { _historyLock.Release(); } + return message; + } + // MAF agent — the live chat path (feat/agent-framework-migration). private AIAgent? _agent; @@ -350,7 +431,8 @@ private void BuildAgent() // as a bogus transport error on slow local generations. The old client (like the old // agent it served) is left for GC rather than disposed — a rebuild can race a call still // in flight on the discarded agent. - var ollamaHttpClient = new HttpClient(new NumCtxHttpHandler(EffectiveNumCtx)) + var ollamaHttpClient = new HttpClient( + _chatTransportForTests is { } stub ? new StubTransport(stub) : new NumCtxHttpHandler(EffectiveNumCtx)) { BaseAddress = new Uri(_config.OllamaEndpoint), Timeout = System.Threading.Timeout.InfiniteTimeSpan @@ -765,6 +847,10 @@ public async IAsyncEnumerable ChatStreamWithHostInstructionAsync( } finally { _historyLock.Release(); } + // Not cleared here: an image may be attached before the turn (a pasted screenshot) or + // during it (a preview capture). Both are real evidence and both must reach the model. + _imageDeliveriesThisTurn = 0; + var deliveredImages = new List(); try { int continuations = 0; @@ -773,6 +859,13 @@ public async IAsyncEnumerable ChatStreamWithHostInstructionAsync( var (response, needsContinuation) = await RunOneChatTurnAsync(continuations, cancellationToken); yield return response; + // An image captured during the turn can only be looked at on a following one. + if (await TakePendingImageMessageAsync() is { } imageMessage) + { + deliveredImages.Add(imageMessage); + if (_imageDeliveriesThisTurn++ < MaxImageDeliveriesPerTurn) needsContinuation = true; + } + if (!needsContinuation) break; @@ -781,10 +874,15 @@ public async IAsyncEnumerable ChatStreamWithHostInstructionAsync( } finally { - if (hostMessage != null) + var transient = deliveredImages; + if (hostMessage != null) transient = [hostMessage, .. deliveredImages]; + if (transient.Count > 0) { await _historyLock.WaitAsync(CancellationToken.None); - try { _chatHistory.Remove(hostMessage); } + // Images are evidence for the turn that captured them. Leaving them in history + // would re-upload megabytes on every later request and crowd out the context the + // model needs; the model's written conclusion about the image is what persists. + try { foreach (var message in transient) _chatHistory.Remove(message); } finally { _historyLock.Release(); } } } diff --git a/src/MandoCode/Services/Ai/PreviewToolPolicy.cs b/src/MandoCode/Services/Ai/PreviewToolPolicy.cs index a6cea3b..3c7bc4c 100644 --- a/src/MandoCode/Services/Ai/PreviewToolPolicy.cs +++ b/src/MandoCode/Services/Ai/PreviewToolPolicy.cs @@ -9,7 +9,7 @@ public static bool IsLiveTool(string name) => name is "open_desktop_preview" or "refresh_desktop_preview" or "inspect_desktop_preview" or "observe_desktop_preview" or "click_desktop_preview" or "press_key_desktop_preview" or "hover_desktop_preview" or "fill_desktop_preview" or "select_desktop_preview" or - "scroll_desktop_preview" or "wait_for_desktop_preview"; + "scroll_desktop_preview" or "wait_for_desktop_preview" or "screenshot_desktop_preview"; public static bool IsFailure(string name, string result) => IsLiveTool(name) && !HasSuccessfulResult(result); diff --git a/tests/MandoCode.Tests/ImageInputDeliveryTests.cs b/tests/MandoCode.Tests/ImageInputDeliveryTests.cs new file mode 100644 index 0000000..b7d68b5 --- /dev/null +++ b/tests/MandoCode.Tests/ImageInputDeliveryTests.cs @@ -0,0 +1,98 @@ +using System.Net; +using System.Text; +using System.Text.Json; +using MandoCode.Models; +using MandoCode.Services; +using Microsoft.Extensions.AI; +using OllamaSharp; +using Xunit; + +namespace MandoCode.Tests; + +public class ImageInputDeliveryTests +{ + private static readonly byte[] Png = [0x89, 0x50, 0x4E, 0x47, 1, 2, 3, 4]; + + [Fact] + public async Task TextOnlyAndUnknownModelsRefuseImages() + { + var ai = Create(new MandoCodeConfig()); + Assert.False(ai.TryAttachImage(Png, "image/png", "shot", out var unknown)); + Assert.Contains("unknown", unknown); + + using var textOnly = Client("{\"capabilities\":[\"tools\"]}"); + await ai.ValidateModelAsync(textOnly); + Assert.False(ai.TryAttachImage(Png, "image/png", "shot", out var refused)); + Assert.Contains("text-only", refused); + } + + [Fact] + public async Task VisionModelAcceptsBoundedImageContentOnly() + { + var ai = await VisionServiceAsync(); + Assert.True(ai.TryAttachImage(Png, "image/png", "screenshot of the page", out _)); + Assert.False(ai.TryAttachImage(ReadOnlyMemory.Empty, "image/png", "", out var empty)); + Assert.Contains("empty", empty); + Assert.False(ai.TryAttachImage(new byte[AIService.MaxImageInputBytes + 1], "image/png", "", out var big)); + Assert.Contains("limit", big); + Assert.False(ai.TryAttachImage(Png, "application/pdf", "", out var wrong)); + Assert.Contains("image content", wrong); + } + + [Fact] + public async Task AttachedImagesAreDeliveredThenRetractedFromHistory() + { + var ai = await VisionServiceAsync(streaming: false); + var seen = new List(); + ai.SetChatClientFactoryForTests(body => { seen.Add(body); return Reply("looked at it"); }); + + Assert.True(ai.TryAttachImage(Png, "image/png", "preview screenshot", out _)); + await foreach (var _ in ai.ChatStreamAsync("check the page")) { } + + // The model must actually receive the bytes... + var delivered = seen.Any(body => body.Contains(Convert.ToBase64String(Png))); + Assert.True(delivered, "the image never reached the model: " + string.Join("\n", seen)); + + // ...and the image must not linger in history to be re-uploaded on every later request. + var history = await ai.GetHistoryAsync(); + Assert.DoesNotContain(history, message => message.Contents.OfType().Any()); + } + + private static async Task VisionServiceAsync(bool streaming = true) + { + var config = new MandoCodeConfig(); + if (!streaming) config.ResponseStreaming = "off"; + var ai = Create(config); + using var vision = Client("{\"capabilities\":[\"vision\"]}"); + await ai.ValidateModelAsync(vision); + Assert.True(ai.SupportsImageInput); + return ai; + } + + private static AIService Create(MandoCodeConfig config) + { + var root = new ProjectRootAccessor(Path.GetTempPath()); + return new AIService(root, config, new TokenTrackingService(), new PlanHandoff(), + new SkillLoader(config, root), new McpClientManager(config), new McpApprovalGate(config), new SpinnerService()); + } + + private static HttpResponseMessage Reply(string text) => new(HttpStatusCode.OK) + { + Content = new StringContent(JsonSerializer.Serialize(new + { + model = "m", + created_at = "2026-01-01T00:00:00Z", + message = new { role = "assistant", content = text }, + done = true, + done_reason = "stop", + }), Encoding.UTF8, "application/json"), + }; + + private static HttpClient Client(string json) => + new(new Handler(_ => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(json) }))); + + private sealed class Handler(Func> send) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => send(request); + } +} diff --git a/tests/MandoCode.Tests/OllamaImageWireFormatProbe.cs b/tests/MandoCode.Tests/OllamaImageWireFormatProbe.cs new file mode 100644 index 0000000..16107ed --- /dev/null +++ b/tests/MandoCode.Tests/OllamaImageWireFormatProbe.cs @@ -0,0 +1,48 @@ +using System.Net; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.AI; +using OllamaSharp; +using Xunit; + +namespace MandoCode.Tests; + +// Probe: does OllamaSharp put DataContent images on the wire where Ollama expects them? +public sealed class OllamaImageWireFormatProbe +{ + [Fact] + public async Task ImageContentReachesTheOllamaImagesArray() + { + string? body = null; + var handler = new Capture(request => + { + body = request.Content!.ReadAsStringAsync().Result; + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"model\":\"m\",\"created_at\":\"2026-01-01T00:00:00Z\"," + + "\"message\":{\"role\":\"assistant\",\"content\":\"ok\"},\"done\":true}", Encoding.UTF8, "application/json") + }; + }); + using var http = new HttpClient(handler) { BaseAddress = new Uri("http://localhost:11434") }; + IChatClient client = new OllamaApiClient(http, "llava"); + + var png = new byte[] { 0x89, 0x50, 0x4E, 0x47, 1, 2, 3, 4 }; + var message = new ChatMessage(ChatRole.User, [ + new TextContent("What is on screen?"), + new DataContent(png, "image/png"), + ]); + await client.GetResponseAsync([message]); + + Assert.NotNull(body); + using var document = JsonDocument.Parse(body!); + var first = document.RootElement.GetProperty("messages")[0]; + Assert.True(first.TryGetProperty("images", out var images), "no images array on the wire: " + body); + Assert.Equal(Convert.ToBase64String(png), images[0].GetString()); + } + + private sealed class Capture(Func send) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => + Task.FromResult(send(request)); + } +} diff --git a/tests/MandoCode.Tests/PreviewToolPolicyTests.cs b/tests/MandoCode.Tests/PreviewToolPolicyTests.cs index 7c7351e..afd111f 100644 --- a/tests/MandoCode.Tests/PreviewToolPolicyTests.cs +++ b/tests/MandoCode.Tests/PreviewToolPolicyTests.cs @@ -10,6 +10,7 @@ public class PreviewToolPolicyTests [Theory] [InlineData("inspect_desktop_preview")] [InlineData("observe_desktop_preview")] + [InlineData("screenshot_desktop_preview")] [InlineData("press_key_desktop_preview")] [InlineData("click_desktop_preview")] [InlineData("hover_desktop_preview")] @@ -44,6 +45,7 @@ public async Task BrowserFailureIsReportedAsFailureInTheTranscript() [InlineData("inspect_desktop_preview", "{\"ok\":true,\"readyState\":\"complete\"}", true)] [InlineData("click_desktop_preview", "{\"ok\":true,\"readyState\":\"complete\"}", true)] [InlineData("observe_desktop_preview", "{\"ok\":true,\"readyState\":\"complete\"}", true)] + [InlineData("screenshot_desktop_preview", "{\"ok\":true,\"readyState\":\"complete\"}", true)] [InlineData("press_key_desktop_preview", "{\"ok\":true,\"readyState\":\"complete\"}", true)] [InlineData("wait_for_desktop_preview", "{\"ok\":true,\"readyState\":\"interactive\"}", true)] [InlineData("open_desktop_preview", "{\"ok\":true,\"readyState\":\"complete\"}", false)]