From 01555f7044c4b8462249ca7b8356932d2c17ca45 Mon Sep 17 00:00:00 2001 From: Armando Fernandez Date: Sun, 6 Sep 2026 15:51:23 -0700 Subject: [PATCH] Add model vision detection and live-result handling for browser tools Model validation now reads the capability list Ollama reports and records whether the selected model accepts image input. The status is shown in the CLI and Desktop and written into the agent's system instructions, so a text-only model is told not to claim visual inspection and a vision model is told a file path is not an image. Detection is keyed to model and endpoint, so it resets when either changes and a late response about a previous model cannot overwrite newer state. Detecting image support does not deliver images. No screenshots are captured or attached; this is capability reporting only. Live preview tool results are also exempted from the agent result cache. Repeating an identical call is normally deduplicated, which is right for reading a file and wrong for clicking a button, where the second click has to actually happen. Failed preview results now register as failures rather than successful steps, and a fresh DOM observation counts as browser evidence for plan checks. Co-Authored-By: Claude Opus 5 (1M context) --- docs/model-capabilities.md | 26 ++++ src/MandoCode/Components/App.razor | 1 + src/MandoCode/Models/ModelVisionSupport.cs | 45 +++++++ src/MandoCode/Services/Ai/AIService.cs | 39 +++++- .../Services/Ai/AgentFunctionMiddleware.cs | 6 +- .../Ai/FallbackFunctionCallExecutor.cs | 2 +- .../Services/Ai/Planning/PlanToolEvidence.cs | 1 + .../Services/Ai/PreviewToolPolicy.cs | 33 ++++++ .../ModelVisionCapabilitiesTests.cs | 112 ++++++++++++++++++ .../MandoCode.Tests/PreviewToolPolicyTests.cs | 65 ++++++++++ 10 files changed, 324 insertions(+), 6 deletions(-) create mode 100644 docs/model-capabilities.md create mode 100644 src/MandoCode/Models/ModelVisionSupport.cs create mode 100644 src/MandoCode/Services/Ai/PreviewToolPolicy.cs create mode 100644 tests/MandoCode.Tests/ModelVisionCapabilitiesTests.cs create mode 100644 tests/MandoCode.Tests/PreviewToolPolicyTests.cs diff --git a/docs/model-capabilities.md b/docs/model-capabilities.md new file mode 100644 index 0000000..dbbb3cc --- /dev/null +++ b/docs/model-capabilities.md @@ -0,0 +1,26 @@ +# Model image capabilities + +Model validation reads Ollama's `/api/show` response. The CLI and Desktop display +the detected status and include it in the agent's system instructions: + +- **Vision supported:** the capabilities array contains `vision`. +- **Text-only model:** a valid, nonempty capabilities array omits `vision`. +- **Vision capability unknown:** metadata is missing, empty, or malformed. + +Unknown metadata does not fail an otherwise successful model validation. HTTP +and connection failures retain the existing model validation behavior and leave +vision support unknown. No model-name guesses or paid image probes are used. + +Detection runs during model validation and when reinitialization or settings +changes select an uninspected endpoint/model. Results belong to that endpoint and +effective model name. Late responses cannot replace a newer inspection or apply +to a different model. Ordinary settings changes preserve conversation history. + +`AIService.VisionSupport` exposes the three states. `SupportsImageInput` is true +only for confirmed support. This describes the model, not the host application's +ability to deliver images. Browser tools must also supply actual image content +before enabling visual inspection; returning a screenshot path is insufficient. +Text/DOM browser tools need no vision capability. This change does not add browser +inspection tools, screenshot capture, or image message delivery. + +Provider contract: https://docs.ollama.com/api-reference/show-model-details diff --git a/src/MandoCode/Components/App.razor b/src/MandoCode/Components/App.razor index dc172d5..eedf78c 100644 --- a/src/MandoCode/Components/App.razor +++ b/src/MandoCode/Components/App.razor @@ -777,6 +777,7 @@ { _modelError = false; _modelWarning = null; + AnsiConsole.MarkupLine($"[dim]{AI.VisionSupport.Label()}[/]"); } } diff --git a/src/MandoCode/Models/ModelVisionSupport.cs b/src/MandoCode/Models/ModelVisionSupport.cs new file mode 100644 index 0000000..a17b8b4 --- /dev/null +++ b/src/MandoCode/Models/ModelVisionSupport.cs @@ -0,0 +1,45 @@ +using System.Text.Json; + +namespace MandoCode.Models; + +public enum ModelVisionSupport { Unknown, Unsupported, Supported } + +/// Provider-reported image input support, independent of host image delivery. +public static class ModelVisionCapabilities +{ + public static ModelVisionSupport Parse(string modelDetails) + { + try + { + using var document = JsonDocument.Parse(modelDetails); + if (document.RootElement.ValueKind != JsonValueKind.Object || + !document.RootElement.TryGetProperty("capabilities", out var capabilities) || + capabilities.ValueKind != JsonValueKind.Array || capabilities.GetArrayLength() == 0) + return ModelVisionSupport.Unknown; + + var vision = false; + foreach (var capability in capabilities.EnumerateArray()) + { + if (capability.ValueKind != JsonValueKind.String || string.IsNullOrWhiteSpace(capability.GetString())) + return ModelVisionSupport.Unknown; + vision |= string.Equals(capability.GetString(), "vision", StringComparison.OrdinalIgnoreCase); + } + return vision ? ModelVisionSupport.Supported : ModelVisionSupport.Unsupported; + } + catch (JsonException) { return ModelVisionSupport.Unknown; } + } + + public static string Label(this ModelVisionSupport support) => support switch + { + ModelVisionSupport.Supported => "Vision supported", + ModelVisionSupport.Unsupported => "Text-only model", + _ => "Vision capability unknown" + }; + + public static string AgentInstruction(this ModelVisionSupport support) => + $"Model image input capability: {support.Label()}. " + + (support == ModelVisionSupport.Supported + ? "Only describe visual details when an image has actually been supplied as image input. A screenshot path or URL alone is not image input. " + : "Use text and DOM observations for browser checks; do not request image input or claim visual inspection. ") + + "Model capability does not grant browser access. Use only available tools and report any limits on visual verification."; +} diff --git a/src/MandoCode/Services/Ai/AIService.cs b/src/MandoCode/Services/Ai/AIService.cs index 490b3a5..7927a28 100644 --- a/src/MandoCode/Services/Ai/AIService.cs +++ b/src/MandoCode/Services/Ai/AIService.cs @@ -28,6 +28,16 @@ public class AIService // straight to _agent.RunAsync with no conversion — see ExecuteAgentModelCallAsync. private readonly List _chatHistory; private string _systemPrompt; + private string? _visionIdentity; + private ModelVisionSupport _visionSupport; + private long _modelInspectionVersion; + private string ModelIdentity => OllamaSetupHelper.BuildUrl(_config.OllamaEndpoint, "api/show") + "\n" + _config.GetEffectiveModelName(); + + public ModelVisionSupport VisionSupport => _visionIdentity == ModelIdentity + ? _visionSupport : ModelVisionSupport.Unknown; + + // A host must also implement actual image delivery before exposing screenshot tools. + public bool SupportsImageInput => VisionSupport == ModelVisionSupport.Supported; // MAF agent — the live chat path (feat/agent-framework-migration). private AIAgent? _agent; @@ -185,6 +195,7 @@ private void RebuildSystemPrompt() { var skillIndex = SystemPrompts.BuildSkillIndex(_skillLoader.GetAll()); _systemPrompt = SystemPrompts.BuildMandoCodeAssistant(_config.EnableWebSearch, _config.AgentName) + "\n\n" + ShellEnvironment.SystemPromptRules; + _systemPrompt += "\n\n" + VisionSupport.AgentInstruction(); if (!string.IsNullOrEmpty(skillIndex)) { _systemPrompt += "\n\n" + skillIndex; @@ -208,6 +219,7 @@ private void RebuildSystemPrompt() public async Task ReinitializeAsync(MandoCodeConfig config) { _config = config; + if (_visionIdentity != ModelIdentity) await ValidateModelAsync(); RebuildSystemPrompt(); BuildAgent(); await AttachMcpPluginsAsync(); @@ -224,6 +236,7 @@ public async Task ReinitializeAsync(MandoCodeConfig config) public async Task RefreshSettingsAsync(MandoCodeConfig config) { _config = config; + if (_visionIdentity != ModelIdentity) await ValidateModelAsync(); RebuildSystemPrompt(); // The history-preserving path still has the OLD system prompt as message 0 — @@ -665,9 +678,17 @@ private int EffectiveNumCtx() /// public async Task<(bool IsValid, string? ErrorMessage)> ValidateModelAsync() { + using var client = new HttpClient { Timeout = TimeSpan.FromSeconds(10) }; + return await ValidateModelAsync(client); + } + + internal async Task<(bool IsValid, string? ErrorMessage)> ValidateModelAsync(HttpClient client) + { + var identity = ModelIdentity; + var version = Interlocked.Increment(ref _modelInspectionVersion); + var support = ModelVisionSupport.Unknown; try { - using var client = new HttpClient { Timeout = TimeSpan.FromSeconds(10) }; var modelName = _config.GetEffectiveModelName(); // Check if model exists and get its info @@ -681,13 +702,27 @@ private int EffectiveNumCtx() return (false, $"Model '{modelName}' not found. Run: ollama pull {modelName}"); } - // Model exists and is available — Ollama handles tool support at the API level + // Successful validation must stay successful when older servers omit metadata. + support = ModelVisionCapabilities.Parse(await response.Content.ReadAsStringAsync()); return (true, null); } catch (Exception ex) { return (false, $"Could not validate model: {ex.Message}"); } + finally + { + // A late response from a previous model must never overwrite the current state. + if (identity == ModelIdentity && version == Interlocked.Read(ref _modelInspectionVersion)) + { + _visionIdentity = identity; + _visionSupport = support; + RebuildSystemPrompt(); + if (_chatHistory.Count > 0 && _chatHistory[0].Role == ChatRole.System) + _chatHistory[0] = new ChatMessage(ChatRole.System, _systemPrompt); + BuildAgent(); + } + } } /// diff --git a/src/MandoCode/Services/Ai/AgentFunctionMiddleware.cs b/src/MandoCode/Services/Ai/AgentFunctionMiddleware.cs index f599cb2..3954b4f 100644 --- a/src/MandoCode/Services/Ai/AgentFunctionMiddleware.cs +++ b/src/MandoCode/Services/Ai/AgentFunctionMiddleware.cs @@ -216,7 +216,7 @@ public InvocationScope BeginScope() var deduplicationWindow = isWriteOperation ? _writeDeduplicationWindow : _readDeduplicationWindow; var callKey = CreateCallKey(functionName, context.Arguments, isWriteOperation); - if (_recentCalls.TryGetValue(callKey, out var cached) && + if (!PreviewToolPolicy.IsLiveTool(functionName) && _recentCalls.TryGetValue(callKey, out var cached) && DateTime.UtcNow - cached.Time < deduplicationWindow) { return cached.Result ?? "Operation already completed."; @@ -440,10 +440,10 @@ public InvocationScope BeginScope() deliveredResult = resultStr; } - _recentCalls[callKey] = (DateTime.UtcNow, deliveredResult); + if (!PreviewToolPolicy.IsLiveTool(functionName)) _recentCalls[callKey] = (DateTime.UtcNow, deliveredResult); CleanupOldEntries(); - var isError = resultStr.StartsWith("Error:", StringComparison.OrdinalIgnoreCase); + var isError = resultStr.StartsWith("Error:", StringComparison.OrdinalIgnoreCase) || PreviewToolPolicy.IsFailure(functionName, resultStr); UpdateScopeForCompletedCall(context, functionName, resultStr, isError); CompleteWith(functionName, resultStr, success: !isError); diff --git a/src/MandoCode/Services/Ai/FallbackFunctionCallExecutor.cs b/src/MandoCode/Services/Ai/FallbackFunctionCallExecutor.cs index dee527a..8b03e4a 100644 --- a/src/MandoCode/Services/Ai/FallbackFunctionCallExecutor.cs +++ b/src/MandoCode/Services/Ai/FallbackFunctionCallExecutor.cs @@ -456,7 +456,7 @@ public static string ToSnakeCase(string input) { FunctionName = functionName, Result = resultString.Length > 200 ? resultString[..200] + "..." : resultString, - Success = true + Success = !PreviewToolPolicy.IsFailure(functionName, resultString) }); return resultString; diff --git a/src/MandoCode/Services/Ai/Planning/PlanToolEvidence.cs b/src/MandoCode/Services/Ai/Planning/PlanToolEvidence.cs index 7047457..cb2e272 100644 --- a/src/MandoCode/Services/Ai/Planning/PlanToolEvidence.cs +++ b/src/MandoCode/Services/Ai/Planning/PlanToolEvidence.cs @@ -72,6 +72,7 @@ public static IReadOnlyDictionary SnapshotFileVersions( needsExecutionCheck |= extension is not (".md" or ".txt" or ".rst"); } else if (call.Name is "execute_command" || + PreviewToolPolicy.IsFreshObservation(call.Name, result.Result?.ToString() ?? "") || call.Name.Contains("test", StringComparison.OrdinalIgnoreCase) || call.Name.Contains("browser", StringComparison.OrdinalIgnoreCase)) { diff --git a/src/MandoCode/Services/Ai/PreviewToolPolicy.cs b/src/MandoCode/Services/Ai/PreviewToolPolicy.cs new file mode 100644 index 0000000..a6cea3b --- /dev/null +++ b/src/MandoCode/Services/Ai/PreviewToolPolicy.cs @@ -0,0 +1,33 @@ +using System.Text.Json; + +namespace MandoCode.Services; + +/// Live host preview operations cannot reuse cached observations or action results. +public static class PreviewToolPolicy +{ + public static bool IsLiveTool(string name) => name is + "open_desktop_preview" or "refresh_desktop_preview" or "inspect_desktop_preview" or + "observe_desktop_preview" or "click_desktop_preview" or "press_key_desktop_preview" or + "hover_desktop_preview" or "fill_desktop_preview" or "select_desktop_preview" or + "scroll_desktop_preview" or "wait_for_desktop_preview"; + + public static bool IsFailure(string name, string result) => IsLiveTool(name) && !HasSuccessfulResult(result); + + public static bool IsFreshObservation(string name, string result) => + IsLiveTool(name) && name is not ("open_desktop_preview" or "refresh_desktop_preview") && + HasSuccessfulResult(result, requireDocument: true); + + private static bool HasSuccessfulResult(string result, bool requireDocument = false) + { + try + { + using var document = JsonDocument.Parse(result); + var root = document.RootElement; + return root.ValueKind == JsonValueKind.Object && root.TryGetProperty("ok", out var ok) && + ok.ValueKind == JsonValueKind.True && (!requireDocument || + root.TryGetProperty("readyState", out var ready) && ready.ValueKind == JsonValueKind.String && + ready.GetString() is "interactive" or "complete"); + } + catch (JsonException) { return false; } + } +} diff --git a/tests/MandoCode.Tests/ModelVisionCapabilitiesTests.cs b/tests/MandoCode.Tests/ModelVisionCapabilitiesTests.cs new file mode 100644 index 0000000..c1dae98 --- /dev/null +++ b/tests/MandoCode.Tests/ModelVisionCapabilitiesTests.cs @@ -0,0 +1,112 @@ +using System.Net; +using MandoCode.Models; +using MandoCode.Services; +using Xunit; + +namespace MandoCode.Tests; + +public class ModelVisionCapabilitiesTests +{ + [Theory] + [InlineData("{\"capabilities\":[\"completion\",\"vision\",\"tools\"]}", ModelVisionSupport.Supported)] + [InlineData("{\"capabilities\":[\"VISION\"]}", ModelVisionSupport.Supported)] + [InlineData("{\"capabilities\":[\"completion\",\"tools\"]}", ModelVisionSupport.Unsupported)] + [InlineData("{}", ModelVisionSupport.Unknown)] + [InlineData("{\"capabilities\":[]}", ModelVisionSupport.Unknown)] + [InlineData("{\"capabilities\":null}", ModelVisionSupport.Unknown)] + [InlineData("{\"capabilities\":\"vision\"}", ModelVisionSupport.Unknown)] + [InlineData("{\"capabilities\":[\"vision\",42]}", ModelVisionSupport.Unknown)] + [InlineData("{\"capabilities\":[\"\"]}", ModelVisionSupport.Unknown)] + [InlineData("not json", ModelVisionSupport.Unknown)] + [InlineData("[]", ModelVisionSupport.Unknown)] + public void ReadsExplicitCapabilitiesOnly(string json, ModelVisionSupport expected) => + Assert.Equal(expected, ModelVisionCapabilities.Parse(json)); + + [Fact] + public async Task ValidationUpdatesAgentAndPreservesConversation_UnknownDoesNotBlock() + { + var config = new MandoCodeConfig(); + var ai = Create(config); + ai.AppendUserNote("Keep my conversation"); + using var client = Client("{\"capabilities\":[\"vision\"]}"); + Assert.True((await ai.ValidateModelAsync(client)).IsValid); + Assert.True(ai.SupportsImageInput); + var history = await ai.GetHistoryAsync(); + Assert.Contains("Vision supported", history[0].Text); + Assert.Contains(history, message => message.Text == "Keep my conversation"); + + using var legacy = Client("not json"); + Assert.True((await ai.ValidateModelAsync(legacy)).IsValid); + Assert.Equal(ModelVisionSupport.Unknown, ai.VisionSupport); + Assert.False(ai.SupportsImageInput); + } + + [Fact] + public async Task ModelOrEndpointMutationImmediatelyInvalidatesVision() + { + var config = new MandoCodeConfig(); + var ai = Create(config); + using var client = Client("{\"capabilities\":[\"vision\"]}"); + await ai.ValidateModelAsync(client); + config.OllamaEndpoint = "http://another-server:11434"; + Assert.Equal(ModelVisionSupport.Unknown, ai.VisionSupport); + await ai.ValidateModelAsync(client); + Assert.True(ai.SupportsImageInput); + config.ModelName = "different-model"; + Assert.Equal(ModelVisionSupport.Unknown, ai.VisionSupport); + } + + [Fact] + public async Task LateResponseCannotOverwriteNewerInspection() + { + var ai = Create(new MandoCodeConfig()); + var pending = new TaskCompletionSource(); + using var slow = new HttpClient(new Handler(_ => pending.Task)); + var oldCheck = ai.ValidateModelAsync(slow); + using var current = Client("{\"capabilities\":[\"completion\"]}"); + await ai.ValidateModelAsync(current); + pending.SetResult(Response("{\"capabilities\":[\"vision\"]}")); + await oldCheck; + Assert.Equal(ModelVisionSupport.Unsupported, ai.VisionSupport); + } + + [Fact] + public async Task LateResponseFromPreviousModelIsIgnored() + { + var config = new MandoCodeConfig(); + var ai = Create(config); + var pending = new TaskCompletionSource(); + using var slow = new HttpClient(new Handler(_ => pending.Task)); + var oldCheck = ai.ValidateModelAsync(slow); + config.ModelName = "replacement-model"; + pending.SetResult(Response("{\"capabilities\":[\"vision\"]}")); + await oldCheck; + Assert.Equal(ModelVisionSupport.Unknown, ai.VisionSupport); + Assert.False(ai.SupportsImageInput); + } + + [Fact] + public async Task FailedValidationClearsPreviouslySupportedVision() + { + var ai = Create(new MandoCodeConfig()); + using var client = Client("{\"capabilities\":[\"vision\"]}"); + await ai.ValidateModelAsync(client); + using var failed = new HttpClient(new Handler(_ => Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound)))); + Assert.False((await ai.ValidateModelAsync(failed)).IsValid); + Assert.Equal(ModelVisionSupport.Unknown, ai.VisionSupport); + } + + private static AIService Create(MandoCodeConfig config) + { + var root = new ProjectRootAccessor(Path.GetTempPath()); + return new AIService(root, config, new TokenTrackingService(), new PlanHandoff(), + new SkillLoader(config, root), new McpClientManager(config), new McpApprovalGate(config), new SpinnerService()); + } + + private static HttpResponseMessage Response(string json) => new(HttpStatusCode.OK) { Content = new StringContent(json) }; + private static HttpClient Client(string json) => new(new Handler(_ => Task.FromResult(Response(json)))); + private sealed class Handler(Func> send) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => send(request); + } +} diff --git a/tests/MandoCode.Tests/PreviewToolPolicyTests.cs b/tests/MandoCode.Tests/PreviewToolPolicyTests.cs new file mode 100644 index 0000000..7c7351e --- /dev/null +++ b/tests/MandoCode.Tests/PreviewToolPolicyTests.cs @@ -0,0 +1,65 @@ +using MandoCode.Models; +using MandoCode.Services; +using Microsoft.Extensions.AI; +using Xunit; + +namespace MandoCode.Tests; + +public class PreviewToolPolicyTests +{ + [Theory] + [InlineData("inspect_desktop_preview")] + [InlineData("observe_desktop_preview")] + [InlineData("press_key_desktop_preview")] + [InlineData("click_desktop_preview")] + [InlineData("hover_desktop_preview")] + [InlineData("fill_desktop_preview")] + [InlineData("select_desktop_preview")] + [InlineData("scroll_desktop_preview")] + [InlineData("wait_for_desktop_preview")] + [InlineData("open_desktop_preview")] + [InlineData("refresh_desktop_preview")] + public async Task LiveOperationsAreNeverReplayedFromCache(string name) + { + var middleware = new AgentFunctionMiddleware(5); + var calls = 0; + var tool = AIFunctionFactory.Create(() => { calls++; return "{\"ok\":true}"; }, new AIFunctionFactoryOptions { Name = name }); + await AgentMiddlewareTestHelpers.InvokeAsync(middleware, tool); + await AgentMiddlewareTestHelpers.InvokeAsync(middleware, tool); + Assert.Equal(2, calls); + } + + [Fact] + public async Task BrowserFailureIsReportedAsFailureInTheTranscript() + { + var middleware = new AgentFunctionMiddleware(5); + FunctionExecutionResult? completed = null; + middleware.OnFunctionCompleted += result => completed = result; + var tool = AIFunctionFactory.Create(() => "{\"ok\":false,\"error\":\"selector is ambiguous\"}", new AIFunctionFactoryOptions { Name = "click_desktop_preview" }); + await AgentMiddlewareTestHelpers.InvokeAsync(middleware, tool); + Assert.False(completed!.Success); + } + + [Theory] + [InlineData("inspect_desktop_preview", "{\"ok\":true,\"readyState\":\"complete\"}", true)] + [InlineData("click_desktop_preview", "{\"ok\":true,\"readyState\":\"complete\"}", true)] + [InlineData("observe_desktop_preview", "{\"ok\":true,\"readyState\":\"complete\"}", true)] + [InlineData("press_key_desktop_preview", "{\"ok\":true,\"readyState\":\"complete\"}", true)] + [InlineData("wait_for_desktop_preview", "{\"ok\":true,\"readyState\":\"interactive\"}", true)] + [InlineData("open_desktop_preview", "{\"ok\":true,\"readyState\":\"complete\"}", false)] + [InlineData("refresh_desktop_preview", "{\"ok\":true,\"readyState\":\"complete\"}", false)] + [InlineData("inspect_desktop_preview", "{\"ok\":false}", false)] + [InlineData("inspect_desktop_preview", "{\"ok\":true,\"readyState\":\"loading\"}", false)] + [InlineData("inspect_desktop_preview", "{\"ok\":true}", false)] + [InlineData("inspect_desktop_preview", "page unavailable", false)] + public void FreshnessRequiresAnActualBrowserObservation(string name, string result, bool fresh) + { + ChatMessage[] history = [ + new(ChatRole.Assistant, [new FunctionCallContent("edit", "edit_file", new Dictionary { ["path"] = "index.html" })]), + new(ChatRole.Tool, [new FunctionResultContent("edit", "edited")]), + new(ChatRole.Assistant, [new FunctionCallContent("browser", name)]), + new(ChatRole.Tool, [new FunctionResultContent("browser", result)]) + ]; + Assert.Equal(fresh, PlanToolEvidence.AssessFreshness(history) == null); + } +}