diff --git a/docs/model-capabilities.md b/docs/model-capabilities.md new file mode 100644 index 0000000..dbbb3cc --- /dev/null +++ b/docs/model-capabilities.md @@ -0,0 +1,26 @@ +# Model image capabilities + +Model validation reads Ollama's `/api/show` response. The CLI and Desktop display +the detected status and include it in the agent's system instructions: + +- **Vision supported:** the capabilities array contains `vision`. +- **Text-only model:** a valid, nonempty capabilities array omits `vision`. +- **Vision capability unknown:** metadata is missing, empty, or malformed. + +Unknown metadata does not fail an otherwise successful model validation. HTTP +and connection failures retain the existing model validation behavior and leave +vision support unknown. No model-name guesses or paid image probes are used. + +Detection runs during model validation and when reinitialization or settings +changes select an uninspected endpoint/model. Results belong to that endpoint and +effective model name. Late responses cannot replace a newer inspection or apply +to a different model. Ordinary settings changes preserve conversation history. + +`AIService.VisionSupport` exposes the three states. `SupportsImageInput` is true +only for confirmed support. This describes the model, not the host application's +ability to deliver images. Browser tools must also supply actual image content +before enabling visual inspection; returning a screenshot path is insufficient. +Text/DOM browser tools need no vision capability. This change does not add browser +inspection tools, screenshot capture, or image message delivery. + +Provider contract: https://docs.ollama.com/api-reference/show-model-details diff --git a/src/MandoCode/Components/App.razor b/src/MandoCode/Components/App.razor index dc172d5..eedf78c 100644 --- a/src/MandoCode/Components/App.razor +++ b/src/MandoCode/Components/App.razor @@ -777,6 +777,7 @@ { _modelError = false; _modelWarning = null; + AnsiConsole.MarkupLine($"[dim]{AI.VisionSupport.Label()}[/]"); } } diff --git a/src/MandoCode/Models/ModelVisionSupport.cs b/src/MandoCode/Models/ModelVisionSupport.cs new file mode 100644 index 0000000..a17b8b4 --- /dev/null +++ b/src/MandoCode/Models/ModelVisionSupport.cs @@ -0,0 +1,45 @@ +using System.Text.Json; + +namespace MandoCode.Models; + +public enum ModelVisionSupport { Unknown, Unsupported, Supported } + +/// Provider-reported image input support, independent of host image delivery. +public static class ModelVisionCapabilities +{ + public static ModelVisionSupport Parse(string modelDetails) + { + try + { + using var document = JsonDocument.Parse(modelDetails); + if (document.RootElement.ValueKind != JsonValueKind.Object || + !document.RootElement.TryGetProperty("capabilities", out var capabilities) || + capabilities.ValueKind != JsonValueKind.Array || capabilities.GetArrayLength() == 0) + return ModelVisionSupport.Unknown; + + var vision = false; + foreach (var capability in capabilities.EnumerateArray()) + { + if (capability.ValueKind != JsonValueKind.String || string.IsNullOrWhiteSpace(capability.GetString())) + return ModelVisionSupport.Unknown; + vision |= string.Equals(capability.GetString(), "vision", StringComparison.OrdinalIgnoreCase); + } + return vision ? ModelVisionSupport.Supported : ModelVisionSupport.Unsupported; + } + catch (JsonException) { return ModelVisionSupport.Unknown; } + } + + public static string Label(this ModelVisionSupport support) => support switch + { + ModelVisionSupport.Supported => "Vision supported", + ModelVisionSupport.Unsupported => "Text-only model", + _ => "Vision capability unknown" + }; + + public static string AgentInstruction(this ModelVisionSupport support) => + $"Model image input capability: {support.Label()}. " + + (support == ModelVisionSupport.Supported + ? "Only describe visual details when an image has actually been supplied as image input. A screenshot path or URL alone is not image input. " + : "Use text and DOM observations for browser checks; do not request image input or claim visual inspection. ") + + "Model capability does not grant browser access. Use only available tools and report any limits on visual verification."; +} diff --git a/src/MandoCode/Services/Ai/AIService.cs b/src/MandoCode/Services/Ai/AIService.cs index 490b3a5..7927a28 100644 --- a/src/MandoCode/Services/Ai/AIService.cs +++ b/src/MandoCode/Services/Ai/AIService.cs @@ -28,6 +28,16 @@ public class AIService // straight to _agent.RunAsync with no conversion — see ExecuteAgentModelCallAsync. private readonly List _chatHistory; private string _systemPrompt; + private string? _visionIdentity; + private ModelVisionSupport _visionSupport; + private long _modelInspectionVersion; + private string ModelIdentity => OllamaSetupHelper.BuildUrl(_config.OllamaEndpoint, "api/show") + "\n" + _config.GetEffectiveModelName(); + + public ModelVisionSupport VisionSupport => _visionIdentity == ModelIdentity + ? _visionSupport : ModelVisionSupport.Unknown; + + // A host must also implement actual image delivery before exposing screenshot tools. + public bool SupportsImageInput => VisionSupport == ModelVisionSupport.Supported; // MAF agent — the live chat path (feat/agent-framework-migration). private AIAgent? _agent; @@ -185,6 +195,7 @@ private void RebuildSystemPrompt() { var skillIndex = SystemPrompts.BuildSkillIndex(_skillLoader.GetAll()); _systemPrompt = SystemPrompts.BuildMandoCodeAssistant(_config.EnableWebSearch, _config.AgentName) + "\n\n" + ShellEnvironment.SystemPromptRules; + _systemPrompt += "\n\n" + VisionSupport.AgentInstruction(); if (!string.IsNullOrEmpty(skillIndex)) { _systemPrompt += "\n\n" + skillIndex; @@ -208,6 +219,7 @@ private void RebuildSystemPrompt() public async Task ReinitializeAsync(MandoCodeConfig config) { _config = config; + if (_visionIdentity != ModelIdentity) await ValidateModelAsync(); RebuildSystemPrompt(); BuildAgent(); await AttachMcpPluginsAsync(); @@ -224,6 +236,7 @@ public async Task ReinitializeAsync(MandoCodeConfig config) public async Task RefreshSettingsAsync(MandoCodeConfig config) { _config = config; + if (_visionIdentity != ModelIdentity) await ValidateModelAsync(); RebuildSystemPrompt(); // The history-preserving path still has the OLD system prompt as message 0 — @@ -665,9 +678,17 @@ private int EffectiveNumCtx() /// public async Task<(bool IsValid, string? ErrorMessage)> ValidateModelAsync() { + using var client = new HttpClient { Timeout = TimeSpan.FromSeconds(10) }; + return await ValidateModelAsync(client); + } + + internal async Task<(bool IsValid, string? ErrorMessage)> ValidateModelAsync(HttpClient client) + { + var identity = ModelIdentity; + var version = Interlocked.Increment(ref _modelInspectionVersion); + var support = ModelVisionSupport.Unknown; try { - using var client = new HttpClient { Timeout = TimeSpan.FromSeconds(10) }; var modelName = _config.GetEffectiveModelName(); // Check if model exists and get its info @@ -681,13 +702,27 @@ private int EffectiveNumCtx() return (false, $"Model '{modelName}' not found. Run: ollama pull {modelName}"); } - // Model exists and is available — Ollama handles tool support at the API level + // Successful validation must stay successful when older servers omit metadata. + support = ModelVisionCapabilities.Parse(await response.Content.ReadAsStringAsync()); return (true, null); } catch (Exception ex) { return (false, $"Could not validate model: {ex.Message}"); } + finally + { + // A late response from a previous model must never overwrite the current state. + if (identity == ModelIdentity && version == Interlocked.Read(ref _modelInspectionVersion)) + { + _visionIdentity = identity; + _visionSupport = support; + RebuildSystemPrompt(); + if (_chatHistory.Count > 0 && _chatHistory[0].Role == ChatRole.System) + _chatHistory[0] = new ChatMessage(ChatRole.System, _systemPrompt); + BuildAgent(); + } + } } /// diff --git a/src/MandoCode/Services/Ai/AgentFunctionMiddleware.cs b/src/MandoCode/Services/Ai/AgentFunctionMiddleware.cs index f599cb2..3954b4f 100644 --- a/src/MandoCode/Services/Ai/AgentFunctionMiddleware.cs +++ b/src/MandoCode/Services/Ai/AgentFunctionMiddleware.cs @@ -216,7 +216,7 @@ public InvocationScope BeginScope() var deduplicationWindow = isWriteOperation ? _writeDeduplicationWindow : _readDeduplicationWindow; var callKey = CreateCallKey(functionName, context.Arguments, isWriteOperation); - if (_recentCalls.TryGetValue(callKey, out var cached) && + if (!PreviewToolPolicy.IsLiveTool(functionName) && _recentCalls.TryGetValue(callKey, out var cached) && DateTime.UtcNow - cached.Time < deduplicationWindow) { return cached.Result ?? "Operation already completed."; @@ -440,10 +440,10 @@ public InvocationScope BeginScope() deliveredResult = resultStr; } - _recentCalls[callKey] = (DateTime.UtcNow, deliveredResult); + if (!PreviewToolPolicy.IsLiveTool(functionName)) _recentCalls[callKey] = (DateTime.UtcNow, deliveredResult); CleanupOldEntries(); - var isError = resultStr.StartsWith("Error:", StringComparison.OrdinalIgnoreCase); + var isError = resultStr.StartsWith("Error:", StringComparison.OrdinalIgnoreCase) || PreviewToolPolicy.IsFailure(functionName, resultStr); UpdateScopeForCompletedCall(context, functionName, resultStr, isError); CompleteWith(functionName, resultStr, success: !isError); diff --git a/src/MandoCode/Services/Ai/FallbackFunctionCallExecutor.cs b/src/MandoCode/Services/Ai/FallbackFunctionCallExecutor.cs index dee527a..8b03e4a 100644 --- a/src/MandoCode/Services/Ai/FallbackFunctionCallExecutor.cs +++ b/src/MandoCode/Services/Ai/FallbackFunctionCallExecutor.cs @@ -456,7 +456,7 @@ public static string ToSnakeCase(string input) { FunctionName = functionName, Result = resultString.Length > 200 ? resultString[..200] + "..." : resultString, - Success = true + Success = !PreviewToolPolicy.IsFailure(functionName, resultString) }); return resultString; diff --git a/src/MandoCode/Services/Ai/Planning/PlanToolEvidence.cs b/src/MandoCode/Services/Ai/Planning/PlanToolEvidence.cs index 7047457..cb2e272 100644 --- a/src/MandoCode/Services/Ai/Planning/PlanToolEvidence.cs +++ b/src/MandoCode/Services/Ai/Planning/PlanToolEvidence.cs @@ -72,6 +72,7 @@ public static IReadOnlyDictionary SnapshotFileVersions( needsExecutionCheck |= extension is not (".md" or ".txt" or ".rst"); } else if (call.Name is "execute_command" || + PreviewToolPolicy.IsFreshObservation(call.Name, result.Result?.ToString() ?? "") || call.Name.Contains("test", StringComparison.OrdinalIgnoreCase) || call.Name.Contains("browser", StringComparison.OrdinalIgnoreCase)) { diff --git a/src/MandoCode/Services/Ai/PreviewToolPolicy.cs b/src/MandoCode/Services/Ai/PreviewToolPolicy.cs new file mode 100644 index 0000000..a6cea3b --- /dev/null +++ b/src/MandoCode/Services/Ai/PreviewToolPolicy.cs @@ -0,0 +1,33 @@ +using System.Text.Json; + +namespace MandoCode.Services; + +/// Live host preview operations cannot reuse cached observations or action results. +public static class PreviewToolPolicy +{ + public static bool IsLiveTool(string name) => name is + "open_desktop_preview" or "refresh_desktop_preview" or "inspect_desktop_preview" or + "observe_desktop_preview" or "click_desktop_preview" or "press_key_desktop_preview" or + "hover_desktop_preview" or "fill_desktop_preview" or "select_desktop_preview" or + "scroll_desktop_preview" or "wait_for_desktop_preview"; + + public static bool IsFailure(string name, string result) => IsLiveTool(name) && !HasSuccessfulResult(result); + + public static bool IsFreshObservation(string name, string result) => + IsLiveTool(name) && name is not ("open_desktop_preview" or "refresh_desktop_preview") && + HasSuccessfulResult(result, requireDocument: true); + + private static bool HasSuccessfulResult(string result, bool requireDocument = false) + { + try + { + using var document = JsonDocument.Parse(result); + var root = document.RootElement; + return root.ValueKind == JsonValueKind.Object && root.TryGetProperty("ok", out var ok) && + ok.ValueKind == JsonValueKind.True && (!requireDocument || + root.TryGetProperty("readyState", out var ready) && ready.ValueKind == JsonValueKind.String && + ready.GetString() is "interactive" or "complete"); + } + catch (JsonException) { return false; } + } +} diff --git a/tests/MandoCode.Tests/ModelVisionCapabilitiesTests.cs b/tests/MandoCode.Tests/ModelVisionCapabilitiesTests.cs new file mode 100644 index 0000000..c1dae98 --- /dev/null +++ b/tests/MandoCode.Tests/ModelVisionCapabilitiesTests.cs @@ -0,0 +1,112 @@ +using System.Net; +using MandoCode.Models; +using MandoCode.Services; +using Xunit; + +namespace MandoCode.Tests; + +public class ModelVisionCapabilitiesTests +{ + [Theory] + [InlineData("{\"capabilities\":[\"completion\",\"vision\",\"tools\"]}", ModelVisionSupport.Supported)] + [InlineData("{\"capabilities\":[\"VISION\"]}", ModelVisionSupport.Supported)] + [InlineData("{\"capabilities\":[\"completion\",\"tools\"]}", ModelVisionSupport.Unsupported)] + [InlineData("{}", ModelVisionSupport.Unknown)] + [InlineData("{\"capabilities\":[]}", ModelVisionSupport.Unknown)] + [InlineData("{\"capabilities\":null}", ModelVisionSupport.Unknown)] + [InlineData("{\"capabilities\":\"vision\"}", ModelVisionSupport.Unknown)] + [InlineData("{\"capabilities\":[\"vision\",42]}", ModelVisionSupport.Unknown)] + [InlineData("{\"capabilities\":[\"\"]}", ModelVisionSupport.Unknown)] + [InlineData("not json", ModelVisionSupport.Unknown)] + [InlineData("[]", ModelVisionSupport.Unknown)] + public void ReadsExplicitCapabilitiesOnly(string json, ModelVisionSupport expected) => + Assert.Equal(expected, ModelVisionCapabilities.Parse(json)); + + [Fact] + public async Task ValidationUpdatesAgentAndPreservesConversation_UnknownDoesNotBlock() + { + var config = new MandoCodeConfig(); + var ai = Create(config); + ai.AppendUserNote("Keep my conversation"); + using var client = Client("{\"capabilities\":[\"vision\"]}"); + Assert.True((await ai.ValidateModelAsync(client)).IsValid); + Assert.True(ai.SupportsImageInput); + var history = await ai.GetHistoryAsync(); + Assert.Contains("Vision supported", history[0].Text); + Assert.Contains(history, message => message.Text == "Keep my conversation"); + + using var legacy = Client("not json"); + Assert.True((await ai.ValidateModelAsync(legacy)).IsValid); + Assert.Equal(ModelVisionSupport.Unknown, ai.VisionSupport); + Assert.False(ai.SupportsImageInput); + } + + [Fact] + public async Task ModelOrEndpointMutationImmediatelyInvalidatesVision() + { + var config = new MandoCodeConfig(); + var ai = Create(config); + using var client = Client("{\"capabilities\":[\"vision\"]}"); + await ai.ValidateModelAsync(client); + config.OllamaEndpoint = "http://another-server:11434"; + Assert.Equal(ModelVisionSupport.Unknown, ai.VisionSupport); + await ai.ValidateModelAsync(client); + Assert.True(ai.SupportsImageInput); + config.ModelName = "different-model"; + Assert.Equal(ModelVisionSupport.Unknown, ai.VisionSupport); + } + + [Fact] + public async Task LateResponseCannotOverwriteNewerInspection() + { + var ai = Create(new MandoCodeConfig()); + var pending = new TaskCompletionSource(); + using var slow = new HttpClient(new Handler(_ => pending.Task)); + var oldCheck = ai.ValidateModelAsync(slow); + using var current = Client("{\"capabilities\":[\"completion\"]}"); + await ai.ValidateModelAsync(current); + pending.SetResult(Response("{\"capabilities\":[\"vision\"]}")); + await oldCheck; + Assert.Equal(ModelVisionSupport.Unsupported, ai.VisionSupport); + } + + [Fact] + public async Task LateResponseFromPreviousModelIsIgnored() + { + var config = new MandoCodeConfig(); + var ai = Create(config); + var pending = new TaskCompletionSource(); + using var slow = new HttpClient(new Handler(_ => pending.Task)); + var oldCheck = ai.ValidateModelAsync(slow); + config.ModelName = "replacement-model"; + pending.SetResult(Response("{\"capabilities\":[\"vision\"]}")); + await oldCheck; + Assert.Equal(ModelVisionSupport.Unknown, ai.VisionSupport); + Assert.False(ai.SupportsImageInput); + } + + [Fact] + public async Task FailedValidationClearsPreviouslySupportedVision() + { + var ai = Create(new MandoCodeConfig()); + using var client = Client("{\"capabilities\":[\"vision\"]}"); + await ai.ValidateModelAsync(client); + using var failed = new HttpClient(new Handler(_ => Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound)))); + Assert.False((await ai.ValidateModelAsync(failed)).IsValid); + Assert.Equal(ModelVisionSupport.Unknown, ai.VisionSupport); + } + + private static AIService Create(MandoCodeConfig config) + { + var root = new ProjectRootAccessor(Path.GetTempPath()); + return new AIService(root, config, new TokenTrackingService(), new PlanHandoff(), + new SkillLoader(config, root), new McpClientManager(config), new McpApprovalGate(config), new SpinnerService()); + } + + private static HttpResponseMessage Response(string json) => new(HttpStatusCode.OK) { Content = new StringContent(json) }; + private static HttpClient Client(string json) => new(new Handler(_ => Task.FromResult(Response(json)))); + private sealed class Handler(Func> send) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => send(request); + } +} diff --git a/tests/MandoCode.Tests/PreviewToolPolicyTests.cs b/tests/MandoCode.Tests/PreviewToolPolicyTests.cs new file mode 100644 index 0000000..7c7351e --- /dev/null +++ b/tests/MandoCode.Tests/PreviewToolPolicyTests.cs @@ -0,0 +1,65 @@ +using MandoCode.Models; +using MandoCode.Services; +using Microsoft.Extensions.AI; +using Xunit; + +namespace MandoCode.Tests; + +public class PreviewToolPolicyTests +{ + [Theory] + [InlineData("inspect_desktop_preview")] + [InlineData("observe_desktop_preview")] + [InlineData("press_key_desktop_preview")] + [InlineData("click_desktop_preview")] + [InlineData("hover_desktop_preview")] + [InlineData("fill_desktop_preview")] + [InlineData("select_desktop_preview")] + [InlineData("scroll_desktop_preview")] + [InlineData("wait_for_desktop_preview")] + [InlineData("open_desktop_preview")] + [InlineData("refresh_desktop_preview")] + public async Task LiveOperationsAreNeverReplayedFromCache(string name) + { + var middleware = new AgentFunctionMiddleware(5); + var calls = 0; + var tool = AIFunctionFactory.Create(() => { calls++; return "{\"ok\":true}"; }, new AIFunctionFactoryOptions { Name = name }); + await AgentMiddlewareTestHelpers.InvokeAsync(middleware, tool); + await AgentMiddlewareTestHelpers.InvokeAsync(middleware, tool); + Assert.Equal(2, calls); + } + + [Fact] + public async Task BrowserFailureIsReportedAsFailureInTheTranscript() + { + var middleware = new AgentFunctionMiddleware(5); + FunctionExecutionResult? completed = null; + middleware.OnFunctionCompleted += result => completed = result; + var tool = AIFunctionFactory.Create(() => "{\"ok\":false,\"error\":\"selector is ambiguous\"}", new AIFunctionFactoryOptions { Name = "click_desktop_preview" }); + await AgentMiddlewareTestHelpers.InvokeAsync(middleware, tool); + Assert.False(completed!.Success); + } + + [Theory] + [InlineData("inspect_desktop_preview", "{\"ok\":true,\"readyState\":\"complete\"}", true)] + [InlineData("click_desktop_preview", "{\"ok\":true,\"readyState\":\"complete\"}", true)] + [InlineData("observe_desktop_preview", "{\"ok\":true,\"readyState\":\"complete\"}", true)] + [InlineData("press_key_desktop_preview", "{\"ok\":true,\"readyState\":\"complete\"}", true)] + [InlineData("wait_for_desktop_preview", "{\"ok\":true,\"readyState\":\"interactive\"}", true)] + [InlineData("open_desktop_preview", "{\"ok\":true,\"readyState\":\"complete\"}", false)] + [InlineData("refresh_desktop_preview", "{\"ok\":true,\"readyState\":\"complete\"}", false)] + [InlineData("inspect_desktop_preview", "{\"ok\":false}", false)] + [InlineData("inspect_desktop_preview", "{\"ok\":true,\"readyState\":\"loading\"}", false)] + [InlineData("inspect_desktop_preview", "{\"ok\":true}", false)] + [InlineData("inspect_desktop_preview", "page unavailable", false)] + public void FreshnessRequiresAnActualBrowserObservation(string name, string result, bool fresh) + { + ChatMessage[] history = [ + new(ChatRole.Assistant, [new FunctionCallContent("edit", "edit_file", new Dictionary { ["path"] = "index.html" })]), + new(ChatRole.Tool, [new FunctionResultContent("edit", "edited")]), + new(ChatRole.Assistant, [new FunctionCallContent("browser", name)]), + new(ChatRole.Tool, [new FunctionResultContent("browser", result)]) + ]; + Assert.Equal(fresh, PlanToolEvidence.AssessFreshness(history) == null); + } +}