feature: model vision detection and live-result handling for browser tools

docs/model-capabilities.md

@@ -0,0 +1,26 @@
+# Model image capabilities
+
+Model validation reads Ollama's `/api/show` response. The CLI and Desktop display
+the detected status and include it in the agent's system instructions:
+
+- **Vision supported:** the capabilities array contains `vision`.
+- **Text-only model:** a valid, nonempty capabilities array omits `vision`.
+- **Vision capability unknown:** metadata is missing, empty, or malformed.
+
+Unknown metadata does not fail an otherwise successful model validation. HTTP
+and connection failures retain the existing model validation behavior and leave
+vision support unknown. No model-name guesses or paid image probes are used.
+
+Detection runs during model validation and when reinitialization or settings
+changes select an uninspected endpoint/model. Results belong to that endpoint and
+effective model name. Late responses cannot replace a newer inspection or apply
+to a different model. Ordinary settings changes preserve conversation history.
+
+`AIService.VisionSupport` exposes the three states. `SupportsImageInput` is true
+only for confirmed support. This describes the model, not the host application's
+ability to deliver images. Browser tools must also supply actual image content
+before enabling visual inspection; returning a screenshot path is insufficient.
+Text/DOM browser tools need no vision capability. This change does not add browser
+inspection tools, screenshot capture, or image message delivery.
+
+Provider contract: https://docs.ollama.com/api-reference/show-model-details

src/MandoCode/Components/App.razor

@@ -777,6 +777,7 @@
  {
  _modelError = false;
  _modelWarning = null;
+ AnsiConsole.MarkupLine($"[dim]{AI.VisionSupport.Label()}[/]");
  }
  }
 

src/MandoCode/Models/ModelVisionSupport.cs

@@ -0,0 +1,45 @@
+using System.Text.Json;
+
+namespace MandoCode.Models;
+
+public enum ModelVisionSupport { Unknown, Unsupported, Supported }
+
+/// <summary>Provider-reported image input support, independent of host image delivery.</summary>
+public static class ModelVisionCapabilities
+{
+ public static ModelVisionSupport Parse(string modelDetails)
+ {
+ try
+ {
+ using var document = JsonDocument.Parse(modelDetails);
+ if (document.RootElement.ValueKind != JsonValueKind.Object ||
+ !document.RootElement.TryGetProperty("capabilities", out var capabilities) ||
+ capabilities.ValueKind != JsonValueKind.Array || capabilities.GetArrayLength() == 0)
+ return ModelVisionSupport.Unknown;
+
+ var vision = false;
+ foreach (var capability in capabilities.EnumerateArray())
+ {
+ if (capability.ValueKind != JsonValueKind.String || string.IsNullOrWhiteSpace(capability.GetString()))
+ return ModelVisionSupport.Unknown;
+ vision |= string.Equals(capability.GetString(), "vision", StringComparison.OrdinalIgnoreCase);
+ }
+ return vision ? ModelVisionSupport.Supported : ModelVisionSupport.Unsupported;
+ }
+ catch (JsonException) { return ModelVisionSupport.Unknown; }
+ }
+
+ public static string Label(this ModelVisionSupport support) => support switch
+ {
+ ModelVisionSupport.Supported => "Vision supported",
+ ModelVisionSupport.Unsupported => "Text-only model",
+ _ => "Vision capability unknown"
+ };
+
+ public static string AgentInstruction(this ModelVisionSupport support) =>
+ $"Model image input capability: {support.Label()}. " +
+ (support == ModelVisionSupport.Supported
+ ? "Only describe visual details when an image has actually been supplied as image input. A screenshot path or URL alone is not image input. "
+ : "Use text and DOM observations for browser checks; do not request image input or claim visual inspection. ") +
+ "Model capability does not grant browser access. Use only available tools and report any limits on visual verification.";
+}

src/MandoCode/Services/Ai/AIService.cs

@@ -28,6 +28,16 @@ public class AIService
  // straight to _agent.RunAsync with no conversion — see ExecuteAgentModelCallAsync.
  private readonly List<ChatMessage> _chatHistory;
  private string _systemPrompt;
+ private string? _visionIdentity;
+ private ModelVisionSupport _visionSupport;
+ private long _modelInspectionVersion;
+ private string ModelIdentity => OllamaSetupHelper.BuildUrl(_config.OllamaEndpoint, "api/show") + "\n" + _config.GetEffectiveModelName();
+
+ public ModelVisionSupport VisionSupport => _visionIdentity == ModelIdentity
+ ? _visionSupport : ModelVisionSupport.Unknown;
+
+ // A host must also implement actual image delivery before exposing screenshot tools.
+ public bool SupportsImageInput => VisionSupport == ModelVisionSupport.Supported;
 
  // MAF agent — the live chat path (feat/agent-framework-migration).
  private AIAgent? _agent;
@@ -185,6 +195,7 @@ private void RebuildSystemPrompt()
  {
  var skillIndex = SystemPrompts.BuildSkillIndex(_skillLoader.GetAll());
  _systemPrompt = SystemPrompts.BuildMandoCodeAssistant(_config.EnableWebSearch, _config.AgentName) + "\n\n" + ShellEnvironment.SystemPromptRules;
+ _systemPrompt += "\n\n" + VisionSupport.AgentInstruction();
  if (!string.IsNullOrEmpty(skillIndex))
  {
  _systemPrompt += "\n\n" + skillIndex;
@@ -208,6 +219,7 @@ private void RebuildSystemPrompt()
  public async Task ReinitializeAsync(MandoCodeConfig config)
  {
  _config = config;
+ if (_visionIdentity != ModelIdentity) await ValidateModelAsync();
  RebuildSystemPrompt();
  BuildAgent();
  await AttachMcpPluginsAsync();
@@ -224,6 +236,7 @@ public async Task ReinitializeAsync(MandoCodeConfig config)
  public async Task RefreshSettingsAsync(MandoCodeConfig config)
  {
  _config = config;
+ if (_visionIdentity != ModelIdentity) await ValidateModelAsync();
  RebuildSystemPrompt();
 
  // The history-preserving path still has the OLD system prompt as message 0 —
@@ -665,9 +678,17 @@ private int EffectiveNumCtx()
  /// </summary>
  public async Task<(bool IsValid, string? ErrorMessage)> ValidateModelAsync()
  {
+ using var client = new HttpClient { Timeout = TimeSpan.FromSeconds(10) };
+ return await ValidateModelAsync(client);
+ }
+
+ internal async Task<(bool IsValid, string? ErrorMessage)> ValidateModelAsync(HttpClient client)
+ {
+ var identity = ModelIdentity;
+ var version = Interlocked.Increment(ref _modelInspectionVersion);
+ var support = ModelVisionSupport.Unknown;
  try
  {
- using var client = new HttpClient { Timeout = TimeSpan.FromSeconds(10) };
  var modelName = _config.GetEffectiveModelName();
 
  // Check if model exists and get its info
@@ -681,13 +702,27 @@ private int EffectiveNumCtx()
  return (false, $"Model '{modelName}' not found. Run: ollama pull {modelName}");
  }
 
- // Model exists and is available — Ollama handles tool support at the API level
+ // Successful validation must stay successful when older servers omit metadata.
+ support = ModelVisionCapabilities.Parse(await response.Content.ReadAsStringAsync());
  return (true, null);
  }
  catch (Exception ex)
  {
  return (false, $"Could not validate model: {ex.Message}");
  }
+ finally
+ {
+ // A late response from a previous model must never overwrite the current state.
+ if (identity == ModelIdentity && version == Interlocked.Read(ref _modelInspectionVersion))
+ {
+ _visionIdentity = identity;
+ _visionSupport = support;
+ RebuildSystemPrompt();
+ if (_chatHistory.Count > 0 && _chatHistory[0].Role == ChatRole.System)
+ _chatHistory[0] = new ChatMessage(ChatRole.System, _systemPrompt);
+ BuildAgent();
+ }
+ }
  }
 
  /// <summary>

src/MandoCode/Services/Ai/AgentFunctionMiddleware.cs

@@ -216,7 +216,7 @@ public InvocationScope BeginScope()
  var deduplicationWindow = isWriteOperation ? _writeDeduplicationWindow : _readDeduplicationWindow;
  var callKey = CreateCallKey(functionName, context.Arguments, isWriteOperation);
 
- if (_recentCalls.TryGetValue(callKey, out var cached) &&
+ if (!PreviewToolPolicy.IsLiveTool(functionName) && _recentCalls.TryGetValue(callKey, out var cached) &&
  DateTime.UtcNow - cached.Time < deduplicationWindow)
  {
  return cached.Result ?? "Operation already completed.";
@@ -440,10 +440,10 @@ public InvocationScope BeginScope()
  deliveredResult = resultStr;
  }
 
- _recentCalls[callKey] = (DateTime.UtcNow, deliveredResult);
+if (!PreviewToolPolicy.IsLiveTool(functionName)) _recentCalls[callKey] = (DateTime.UtcNow, deliveredResult);
  CleanupOldEntries();
 
- var isError = resultStr.StartsWith("Error:", StringComparison.OrdinalIgnoreCase);
+ var isError = resultStr.StartsWith("Error:", StringComparison.OrdinalIgnoreCase) || PreviewToolPolicy.IsFailure(functionName, resultStr);
  UpdateScopeForCompletedCall(context, functionName, resultStr, isError);
 
  CompleteWith(functionName, resultStr, success: !isError);

src/MandoCode/Services/Ai/FallbackFunctionCallExecutor.cs

@@ -456,7 +456,7 @@ public static string ToSnakeCase(string input)
  {
  FunctionName = functionName,
  Result = resultString.Length > 200 ? resultString[..200] + "..." : resultString,
- Success = true
+ Success = !PreviewToolPolicy.IsFailure(functionName, resultString)
  });
 
  return resultString;

src/MandoCode/Services/Ai/Planning/PlanToolEvidence.cs

@@ -72,6 +72,7 @@ public static IReadOnlyDictionary<string, string> SnapshotFileVersions(
  needsExecutionCheck |= extension is not (".md" or ".txt" or ".rst");
  }
  else if (call.Name is "execute_command" ||
+ PreviewToolPolicy.IsFreshObservation(call.Name, result.Result?.ToString() ?? "") ||
  call.Name.Contains("test", StringComparison.OrdinalIgnoreCase) ||
  call.Name.Contains("browser", StringComparison.OrdinalIgnoreCase))
  {

src/MandoCode/Services/Ai/PreviewToolPolicy.cs

@@ -0,0 +1,33 @@
+using System.Text.Json;
+
+namespace MandoCode.Services;
+
+/// <summary>Live host preview operations cannot reuse cached observations or action results.</summary>
+public static class PreviewToolPolicy
+{
+ public static bool IsLiveTool(string name) => name is
+ "open_desktop_preview" or "refresh_desktop_preview" or "inspect_desktop_preview" or
+ "observe_desktop_preview" or "click_desktop_preview" or "press_key_desktop_preview" or
+ "hover_desktop_preview" or "fill_desktop_preview" or "select_desktop_preview" or
+ "scroll_desktop_preview" or "wait_for_desktop_preview";
+
+ public static bool IsFailure(string name, string result) => IsLiveTool(name) && !HasSuccessfulResult(result);
+
+ public static bool IsFreshObservation(string name, string result) =>
+ IsLiveTool(name) && name is not ("open_desktop_preview" or "refresh_desktop_preview") &&
+ HasSuccessfulResult(result, requireDocument: true);
+
+ private static bool HasSuccessfulResult(string result, bool requireDocument = false)
+ {
+ try
+ {
+ using var document = JsonDocument.Parse(result);
+ var root = document.RootElement;
+ return root.ValueKind == JsonValueKind.Object && root.TryGetProperty("ok", out var ok) &&
+ ok.ValueKind == JsonValueKind.True && (!requireDocument ||
+ root.TryGetProperty("readyState", out var ready) && ready.ValueKind == JsonValueKind.String &&
+ ready.GetString() is "interactive" or "complete");
+ }
+ catch (JsonException) { return false; }
+ }
+}

tests/MandoCode.Tests/ModelVisionCapabilitiesTests.cs

@@ -0,0 +1,112 @@
+using System.Net;
+using MandoCode.Models;
+using MandoCode.Services;
+using Xunit;
+
+namespace MandoCode.Tests;
+
+public class ModelVisionCapabilitiesTests
+{
+ [Theory]
+ [InlineData("{\"capabilities\":[\"completion\",\"vision\",\"tools\"]}", ModelVisionSupport.Supported)]
+ [InlineData("{\"capabilities\":[\"VISION\"]}", ModelVisionSupport.Supported)]
+ [InlineData("{\"capabilities\":[\"completion\",\"tools\"]}", ModelVisionSupport.Unsupported)]
+ [InlineData("{}", ModelVisionSupport.Unknown)]
+ [InlineData("{\"capabilities\":[]}", ModelVisionSupport.Unknown)]
+ [InlineData("{\"capabilities\":null}", ModelVisionSupport.Unknown)]
+ [InlineData("{\"capabilities\":\"vision\"}", ModelVisionSupport.Unknown)]
+ [InlineData("{\"capabilities\":[\"vision\",42]}", ModelVisionSupport.Unknown)]
+ [InlineData("{\"capabilities\":[\"\"]}", ModelVisionSupport.Unknown)]
+ [InlineData("not json", ModelVisionSupport.Unknown)]
+ [InlineData("[]", ModelVisionSupport.Unknown)]
+ public void ReadsExplicitCapabilitiesOnly(string json, ModelVisionSupport expected) =>
+ Assert.Equal(expected, ModelVisionCapabilities.Parse(json));
+
+ [Fact]
+ public async Task ValidationUpdatesAgentAndPreservesConversation_UnknownDoesNotBlock()
+ {
+ var config = new MandoCodeConfig();
+ var ai = Create(config);
+ ai.AppendUserNote("Keep my conversation");
+ using var client = Client("{\"capabilities\":[\"vision\"]}");
+ Assert.True((await ai.ValidateModelAsync(client)).IsValid);
+ Assert.True(ai.SupportsImageInput);
+ var history = await ai.GetHistoryAsync();
+ Assert.Contains("Vision supported", history[0].Text);
+ Assert.Contains(history, message => message.Text == "Keep my conversation");
+
+ using var legacy = Client("not json");
+ Assert.True((await ai.ValidateModelAsync(legacy)).IsValid);
+ Assert.Equal(ModelVisionSupport.Unknown, ai.VisionSupport);
+ Assert.False(ai.SupportsImageInput);
+ }
+
+ [Fact]
+ public async Task ModelOrEndpointMutationImmediatelyInvalidatesVision()
+ {
+ var config = new MandoCodeConfig();
+ var ai = Create(config);
+ using var client = Client("{\"capabilities\":[\"vision\"]}");
+ await ai.ValidateModelAsync(client);
+ config.OllamaEndpoint = "http://another-server:11434";
+ Assert.Equal(ModelVisionSupport.Unknown, ai.VisionSupport);
+ await ai.ValidateModelAsync(client);
+ Assert.True(ai.SupportsImageInput);
+ config.ModelName = "different-model";
+ Assert.Equal(ModelVisionSupport.Unknown, ai.VisionSupport);
+ }
+
+ [Fact]
+ public async Task LateResponseCannotOverwriteNewerInspection()
+ {
+ var ai = Create(new MandoCodeConfig());
+ var pending = new TaskCompletionSource<HttpResponseMessage>();
+ using var slow = new HttpClient(new Handler(_ => pending.Task));
+ var oldCheck = ai.ValidateModelAsync(slow);
+ using var current = Client("{\"capabilities\":[\"completion\"]}");
+ await ai.ValidateModelAsync(current);
+ pending.SetResult(Response("{\"capabilities\":[\"vision\"]}"));
+ await oldCheck;
+ Assert.Equal(ModelVisionSupport.Unsupported, ai.VisionSupport);
+ }
+
+ [Fact]
+ public async Task LateResponseFromPreviousModelIsIgnored()
+ {
+ var config = new MandoCodeConfig();
+ var ai = Create(config);
+ var pending = new TaskCompletionSource<HttpResponseMessage>();
+ using var slow = new HttpClient(new Handler(_ => pending.Task));
+ var oldCheck = ai.ValidateModelAsync(slow);
+ config.ModelName = "replacement-model";
+ pending.SetResult(Response("{\"capabilities\":[\"vision\"]}"));
+ await oldCheck;
+ Assert.Equal(ModelVisionSupport.Unknown, ai.VisionSupport);
+ Assert.False(ai.SupportsImageInput);
+ }
+
+ [Fact]
+ public async Task FailedValidationClearsPreviouslySupportedVision()
+ {
+ var ai = Create(new MandoCodeConfig());
+ using var client = Client("{\"capabilities\":[\"vision\"]}");
+ await ai.ValidateModelAsync(client);
+ using var failed = new HttpClient(new Handler(_ => Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound))));
+ Assert.False((await ai.ValidateModelAsync(failed)).IsValid);
+ Assert.Equal(ModelVisionSupport.Unknown, ai.VisionSupport);
+ }
+
+ private static AIService Create(MandoCodeConfig config)
+ {
+ var root = new ProjectRootAccessor(Path.GetTempPath());
+ return new AIService(root, config, new TokenTrackingService(), new PlanHandoff(),
+ new SkillLoader(config, root), new McpClientManager(config), new McpApprovalGate(config), new SpinnerService());
+ }
+
+ private static HttpResponseMessage Response(string json) => new(HttpStatusCode.OK) { Content = new StringContent(json) };
+ private static HttpClient Client(string json) => new(new Handler(_ => Task.FromResult(Response(json))));
+ private sealed class Handler(Func<HttpRequestMessage, Task<HttpResponseMessage>> send) : HttpMessageHandler
+ {
+ protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => send(request);
+ }
+}