From 6128fb77f44470ef93df4398e87fe290ab6d78f8 Mon Sep 17 00:00:00 2001 From: idevlab Date: Thu, 27 Aug 2026 16:14:33 +0800 Subject: [PATCH 01/14] Add Espresso ANE inference backend Amp-Thread-ID: https://ampcode.com/threads/T-01a03e89-5ac5-732e-9da8-9eafeb591994 Co-authored-by: Amp --- Package.resolved | 11 ++- Package.swift | 5 +- Sources/App/VoicePipeline+Models.swift | 44 +++++++-- Sources/App/VoicePipeline+ScreenContext.swift | 3 +- Sources/App/VoicePipeline.swift | 10 +- Sources/Config/AppSettings.swift | 20 +++- .../Integration/InputSessionCoordinator.swift | 2 +- Sources/LLM/EspressoLLMEngine.swift | 93 +++++++++++++++++++ .../Processing/TextProcessingOptions.swift | 4 + .../Processing/TextProcessor+Generation.swift | 29 ++++-- Sources/Processing/TextProcessor.swift | 16 +++- .../Resources/en.lproj/Localizable.strings | 8 +- .../zh-Hans.lproj/Localizable.strings | 8 +- Sources/UI/ModelManagementFamilies.swift | 44 ++++++++- Sources/UI/ModelManagementRows.swift | 1 + Sources/UI/ModelManagementSections.swift | 34 ++++++- Sources/UI/ModelManagementView.swift | 24 +++++ Tests/OpenTypeTests/ConfigurationTests.swift | 31 +++++++ 18 files changed, 351 insertions(+), 36 deletions(-) create mode 100644 Sources/LLM/EspressoLLMEngine.swift diff --git a/Package.resolved b/Package.resolved index 9358719f..de1e6b56 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "7f808a86fcdc955b5832bd1753d25aa3fc629c44e8e2102637d2722b2517dbf8", + "originHash" : "df115d1ba7786c21c91c720b1530dcd190051b18e6df521ee04e7a4f435221a5", "pins" : [ { "identity" : "argmax-oss-swift", @@ -10,6 +10,15 @@ "version" : "1.0.0" } }, + { + "identity" : "espresso", + "kind" : "remoteSourceControl", + "location" : "https://github.com/christopherkarani/Espresso.git", + "state" : { + "revision" : "a8629bbcb4d4ae7179b0c3995c36c868e690df87", + "version" : "0.9.0" + } + }, { "identity" : "eventsource", "kind" : "remoteSourceControl", diff --git a/Package.swift b/Package.swift index e46d467a..a1612a58 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 6.0 +// swift-tools-version: 6.2 import PackageDescription let package = Package( @@ -13,6 +13,7 @@ let package = Package( ], dependencies: [ .package(url: "https://github.com/argmaxinc/argmax-oss-swift.git", from: "1.0.0"), + .package(url: "https://github.com/christopherkarani/Espresso.git", from: "0.9.0"), .package(url: "https://github.com/huggingface/swift-transformers", from: "1.3.3"), .package(url: "https://github.com/ml-explore/mlx-swift-lm", branch: "main"), ], @@ -21,6 +22,8 @@ let package = Package( name: "OpenType", dependencies: [ .product(name: "WhisperKit", package: "argmax-oss-swift"), + .product(name: "ESPRuntime", package: "Espresso"), + .product(name: "RealModelInference", package: "Espresso"), .product(name: "Hub", package: "swift-transformers"), .product(name: "Tokenizers", package: "swift-transformers"), .product(name: "MLXLLM", package: "mlx-swift-lm"), diff --git a/Sources/App/VoicePipeline+Models.swift b/Sources/App/VoicePipeline+Models.swift index b0ca06b1..56a22ceb 100644 --- a/Sources/App/VoicePipeline+Models.swift +++ b/Sources/App/VoicePipeline+Models.swift @@ -40,32 +40,56 @@ extension VoicePipeline { return } - let model = appState.settings.llmModel.trimmingCharacters(in: .whitespacesAndNewlines) - guard !model.isEmpty else { return } + let settings = appState.settings + let backend = settings.localLLMBackend + let model = settings.llmModel.trimmingCharacters(in: .whitespacesAndNewlines) + let espressoPath = settings.espressoModelPath.trimmingCharacters(in: .whitespacesAndNewlines) + let selectedModel = backend == .espresso ? espressoPath : model + guard !selectedModel.isEmpty else { return } let catalog = ModelCatalog.shared - catalog.refreshStatus() - let modelStatus = catalog.llmModels.first(where: { $0.id == model })?.status - guard modelStatus == .downloaded || modelStatus == .ready else { + let modelIsAvailable: Bool + if backend == .espresso { + modelIsAvailable = FileManager.default.fileExists( + atPath: NSString(string: espressoPath).expandingTildeInPath + ) + } else { + catalog.refreshStatus() + let status = catalog.llmModels.first(where: { $0.id == model })?.status + modelIsAvailable = status == .downloaded || status == .ready + } + guard modelIsAvailable else { let message = L("model.download_required") - catalog.updateLLMStatus(model, status: .error(message)) + if backend == .mlx { + catalog.updateLLMStatus(model, status: .error(message)) + } appState.statusMessage = showFailureInStatus ? message : L("status.ready") return } appState.statusMessage = L("pipeline.loading_llm") - catalog.updateLLMStatus(model, status: .loading, detail: L("model.loading")) + if backend == .mlx { + catalog.updateLLMStatus(model, status: .loading, detail: L("model.loading")) + } - let loaded = await textProcessor.warmUpLLM(model: model) + let loaded = await textProcessor.warmUpLLM( + model: model, + backend: backend, + espressoModelPath: espressoPath + ) let ready = await textProcessor.isLLMReady appState.llmModelReady = loaded && ready if appState.llmModelReady { - catalog.updateLLMStatus(model, status: .ready) + if backend == .mlx { + catalog.updateLLMStatus(model, status: .ready) + } Log.info("[VoicePipeline] LLM model loaded into memory, ready for instant inference") appState.statusMessage = L("status.ready") } else { - catalog.updateLLMStatus(model, status: .error(L("pipeline.model_load_failed"))) + if backend == .mlx { + catalog.updateLLMStatus(model, status: .error(L("pipeline.model_load_failed"))) + } Log.info("[VoicePipeline] LLM warmup failed, will retry on demand") appState.statusMessage = showFailureInStatus ? L("pipeline.model_load_failed") : L("status.ready") } diff --git a/Sources/App/VoicePipeline+ScreenContext.swift b/Sources/App/VoicePipeline+ScreenContext.swift index 55f9cbe5..276b2c8c 100644 --- a/Sources/App/VoicePipeline+ScreenContext.swift +++ b/Sources/App/VoicePipeline+ScreenContext.swift @@ -15,7 +15,8 @@ extension VoicePipeline { screenOCRStartedAt = CFAbsoluteTimeGetCurrent() let mode = ScreenContextMode.effectiveCaptureMode( preference: appState.settings.screenContextMode, - useRemoteLLM: appState.settings.useRemoteLLM, + useRemoteLLM: appState.settings.useRemoteLLM + || appState.settings.localLLMBackend == .espresso, modelID: appState.settings.llmModel ) screenOCRTask = Task.detached(priority: .utility) { diff --git a/Sources/App/VoicePipeline.swift b/Sources/App/VoicePipeline.swift index 1c553bc2..5d28c1d3 100644 --- a/Sources/App/VoicePipeline.swift +++ b/Sources/App/VoicePipeline.swift @@ -41,6 +41,12 @@ final class VoicePipeline { let catalog = ModelCatalog.shared catalog.refreshStatus(recheckingErrors: true) let llmStatus = catalog.llmModels.first(where: { $0.id == settings.llmModel })?.status + let formattingModelID = settings.localLLMBackend == .espresso + ? settings.espressoModelPath + : settings.llmModel + let formattingModelAvailable = settings.localLLMBackend == .espresso + ? FileManager.default.fileExists(atPath: NSString(string: settings.espressoModelPath).expandingTildeInPath) + : (llmStatus == .downloaded || llmStatus == .ready) let shouldLoadSpeech = StartupModelPreloadPolicy.shouldPreloadSpeechModel( enabled: settings.preloadSpeechModelOnLaunch, speechEngine: settings.speechEngine, @@ -49,8 +55,8 @@ final class VoicePipeline { let shouldLoadFormatting = StartupModelPreloadPolicy.shouldPreloadFormattingModel( enabled: settings.preloadFormattingModelOnLaunch, useRemoteLLM: settings.useRemoteLLM, - modelID: settings.llmModel, - modelDownloaded: llmStatus == .downloaded || llmStatus == .ready + modelID: formattingModelID, + modelDownloaded: formattingModelAvailable ) if shouldLoadSpeech { diff --git a/Sources/Config/AppSettings.swift b/Sources/Config/AppSettings.swift index 134d0b85..4efd2ebb 100644 --- a/Sources/Config/AppSettings.swift +++ b/Sources/Config/AppSettings.swift @@ -46,6 +46,11 @@ enum SpeechEngineType: String, Codable, CaseIterable { } } +enum LocalLLMBackend: String, Codable, CaseIterable { + case mlx + case espresso +} + enum LanguageStyle: String, Codable, CaseIterable { case casual = "casual" case professional = "professional" @@ -279,6 +284,8 @@ final class AppSettings: ObservableObject { @Published var useCustomSystemPrompt: Bool @Published var customSystemPrompt: String @Published var useRemoteLLM: Bool + @Published var localLLMBackend: LocalLLMBackend + @Published var espressoModelPath: String @Published var remoteProvider: RemoteProvider @Published var remoteAPIKey: String @Published var remoteBaseURL: String @@ -312,7 +319,8 @@ final class AppSettings: ObservableObject { case useScreenContext, screenContextMode, enableInstantInsert, hasCompletedOnboarding, uiLanguage, historyRetention case enableMemory, memoryWindowMinutes, enableCorrectionLearning case useCustomSystemPrompt, customSystemPrompt - case useRemoteLLM, remoteProvider, remoteAPIKey, remoteBaseURL, remoteModel + case useRemoteLLM, localLLMBackend, espressoModelPath + case remoteProvider, remoteAPIKey, remoteBaseURL, remoteModel case menuBarIcon, appIconAppearance case volcAppKey, volcAccessKey, volcResourceId case localASRPythonPath @@ -378,6 +386,10 @@ final class AppSettings: ObservableObject { useCustomSystemPrompt = ud.bool(forKey: Key.useCustomSystemPrompt.rawValue) customSystemPrompt = ud.string(forKey: Key.customSystemPrompt.rawValue) ?? "" useRemoteLLM = ud.bool(forKey: Key.useRemoteLLM.rawValue) + localLLMBackend = LocalLLMBackend( + rawValue: ud.string(forKey: Key.localLLMBackend.rawValue) ?? "" + ) ?? .mlx + espressoModelPath = ud.string(forKey: Key.espressoModelPath.rawValue) ?? "" remoteProvider = RemoteProvider(rawValue: ud.string(forKey: Key.remoteProvider.rawValue) ?? "") ?? .custom remoteAPIKey = ud.string(forKey: Key.remoteAPIKey.rawValue) ?? "" remoteBaseURL = ud.string(forKey: Key.remoteBaseURL.rawValue) ?? "" @@ -447,6 +459,12 @@ final class AppSettings: ObservableObject { $useCustomSystemPrompt.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.useCustomSystemPrompt.rawValue) }.store(in: &cancellables) $customSystemPrompt.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.customSystemPrompt.rawValue) }.store(in: &cancellables) $useRemoteLLM.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.useRemoteLLM.rawValue) }.store(in: &cancellables) + $localLLMBackend.dropFirst().sink { + [defaults] in defaults.set($0.rawValue, forKey: Key.localLLMBackend.rawValue) + }.store(in: &cancellables) + $espressoModelPath.dropFirst().sink { + [defaults] in defaults.set($0, forKey: Key.espressoModelPath.rawValue) + }.store(in: &cancellables) $remoteProvider.dropFirst().sink { [defaults] in defaults.set($0.rawValue, forKey: Key.remoteProvider.rawValue) }.store(in: &cancellables) $remoteAPIKey.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.remoteAPIKey.rawValue) }.store(in: &cancellables) $remoteBaseURL.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.remoteBaseURL.rawValue) }.store(in: &cancellables) diff --git a/Sources/Integration/InputSessionCoordinator.swift b/Sources/Integration/InputSessionCoordinator.swift index b0d3b0f7..9a872f1b 100644 --- a/Sources/Integration/InputSessionCoordinator.swift +++ b/Sources/Integration/InputSessionCoordinator.swift @@ -230,7 +230,7 @@ final class InputSessionCoordinator { } let contextMode = ScreenContextMode.effectiveCaptureMode( preference: settings.screenContextMode, - useRemoteLLM: settings.useRemoteLLM, + useRemoteLLM: settings.useRemoteLLM || settings.localLLMBackend == .espresso, modelID: settings.llmModel ) return Task.detached(priority: .utility) { diff --git a/Sources/LLM/EspressoLLMEngine.swift b/Sources/LLM/EspressoLLMEngine.swift new file mode 100644 index 00000000..20f85915 --- /dev/null +++ b/Sources/LLM/EspressoLLMEngine.swift @@ -0,0 +1,93 @@ +import ESPRuntime +import Foundation +import RealModelInference + +actor EspressoLLMEngine { + private final class LoadedModel { + let path: String + let name: String + var engine: RealModelInferenceEngine + + init(path: String, name: String, engine: consuming RealModelInferenceEngine) { + self.path = path + self.name = name + self.engine = engine + } + } + + private var model: LoadedModel? + + func loadModel(path: String) throws { + let expandedPath = NSString(string: path).expandingTildeInPath + let url = URL(fileURLWithPath: expandedPath, isDirectory: true).standardizedFileURL + guard model?.path != url.path else { return } + + Log.info("[EspressoLLMEngine] loading bundle: \(url.path)") + let bundle = try ESPRuntimeBundle.open(at: url) + let selection = try ESPRuntimeRunner.resolve(bundle: bundle) + guard selection.backend == .anePrivate else { + throw EspressoLLMError.aneBackendUnavailable + } + + let engine = try RealModelInferenceEngine.build( + config: bundle.config, + weightDir: bundle.archive.weightsURL.path, + tokenizerDir: bundle.archive.tokenizerURL.path + ) + model = LoadedModel(path: url.path, name: bundle.config.name, engine: engine) + Log.info("[EspressoLLMEngine] bundle ready for ANE inference") + } + + func generate( + prompt: String, + systemPrompt: String, + maxTokens: Int, + temperature: Double + ) throws -> String { + guard let model else { throw EspressoLLMError.modelNotLoaded } + let input = Self.formatPrompt( + user: prompt, + system: systemPrompt, + modelName: model.name + ) + let started = CFAbsoluteTimeGetCurrent() + let result = try model.engine.generate( + prompt: input, + maxTokens: maxTokens, + temperature: Float(temperature) + ) + let elapsed = CFAbsoluteTimeGetCurrent() - started + Log.info( + "[EspressoLLMEngine] generated \(result.text.count) chars on ANE in " + + "\(String(format: "%.1f", elapsed))s (\(String(format: "%.1f", result.tokensPerSecond)) tok/s)" + ) + return result.text + } + + var isLoaded: Bool { model != nil } + + func unload() { + model = nil + } + + static func formatPrompt(user: String, system: String, modelName: String) -> String { + if modelName.lowercased().contains("qwen") { + return "<|im_start|>system\n\(system)<|im_end|>\n" + + "<|im_start|>user\n\(user)<|im_end|>\n" + + "<|im_start|>assistant\n" + } + return "System:\n\(system)\n\nUser:\n\(user)\n\nAssistant:\n" + } +} + +enum EspressoLLMError: LocalizedError { + case modelNotLoaded + case aneBackendUnavailable + + var errorDescription: String? { + switch self { + case .modelNotLoaded: return L("error.espresso_not_loaded") + case .aneBackendUnavailable: return L("error.espresso_ane_unavailable") + } + } +} diff --git a/Sources/Processing/TextProcessingOptions.swift b/Sources/Processing/TextProcessingOptions.swift index 6f1b6e25..d697c420 100644 --- a/Sources/Processing/TextProcessingOptions.swift +++ b/Sources/Processing/TextProcessingOptions.swift @@ -16,6 +16,8 @@ struct TextProcessingOptions { var customStylePrompt: String var llmModel: String var useRemoteLLM: Bool + var localLLMBackend: LocalLLMBackend + var espressoModelPath: String var remoteBaseURL: String var remoteAPIKey: String var remoteModel: String @@ -38,6 +40,8 @@ struct TextProcessingOptions { self.customStylePrompt = settings.customStylePrompt self.llmModel = settings.llmModel self.useRemoteLLM = settings.useRemoteLLM + self.localLLMBackend = settings.localLLMBackend + self.espressoModelPath = settings.espressoModelPath self.remoteBaseURL = settings.remoteBaseURL self.remoteAPIKey = settings.remoteAPIKey self.remoteModel = settings.remoteModel diff --git a/Sources/Processing/TextProcessor+Generation.swift b/Sources/Processing/TextProcessor+Generation.swift index a6d00542..592ced43 100644 --- a/Sources/Processing/TextProcessor+Generation.swift +++ b/Sources/Processing/TextProcessor+Generation.swift @@ -22,13 +22,24 @@ extension TextProcessor { ) } - await ensureModelLoaded(options.llmModel) - return try await llm.generate( - prompt: prompt, - systemPrompt: systemPrompt, - maxTokens: maxTokens, - temperature: temperature - ) + switch options.localLLMBackend { + case .mlx: + await ensureModelLoaded(options.llmModel) + return try await llm.generate( + prompt: prompt, + systemPrompt: systemPrompt, + maxTokens: maxTokens, + temperature: temperature + ) + case .espresso: + try await espressoLLM.loadModel(path: options.espressoModelPath) + return try await espressoLLM.generate( + prompt: prompt, + systemPrompt: systemPrompt, + maxTokens: maxTokens, + temperature: temperature + ) + } } func generateWithScreenImage( @@ -51,7 +62,9 @@ extension TextProcessor { func shouldUseScreenImage(options: TextProcessingOptions, image: CGImage?) -> Bool { guard image != nil else { return false } - guard options.screenContextMode == .multimodal, !options.useRemoteLLM else { return false } + guard options.screenContextMode == .multimodal, + !options.useRemoteLLM, + options.localLLMBackend == .mlx else { return false } return ScreenContextMode.supportsScreenImageContext(modelID: options.llmModel) } diff --git a/Sources/Processing/TextProcessor.swift b/Sources/Processing/TextProcessor.swift index dfa41937..8e5d2fa4 100644 --- a/Sources/Processing/TextProcessor.swift +++ b/Sources/Processing/TextProcessor.swift @@ -5,26 +5,36 @@ final class TextProcessor { static let defaultAllowsPreparedFallback = false let llm = LLMEngine() + let espressoLLM = EspressoLLMEngine() let vlm = VLMEngine() let remoteLLMClient = RemoteLLMClient() private let dictionary = PersonalDictionary.shared var isLLMReady: Bool { get async { if AppSettings.shared.useRemoteLLM { return true } - return await llm.isLoaded + switch AppSettings.shared.localLLMBackend { + case .mlx: return await llm.isLoaded + case .espresso: return await espressoLLM.isLoaded + } } } func unloadLLM() async { await llm.unload() + await espressoLLM.unload() await vlm.unload() } @discardableResult - func warmUpLLM(model: String) async -> Bool { + func warmUpLLM(model: String, backend: LocalLLMBackend, espressoModelPath: String) async -> Bool { if AppSettings.shared.useRemoteLLM { return true } do { - try await llm.loadModel(id: model) + switch backend { + case .mlx: + try await llm.loadModel(id: model) + case .espresso: + try await espressoLLM.loadModel(path: espressoModelPath) + } return true } catch { Log.error("[TextProcessor] LLM warmup failed: \(error.localizedDescription)") diff --git a/Sources/Resources/en.lproj/Localizable.strings b/Sources/Resources/en.lproj/Localizable.strings index 6c67a818..00ac93d4 100644 --- a/Sources/Resources/en.lproj/Localizable.strings +++ b/Sources/Resources/en.lproj/Localizable.strings @@ -192,7 +192,7 @@ "model.preload.speech" = "Preload dictation model on launch"; "model.preload.speech_help" = "Applies to already downloaded WhisperKit models. Use the Download button in Models first."; "model.preload.formatting" = "Preload formatting model on launch"; -"model.preload.formatting_help" = "Applies to local MLX models. Remote LLM providers are called when formatting starts."; +"model.preload.formatting_help" = "Applies to local MLX and Espresso models. Remote LLM providers are called when formatting starts."; "model.speech_recognition" = "Speech Recognition"; "model.apple_managed_by_system" = "Apple Speech uses the language and recognition services managed by macOS."; "model.text_formatting" = "Text Formatting (LLM)"; @@ -201,6 +201,10 @@ "model.family.gemma" = "Google Gemma - Lightweight"; "model.family.llama" = "Meta Llama - General Purpose"; "model.family.remote" = "Remote"; +"model.espresso.description" = "Run a compatible .esp bundle with Espresso's Apple Neural Engine backend."; +"model.espresso.no_bundle" = "No .esp bundle selected"; +"model.espresso.choose" = "Choose .esp Bundle…"; +"model.espresso.private_api_warning" = "Espresso uses private Apple Neural Engine APIs. It may break after macOS updates and is not eligible for Mac App Store distribution."; "model.custom_id_placeholder" = "Custom model ID (e.g. mlx-community/…)"; "model.active" = "Active"; "model.use" = "Use"; @@ -484,5 +488,7 @@ "model.asr_runtime_incomplete" = "Runtime files are incomplete"; "error.llm_not_loaded" = "The model is stored locally but is not loaded into memory. Run the action again and Utter will retry."; "error.llm_not_downloaded" = "The model files have not been downloaded. Open Settings → Models and confirm the data usage first."; +"error.espresso_not_loaded" = "The Espresso model bundle is not loaded. Select a valid .esp bundle in Settings → Models."; +"error.espresso_ane_unavailable" = "This Espresso bundle does not provide an Apple Neural Engine backend on this Mac."; "onboarding.download_notice" = "No download starts automatically. Review the size, then confirm if you want this local model."; "onboarding.download_size" = "Download (%@)"; diff --git a/Sources/Resources/zh-Hans.lproj/Localizable.strings b/Sources/Resources/zh-Hans.lproj/Localizable.strings index a1566d23..56e1ddad 100644 --- a/Sources/Resources/zh-Hans.lproj/Localizable.strings +++ b/Sources/Resources/zh-Hans.lproj/Localizable.strings @@ -192,7 +192,7 @@ "model.preload.speech" = "启动时预加载听写模型"; "model.preload.speech_help" = "仅对已下载的 WhisperKit 模型生效。请先在模型页点击下载按钮。"; "model.preload.formatting" = "启动时预加载修稿模型"; -"model.preload.formatting_help" = "仅对本地 MLX 模型生效。远程 LLM 会在整理开始时调用。"; +"model.preload.formatting_help" = "对本地 MLX 和 Espresso 模型生效。远程 LLM 会在整理开始时调用。"; "model.speech_recognition" = "语音识别"; "model.apple_managed_by_system" = "Apple 语音使用由 macOS 管理的语言与识别服务。"; "model.text_formatting" = "文本整理 (LLM)"; @@ -201,6 +201,10 @@ "model.family.gemma" = "Google Gemma - 轻量"; "model.family.llama" = "Meta Llama - 通用"; "model.family.remote" = "远程"; +"model.espresso.description" = "使用 Espresso 的 Apple 神经网络引擎后端运行兼容的 .esp 模型包。"; +"model.espresso.no_bundle" = "尚未选择 .esp 模型包"; +"model.espresso.choose" = "选择 .esp 模型包…"; +"model.espresso.private_api_warning" = "Espresso 使用 Apple 神经网络引擎私有 API,可能在 macOS 更新后失效,也无法通过 Mac App Store 审核。"; "model.custom_id_placeholder" = "自定义模型 ID(如 mlx-community/…)"; "model.active" = "当前"; "model.use" = "启用"; @@ -484,5 +488,7 @@ "model.asr_runtime_incomplete" = "运行文件不完整"; "error.llm_not_loaded" = "模型文件已在本地,但当前尚未加载到内存。请重新执行;Utter 会再次尝试加载"; "error.llm_not_downloaded" = "模型文件尚未下载。请前往 设置 → 模型,确认流量后下载"; +"error.espresso_not_loaded" = "Espresso 模型包尚未加载。请在 设置 → 模型 中选择有效的 .esp 模型包。"; +"error.espresso_ane_unavailable" = "这个 Espresso 模型包无法在此 Mac 上使用 Apple 神经网络引擎后端。"; "onboarding.download_notice" = "这里不会自动下载。请先确认体积,再决定是否下载这个本地模型。"; "onboarding.download_size" = "下载(%@)"; diff --git a/Sources/UI/ModelManagementFamilies.swift b/Sources/UI/ModelManagementFamilies.swift index fd5b67a2..8e4509c8 100644 --- a/Sources/UI/ModelManagementFamilies.swift +++ b/Sources/UI/ModelManagementFamilies.swift @@ -6,6 +6,7 @@ extension ModelManagementView { ForEach(ModelCatalog.ModelFamily.allCases, id: \.self) { family in familyButton(family) } + espressoFamilyButton remoteFamilyButton } .background(Color(nsColor: .controlBackgroundColor)) @@ -17,7 +18,9 @@ extension ModelManagementView { } func familyButton(_ family: ModelCatalog.ModelFamily) -> some View { - let isSelected = !settings.useRemoteLLM && selectedModelFamily == family + let isSelected = !settings.useRemoteLLM + && settings.localLLMBackend == .mlx + && selectedModelFamily == family return Button(action: { selectLocalFamily(family) }) { VStack(spacing: 2) { @@ -42,6 +45,23 @@ extension ModelManagementView { ) } + var espressoFamilyButton: some View { + let isSelected = !settings.useRemoteLLM && settings.localLLMBackend == .espresso + return Button(action: selectEspressoLLM) { + VStack(spacing: 2) { + Image(systemName: "neural.engine") + .font(.system(size: 14)) + Text("ANE") + .font(.system(size: 10, weight: isSelected ? .semibold : .medium)) + } + .frame(maxWidth: .infinity, minHeight: 44) + .contentShape(Rectangle()) + } + .buttonStyle(.plain) + .background(isSelected ? Color.accentColor.opacity(0.15) : Color.clear) + .foregroundStyle(isSelected ? Color.accentColor : Color.primary) + } + var remoteFamilyButton: some View { Button(action: selectRemoteLLM) { VStack(spacing: 2) { @@ -68,11 +88,25 @@ extension ModelManagementView { func selectLocalFamily(_ family: ModelCatalog.ModelFamily) { selectedModelFamily = family - if settings.useRemoteLLM { + let changedBackend = settings.useRemoteLLM || settings.localLLMBackend != .mlx + if changedBackend { + onUnloadLLM?() settings.useRemoteLLM = false - if catalog.llmModels.first(where: { $0.id == settings.llmModel })?.family == family { - onLoadLLM?() - } + settings.localLLMBackend = .mlx + } + if changedBackend, + catalog.llmModels.first(where: { $0.id == settings.llmModel })?.family == family { + onLoadLLM?() + } + } + + func selectEspressoLLM() { + guard settings.useRemoteLLM || settings.localLLMBackend != .espresso else { return } + onUnloadLLM?() + settings.useRemoteLLM = false + settings.localLLMBackend = .espresso + if !settings.espressoModelPath.isEmpty { + onLoadLLM?() } } diff --git a/Sources/UI/ModelManagementRows.swift b/Sources/UI/ModelManagementRows.swift index 0a3f739f..2973df4f 100644 --- a/Sources/UI/ModelManagementRows.swift +++ b/Sources/UI/ModelManagementRows.swift @@ -203,6 +203,7 @@ extension ModelManagementView { case .llm: onUnloadLLM?() settings.useRemoteLLM = false + settings.localLLMBackend = .mlx settings.llmModel = model.id if let family = model.family { selectedModelFamily = family diff --git a/Sources/UI/ModelManagementSections.swift b/Sources/UI/ModelManagementSections.swift index 46b928e7..c9f43769 100644 --- a/Sources/UI/ModelManagementSections.swift +++ b/Sources/UI/ModelManagementSections.swift @@ -133,6 +133,8 @@ extension ModelManagementView { if settings.useRemoteLLM { RemoteLLMConfigView() + } else if settings.localLLMBackend == .espresso { + espressoLLMSection } else { localLLMModelsSection } @@ -176,8 +178,38 @@ extension ModelManagementView { .controlSize(.small) } + var espressoLLMSection: some View { + VStack(alignment: .leading, spacing: 10) { + Label("Espresso", systemImage: "neural.engine") + .font(.system(size: 12, weight: .semibold)) + + Text(L("model.espresso.description")) + .font(.system(size: 11)) + .foregroundStyle(.secondary) + + HStack(spacing: 8) { + Text(settings.espressoModelPath.isEmpty + ? L("model.espresso.no_bundle") + : settings.espressoModelPath) + .font(.system(size: 10, design: .monospaced)) + .foregroundStyle(settings.espressoModelPath.isEmpty ? .secondary : .primary) + .lineLimit(2) + .textSelection(.enabled) + Spacer() + Button(L("model.espresso.choose")) { + chooseEspressoBundle() + } + .controlSize(.small) + } + + Label(L("model.espresso.private_api_warning"), systemImage: "exclamationmark.triangle.fill") + .font(.system(size: 10)) + .foregroundStyle(.orange) + } + } + func syncSelectedFamilyFromActiveModel() { - guard !settings.useRemoteLLM else { return } + guard !settings.useRemoteLLM, settings.localLLMBackend == .mlx else { return } if let family = catalog.llmModels.first(where: { $0.id == settings.llmModel })?.family { selectedModelFamily = family } diff --git a/Sources/UI/ModelManagementView.swift b/Sources/UI/ModelManagementView.swift index 64fe0c1c..95ed2f8e 100644 --- a/Sources/UI/ModelManagementView.swift +++ b/Sources/UI/ModelManagementView.swift @@ -1,5 +1,6 @@ import SwiftUI import AppKit +import ESPRuntime struct ModelManagementView: View { @EnvironmentObject var settings: AppSettings @@ -55,6 +56,7 @@ struct ModelManagementView: View { syncSelectedFamilyFromActiveModel() } .onChange(of: settings.llmModel) { _, _ in syncSelectedFamilyFromActiveModel() } + .onChange(of: settings.localLLMBackend) { _, _ in syncSelectedFamilyFromActiveModel() } .onChange(of: settings.localASRPythonPath) { _, _ in onUnloadLocalASR?() } .onChange(of: settings.mimoASRRepoPath) { _, _ in onUnloadLocalASR?() } .onChange(of: settings.qwenASRModel) { _, _ in onUnloadLocalASR?() } @@ -163,6 +165,28 @@ extension ModelManagementView { } } + func chooseEspressoBundle() { + let panel = NSOpenPanel() + panel.canChooseFiles = false + panel.canChooseDirectories = true + panel.allowsMultipleSelection = false + panel.canCreateDirectories = false + panel.message = L("model.espresso.choose") + guard panel.runModal() == .OK, let url = panel.url else { return } + + do { + _ = try ESPRuntimeBundle.open(at: url) + onUnloadLLM?() + settings.espressoModelPath = url.path + settings.localLLMBackend = .espresso + settings.useRemoteLLM = false + onLoadLLM?() + } catch { + importErrorMessage = error.localizedDescription + showImportError = true + } + } + private func isValidWhisperFolder(_ url: URL) -> Bool { ModelStorage.whisperModelIsComplete(at: url) } diff --git a/Tests/OpenTypeTests/ConfigurationTests.swift b/Tests/OpenTypeTests/ConfigurationTests.swift index be79b797..cc3366ac 100644 --- a/Tests/OpenTypeTests/ConfigurationTests.swift +++ b/Tests/OpenTypeTests/ConfigurationTests.swift @@ -253,6 +253,37 @@ final class ConfigurationTests: XCTestCase { XCTAssertFalse(settings.developerInterfaceEnabled) } + func testLocalLLMBackendDefaultsToMLXAndPersistsEspressoSelection() { + let (defaults, suiteName) = makeIsolatedDefaults() + defer { defaults.removePersistentDomain(forName: suiteName) } + + let settings = AppSettings(defaults: defaults) + XCTAssertEqual(settings.localLLMBackend, .mlx) + XCTAssertTrue(settings.espressoModelPath.isEmpty) + + settings.localLLMBackend = .espresso + settings.espressoModelPath = "/tmp/qwen.esp" + + let reloaded = AppSettings(defaults: defaults) + XCTAssertEqual(reloaded.localLLMBackend, .espresso) + XCTAssertEqual(reloaded.espressoModelPath, "/tmp/qwen.esp") + } + + func testEspressoPromptUsesQwenChatTemplate() { + let prompt = EspressoLLMEngine.formatPrompt( + user: "整理这句话", + system: "只输出结果", + modelName: "Qwen2.5-0.5B-Instruct" + ) + + XCTAssertEqual( + prompt, + "<|im_start|>system\n只输出结果<|im_end|>\n" + + "<|im_start|>user\n整理这句话<|im_end|>\n" + + "<|im_start|>assistant\n" + ) + } + func testDeveloperHTTPTokenCanBeReset() { let (defaults, suiteName) = makeIsolatedDefaults() defer { defaults.removePersistentDomain(forName: suiteName) } From 10bf15886ef5fefaae3d37d443ffc97ef5a4cdd6 Mon Sep 17 00:00:00 2001 From: idevlab Date: Sat, 29 Aug 2026 16:01:14 +0800 Subject: [PATCH 02/14] Document Espresso ANE verification Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a03e89-5ac5-732e-9da8-9eafeb591994 --- .../changes/2026-08-29-espresso-ane/intent.md | 39 +++++++++++++++ .../changes/2026-08-29-espresso-ane/plan.md | 24 +++++++++ .../changes/2026-08-29-espresso-ane/spec.md | 49 +++++++++++++++++++ .../2026-08-29-espresso-ane/state.json | 29 +++++++++++ .../2026-08-29-espresso-ane/verification.md | 35 +++++++++++++ 5 files changed, 176 insertions(+) create mode 100644 docs/sdlc/changes/2026-08-29-espresso-ane/intent.md create mode 100644 docs/sdlc/changes/2026-08-29-espresso-ane/plan.md create mode 100644 docs/sdlc/changes/2026-08-29-espresso-ane/spec.md create mode 100644 docs/sdlc/changes/2026-08-29-espresso-ane/state.json create mode 100644 docs/sdlc/changes/2026-08-29-espresso-ane/verification.md diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/intent.md b/docs/sdlc/changes/2026-08-29-espresso-ane/intent.md new file mode 100644 index 00000000..c4f1dd95 --- /dev/null +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/intent.md @@ -0,0 +1,39 @@ +# Intent: Add a selectable Espresso ANE inference backend + +## Problem + +Utter's local LLM processing is limited to MLX, so users cannot choose an +Apple Neural Engine runtime for local post-processing. + +## Outcome + +The model settings let users choose Espresso, select an `.esp` model bundle, +and route local warmup and text generation through Espresso's ANE runtime. + +## Scope + +This change covers the local LLM backend setting, Espresso bundle selection, +runtime dispatch, persistence, localization, and dependency integration. It does +not create or download Espresso bundles, alter speech recognition, or claim App +Store compatibility for Espresso's private ANE API. + +## Constraints + +- Utter remains local-first and sends no new data to a remote service. +- The existing MLX backend remains the default. +- Espresso uses private Apple APIs and may fail across macOS or SoC revisions. +- Model preparation remains an upstream Espresso workflow. + +## Acceptance criteria + +- MLX and Espresso are selectable local LLM backends and the choice persists. +- A valid `.esp` directory can be selected while malformed bundles are rejected. +- Espresso selection routes warmup and generation through the Espresso engine. +- Existing local MLX and remote LLM behavior remains covered by passing tests. +- Real text generation succeeds on a supported Apple Silicon/macOS combination. + +## Open questions + +Which macOS 27 and M5 combinations Espresso will support remains an upstream +compatibility question. Real inference on the available M5 Max/macOS 27 host is +currently blocked by the ANE compiler. diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md b/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md new file mode 100644 index 00000000..d67bdf7d --- /dev/null +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md @@ -0,0 +1,24 @@ +# Plan: Add a selectable Espresso ANE inference backend + +## Work items + +- [x] Add and resolve the Espresso Swift package products. +- [x] Persist the local backend and Espresso bundle path with MLX defaults. +- [x] Add bundle selection, validation, localization, and private-API warning. +- [x] Route local warmup, readiness, unload, and generation to Espresso. +- [x] Add focused settings and prompt-format tests. +- [x] Exercise a prepared GPT-2 bundle on the available ANE host. +- [x] Merge current `origin/main` and preserve both dependency sets. + +## Verification plan + +- [x] `bash scripts/ci-basic-checks.sh` +- [x] `swift test` +- [x] Targeted `ConfigurationTests` +- [x] Release-style app build +- [x] Real GPT-2 `.esp` inspection and generation attempt + +## Human gates + +A maintainer must decide whether to merge while M5 Max/macOS 27 inference is +blocked upstream. Publishing or releasing remains a separate approval. diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/spec.md b/docs/sdlc/changes/2026-08-29-espresso-ane/spec.md new file mode 100644 index 00000000..f3476860 --- /dev/null +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/spec.md @@ -0,0 +1,49 @@ +# Spec: Add a selectable Espresso ANE inference backend + +## Context + +`AppSettings` owns persisted model choices, `ModelManagementView` owns model +selection, and `TextProcessor` owns LLM dispatch. Existing local generation uses +`LLMEngine` and MLX; remote generation remains independent. Espresso packages +models and tokenizers as `.esp` directories consumed by `ESPRuntime` and +`RealModelInference`. + +## Design + +Add a persisted `LocalLLMBackend` choice with MLX as the default and a persisted +Espresso bundle path. The models UI validates a selected directory with +`ESPRuntimeBundle.open`, displays the private-API warning, and requests warmup. + +`TextProcessor` owns one `EspressoLLMEngine` actor alongside the MLX engine and +dispatches warmup, readiness, unloading, and generation according to the +captured processing options. The actor retains Espresso's move-only inference +engine, verifies that the bundle resolves to the private ANE backend, applies a +Qwen chat template when appropriate, and returns generated text through the +existing processing pipeline. Remote LLM behavior is unchanged. + +## Safety and failure modes + +The model and prompt remain on-device. Espresso depends on private ANE APIs, so +OS or hardware changes can reject generated ANE programs even when bundle +metadata is valid. Such failures propagate through the existing model-load or +generation error path; MLX remains available as the default and rollback. +Selecting a missing or malformed bundle does not replace the current setting. + +The UI warning explicitly states the private-API and App Store limitation. No +attempt is made to silently fall back from Espresso to MLX because that would +misrepresent the selected execution backend. + +## Test strategy + +Persistence and prompt formatting have focused unit tests. The complete Swift +suite and repository checks cover existing paths and package integration. A +release-style app build checks dependency and Metal/resource packaging. Real +inference is exercised with a prepared GPT-2 `.esp` bundle and recorded even if +the host's private ANE compiler rejects it. + +## Rollout and rollback + +Ship behind an explicit non-default setting. Stop rollout if Espresso cannot +compile a real bundle on the supported release environment. Users can return to +MLX immediately; repository rollback removes the Espresso dependency, engine, +settings fields, and settings UI without migrating stored user data. diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/state.json b/docs/sdlc/changes/2026-08-29-espresso-ane/state.json new file mode 100644 index 00000000..a676b6c1 --- /dev/null +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/state.json @@ -0,0 +1,29 @@ +{ + "schemaVersion": 1, + "id": "2026-08-29-espresso-ane", + "title": "Add a selectable Espresso ANE inference backend", + "risk": "high", + "status": "verified", + "owners": [ + "repository maintainer" + ], + "acceptanceCriteria": [ + "Users can select MLX or Espresso for local LLM processing and the selection persists.", + "Users can select a valid Espresso bundle and invalid bundles are rejected before loading.", + "When Espresso is selected, local text generation and model warmup use its ANE runtime.", + "The existing MLX and remote LLM paths continue to pass the automated test suite.", + "A real Espresso bundle can generate text on a supported Apple Silicon and macOS combination." + ], + "governedPaths": [ + "Package.swift", + "Package.resolved", + "Sources/", + "Tests/" + ], + "artifacts": { + "intent": "intent.md", + "spec": "spec.md", + "plan": "plan.md", + "verification": "verification.md" + } +} diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md b/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md new file mode 100644 index 00000000..4e796a19 --- /dev/null +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md @@ -0,0 +1,35 @@ +# Verification: Add a selectable Espresso ANE inference backend + +## Evidence + +| Check | Result | Evidence | +|---|---|---| +| `bash scripts/ci-basic-checks.sh` | Pass | Repository basic checks completed after merging `origin/main` | +| `swift test` | Pass | 566 tests passed, 8 skipped, 0 failures | +| Targeted `ConfigurationTests` | Pass | 35 tests passed, 0 failures | +| `DEVELOPER_DIR=/Applications/Xcode-beta.app/Contents/Developer bash scripts/build-app.sh --app-only --sign=-` | Pass | Release app and CLI built, assembled, ad-hoc signed, and passed release artifact verification | +| Espresso 0.9.0 GPT-2 generation | Fail | M5 Max/macOS 27 ANE compiler returned code 10, `verifyBundleAtPath: invalid model`, while compiling layer 0 attention | +| Espresso main `eafb33d` GPT-2 generation | Fail | Latest upstream source produced the same ANE code 10 on the same host | + +## Acceptance criteria + +- Backend selection and persistence — pass; focused settings test and full suite. +- Bundle selection and malformed-bundle rejection — pass at metadata validation level through `ESPRuntimeBundle.open`. +- Espresso warmup and generation dispatch — pass by code path and build coverage. +- Existing MLX and remote behavior — pass; complete suite has no failures. +- Real generation on a supported host — blocked on the available M5 Max/macOS 27 host; both the pinned release and upstream main fail in Apple's private ANE compiler. + +## Residual risk + +Espresso relies on a private ANE interface whose generated programs are rejected +on the available M5 Max/macOS 27 environment. Bundle inspection succeeds, so the +failure is only discovered during ANE kernel compilation. The repository +maintainer owns the decision to wait for upstream compatibility, constrain the +supported hardware/OS matrix, or accept the experimental backend. A real-window +light/dark UI pass has not yet been recorded. + +## Decision + +Implementation and regression checks are ready for review, but real inference +acceptance is blocked on the tested host. Do not describe the backend as runtime- +verified on M5 Max/macOS 27 or merge without explicit acceptance of this risk. From 11318c0300cc61e8600cd6c8f486c50787f9641a Mon Sep 17 00:00:00 2001 From: idevlab Date: Sun, 30 Aug 2026 14:14:33 +0800 Subject: [PATCH 03/14] Handle Espresso runtime failures safely Amp-Thread-ID: https://ampcode.com/threads/T-01a03e89-5ac5-732e-9da8-9eafeb591994 Co-authored-by: Amp --- Sources/App/VoicePipeline+Models.swift | 39 +++++++++-- Sources/App/VoicePipeline+Processing.swift | 3 +- Sources/App/VoicePipeline.swift | 1 + Sources/LLM/EspressoLLMEngine.swift | 64 +++++++++++++------ Sources/Processing/TextProcessor.swift | 30 +++++---- .../Resources/en.lproj/Localizable.strings | 1 + .../zh-Hans.lproj/Localizable.strings | 1 + Tests/OpenTypeTests/ConfigurationTests.swift | 9 +++ .../changes/2026-08-29-espresso-ane/plan.md | 1 + .../2026-08-29-espresso-ane/verification.md | 5 +- 10 files changed, 118 insertions(+), 36 deletions(-) diff --git a/Sources/App/VoicePipeline+Models.swift b/Sources/App/VoicePipeline+Models.swift index 0e9803f2..14a62ddd 100644 --- a/Sources/App/VoicePipeline+Models.swift +++ b/Sources/App/VoicePipeline+Models.swift @@ -10,6 +10,7 @@ extension VoicePipeline { } func unloadLLM() { + formattingPreloadGeneration += 1 processingTask?.cancel() processingTask = nil replacementTask?.cancel() @@ -18,6 +19,8 @@ extension VoicePipeline { if appState.phase == .processing { appState.phase = .idle appState.statusMessage = L("status.ready") + } else if appState.statusMessage == L("pipeline.loading_llm") { + appState.statusMessage = L("status.ready") } appState.llmModelReady = false Task { await textProcessor.unloadLLM() } @@ -34,6 +37,8 @@ extension VoicePipeline { } func preloadFormattingModel(showFailureInStatus: Bool) async { + formattingPreloadGeneration += 1 + let preloadGeneration = formattingPreloadGeneration guard !appState.settings.useRemoteLLM else { appState.llmModelReady = true return @@ -71,13 +76,21 @@ extension VoicePipeline { catalog.updateLLMStatus(model, status: .loading, detail: L("model.loading")) } - let loaded = await textProcessor.warmUpLLM( + let warmup = await textProcessor.warmUpLLM( model: model, backend: backend, espressoModelPath: espressoPath ) - let ready = await textProcessor.isLLMReady - appState.llmModelReady = loaded && ready + guard preloadGeneration == formattingPreloadGeneration, + formattingSelectionMatches(backend: backend, model: model, espressoPath: espressoPath) else { + return + } + let ready = warmup.loaded ? await textProcessor.isLLMReady(for: backend) : false + guard preloadGeneration == formattingPreloadGeneration, + formattingSelectionMatches(backend: backend, model: model, espressoPath: espressoPath) else { + return + } + appState.llmModelReady = warmup.loaded && ready if appState.llmModelReady { if backend == .mlx { @@ -90,7 +103,25 @@ extension VoicePipeline { catalog.updateLLMStatus(model, status: .error(L("pipeline.model_load_failed"))) } Log.info("[VoicePipeline] LLM warmup failed, will retry on demand") - appState.statusMessage = showFailureInStatus ? L("pipeline.model_load_failed") : L("status.ready") + let message = backend == .espresso + ? (warmup.errorMessage ?? L("error.espresso_runtime_failed")) + : L("pipeline.model_load_failed") + appState.statusMessage = showFailureInStatus ? message : L("status.ready") + } + } + + private func formattingSelectionMatches( + backend: LocalLLMBackend, + model: String, + espressoPath: String + ) -> Bool { + let settings = appState.settings + guard !settings.useRemoteLLM, settings.localLLMBackend == backend else { return false } + switch backend { + case .mlx: + return settings.llmModel.trimmingCharacters(in: .whitespacesAndNewlines) == model + case .espresso: + return settings.espressoModelPath.trimmingCharacters(in: .whitespacesAndNewlines) == espressoPath } } diff --git a/Sources/App/VoicePipeline+Processing.swift b/Sources/App/VoicePipeline+Processing.swift index 1e767153..b9e3489f 100644 --- a/Sources/App/VoicePipeline+Processing.swift +++ b/Sources/App/VoicePipeline+Processing.swift @@ -253,7 +253,8 @@ extension VoicePipeline { let finalText = output.text guard !finalText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { Log.info("[VoicePipeline] skipping empty final text") - showErrorHint(L("error.operation_failed")) + let espressoFailure = await textProcessor.consumeEspressoFailureMessage() + showErrorHint(espressoFailure ?? L("error.operation_failed")) return } diff --git a/Sources/App/VoicePipeline.swift b/Sources/App/VoicePipeline.swift index dc860f54..13ed914b 100644 --- a/Sources/App/VoicePipeline.swift +++ b/Sources/App/VoicePipeline.swift @@ -20,6 +20,7 @@ final class VoicePipeline { var replacementTask: Task? var hideOverlayTask: Task? var recordingTargetApp: NSRunningApplication? + var formattingPreloadGeneration = 0 var currentEngine: (any SpeechEngine)? { switch appState.settings.speechEngine { diff --git a/Sources/LLM/EspressoLLMEngine.swift b/Sources/LLM/EspressoLLMEngine.swift index 20f85915..11f63768 100644 --- a/Sources/LLM/EspressoLLMEngine.swift +++ b/Sources/LLM/EspressoLLMEngine.swift @@ -16,26 +16,32 @@ actor EspressoLLMEngine { } private var model: LoadedModel? + private var lastFailureMessage: String? func loadModel(path: String) throws { let expandedPath = NSString(string: path).expandingTildeInPath let url = URL(fileURLWithPath: expandedPath, isDirectory: true).standardizedFileURL guard model?.path != url.path else { return } - Log.info("[EspressoLLMEngine] loading bundle: \(url.path)") - let bundle = try ESPRuntimeBundle.open(at: url) - let selection = try ESPRuntimeRunner.resolve(bundle: bundle) - guard selection.backend == .anePrivate else { - throw EspressoLLMError.aneBackendUnavailable - } + lastFailureMessage = nil + Log.info("[EspressoLLMEngine] loading bundle: \(url.lastPathComponent)") + do { + let bundle = try ESPRuntimeBundle.open(at: url) + let selection = try ESPRuntimeRunner.resolve(bundle: bundle) + guard selection.backend == .anePrivate else { + throw EspressoLLMError.aneBackendUnavailable + } - let engine = try RealModelInferenceEngine.build( - config: bundle.config, - weightDir: bundle.archive.weightsURL.path, - tokenizerDir: bundle.archive.tokenizerURL.path - ) - model = LoadedModel(path: url.path, name: bundle.config.name, engine: engine) - Log.info("[EspressoLLMEngine] bundle ready for ANE inference") + let engine = try RealModelInferenceEngine.build( + config: bundle.config, + weightDir: bundle.archive.weightsURL.path, + tokenizerDir: bundle.archive.tokenizerURL.path + ) + model = LoadedModel(path: url.path, name: bundle.config.name, engine: engine) + Log.info("[EspressoLLMEngine] bundle ready for ANE inference") + } catch { + throw recordFailure(error) + } } func generate( @@ -45,17 +51,23 @@ actor EspressoLLMEngine { temperature: Double ) throws -> String { guard let model else { throw EspressoLLMError.modelNotLoaded } + lastFailureMessage = nil let input = Self.formatPrompt( user: prompt, system: systemPrompt, modelName: model.name ) let started = CFAbsoluteTimeGetCurrent() - let result = try model.engine.generate( - prompt: input, - maxTokens: maxTokens, - temperature: Float(temperature) - ) + let result: GenerationResult + do { + result = try model.engine.generate( + prompt: input, + maxTokens: maxTokens, + temperature: Float(temperature) + ) + } catch { + throw recordFailure(error) + } let elapsed = CFAbsoluteTimeGetCurrent() - started Log.info( "[EspressoLLMEngine] generated \(result.text.count) chars on ANE in " @@ -68,6 +80,20 @@ actor EspressoLLMEngine { func unload() { model = nil + lastFailureMessage = nil + } + + func consumeLastFailureMessage() -> String? { + defer { lastFailureMessage = nil } + return lastFailureMessage + } + + private func recordFailure(_ error: Error) -> EspressoLLMError { + let mapped = error as? EspressoLLMError ?? .runtimeFailure + Log.sensitive("[EspressoLLMEngine] ANE runtime detail: \(error.localizedDescription)") + Log.error("[EspressoLLMEngine] \(mapped.localizedDescription)") + lastFailureMessage = mapped.localizedDescription + return mapped } static func formatPrompt(user: String, system: String, modelName: String) -> String { @@ -83,11 +109,13 @@ actor EspressoLLMEngine { enum EspressoLLMError: LocalizedError { case modelNotLoaded case aneBackendUnavailable + case runtimeFailure var errorDescription: String? { switch self { case .modelNotLoaded: return L("error.espresso_not_loaded") case .aneBackendUnavailable: return L("error.espresso_ane_unavailable") + case .runtimeFailure: return L("error.espresso_runtime_failed") } } } diff --git a/Sources/Processing/TextProcessor.swift b/Sources/Processing/TextProcessor.swift index fde46c22..83e48283 100644 --- a/Sources/Processing/TextProcessor.swift +++ b/Sources/Processing/TextProcessor.swift @@ -9,13 +9,11 @@ final class TextProcessor { let vlm = VLMEngine() let remoteLLMClient = RemoteLLMClient() private let dictionary = PersonalDictionary.shared - var isLLMReady: Bool { - get async { - if AppSettings.shared.useRemoteLLM { return true } - switch AppSettings.shared.localLLMBackend { - case .mlx: return await llm.isLoaded - case .espresso: return await espressoLLM.isLoaded - } + + func isLLMReady(for backend: LocalLLMBackend) async -> Bool { + switch backend { + case .mlx: return await llm.isLoaded + case .espresso: return await espressoLLM.isLoaded } } @@ -26,8 +24,11 @@ final class TextProcessor { } @discardableResult - func warmUpLLM(model: String, backend: LocalLLMBackend, espressoModelPath: String) async -> Bool { - if AppSettings.shared.useRemoteLLM { return true } + func warmUpLLM( + model: String, + backend: LocalLLMBackend, + espressoModelPath: String + ) async -> (loaded: Bool, errorMessage: String?) { do { switch backend { case .mlx: @@ -35,13 +36,20 @@ final class TextProcessor { case .espresso: try await espressoLLM.loadModel(path: espressoModelPath) } - return true + return (true, nil) } catch { Log.error("[TextProcessor] LLM warmup failed: \(error.localizedDescription)") - return false + if backend == .espresso { + _ = await espressoLLM.consumeLastFailureMessage() + } + return (false, error.localizedDescription) } } + func consumeEspressoFailureMessage() async -> String? { + await espressoLLM.consumeLastFailureMessage() + } + func basicClean( text: String, inputLanguage: InputLanguage = .auto, diff --git a/Sources/Resources/en.lproj/Localizable.strings b/Sources/Resources/en.lproj/Localizable.strings index d0abeee1..64f2dcbe 100644 --- a/Sources/Resources/en.lproj/Localizable.strings +++ b/Sources/Resources/en.lproj/Localizable.strings @@ -464,6 +464,7 @@ "error.load_failed" = "Model loading failed: %@"; "error.network_request_failed" = "Network request failed — please try again"; "error.operation_failed" = "Operation failed — please try again"; +"error.espresso_runtime_failed" = "Espresso could not run this model on the ANE. Switch to MLX or try a supported macOS and device."; "error.volc_not_configured" = "Doubao ASR not configured — enter API credentials in Settings → Models"; "error.volc_invalid_endpoint" = "Invalid ASR endpoint URL"; "error.volc_audio_conversion" = "Audio conversion to PCM 16kHz failed"; diff --git a/Sources/Resources/zh-Hans.lproj/Localizable.strings b/Sources/Resources/zh-Hans.lproj/Localizable.strings index b7ca6305..0900b53a 100644 --- a/Sources/Resources/zh-Hans.lproj/Localizable.strings +++ b/Sources/Resources/zh-Hans.lproj/Localizable.strings @@ -464,6 +464,7 @@ "error.load_failed" = "模型加载失败: %@"; "error.network_request_failed" = "网络请求失败,请稍后重试"; "error.operation_failed" = "操作失败,请重试"; +"error.espresso_runtime_failed" = "Espresso 无法在这台设备的 ANE 上运行此模型。请切换到 MLX,或改用受支持的 macOS 与设备。"; "error.volc_not_configured" = "豆包语音识别未配置 — 请在 设置 → 模型 中填写 API 凭据"; "error.volc_invalid_endpoint" = "语音识别接口地址无效"; "error.volc_audio_conversion" = "音频转换为 PCM 16kHz 失败"; diff --git a/Tests/OpenTypeTests/ConfigurationTests.swift b/Tests/OpenTypeTests/ConfigurationTests.swift index 3050f2e3..e24736d9 100644 --- a/Tests/OpenTypeTests/ConfigurationTests.swift +++ b/Tests/OpenTypeTests/ConfigurationTests.swift @@ -294,6 +294,15 @@ final class ConfigurationTests: XCTestCase { ) } + func testEspressoRuntimeFailureSuggestsMLXFallback() { + XCTAssertTrue( + Loc.string("error.espresso_runtime_failed", language: .english).contains("MLX") + ) + XCTAssertTrue( + Loc.string("error.espresso_runtime_failed", language: .chinese).contains("MLX") + ) + } + func testDeveloperHTTPTokenCanBeReset() { let (defaults, suiteName) = makeIsolatedDefaults() defer { defaults.removePersistentDomain(forName: suiteName) } diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md b/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md index d67bdf7d..a1930287 100644 --- a/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md @@ -9,6 +9,7 @@ - [x] Add focused settings and prompt-format tests. - [x] Exercise a prepared GPT-2 bundle on the available ANE host. - [x] Merge current `origin/main` and preserve both dependency sets. +- [x] Resolve independent-review findings for error visibility, stale preload state, and path privacy. ## Verification plan diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md b/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md index 4e796a19..4c93db5f 100644 --- a/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md @@ -5,9 +5,10 @@ | Check | Result | Evidence | |---|---|---| | `bash scripts/ci-basic-checks.sh` | Pass | Repository basic checks completed after merging `origin/main` | -| `swift test` | Pass | 566 tests passed, 8 skipped, 0 failures | -| Targeted `ConfigurationTests` | Pass | 35 tests passed, 0 failures | +| `swift test` | Pass | 567 XCTest tests passed, 8 skipped, plus 1 Swift Testing test passed | +| Targeted `ConfigurationTests` | Pass | 36 tests passed, 0 failures | | `DEVELOPER_DIR=/Applications/Xcode-beta.app/Contents/Developer bash scripts/build-app.sh --app-only --sign=-` | Pass | Release app and CLI built, assembled, ad-hoc signed, and passed release artifact verification | +| Independent high-risk review | Completed; findings addressed | Review found generic ANE errors, stale preload publication, public path logging, and stale loading status; fixes surface localized guidance, use a preload generation token, clear owned loading state, and keep path-bearing details private | | Espresso 0.9.0 GPT-2 generation | Fail | M5 Max/macOS 27 ANE compiler returned code 10, `verifyBundleAtPath: invalid model`, while compiling layer 0 attention | | Espresso main `eafb33d` GPT-2 generation | Fail | Latest upstream source produced the same ANE code 10 on the same host | From ef4872e0bba8c7ae6c65983353b9ca64dbd51c0c Mon Sep 17 00:00:00 2001 From: idevlab Date: Sun, 30 Aug 2026 14:47:07 +0800 Subject: [PATCH 04/14] Fix Espresso linkage on Swift 6.2 --- Package.resolved | 7 +++---- Package.swift | 5 ++++- docs/sdlc/changes/2026-08-29-espresso-ane/plan.md | 2 ++ docs/sdlc/changes/2026-08-29-espresso-ane/spec.md | 11 +++++++++++ .../2026-08-29-espresso-ane/verification.md | 14 +++++++++----- 5 files changed, 29 insertions(+), 10 deletions(-) diff --git a/Package.resolved b/Package.resolved index e10535cb..df2b5d2d 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "df115d1ba7786c21c91c720b1530dcd190051b18e6df521ee04e7a4f435221a5", + "originHash" : "113df5803768310e85daa76109055f7d79bff777eab44985d49c77f715d46341", "pins" : [ { "identity" : "argmax-oss-swift", @@ -13,10 +13,9 @@ { "identity" : "espresso", "kind" : "remoteSourceControl", - "location" : "https://github.com/christopherkarani/Espresso.git", + "location" : "https://github.com/IchenDEV/Espresso.git", "state" : { - "revision" : "a8629bbcb4d4ae7179b0c3995c36c868e690df87", - "version" : "0.9.0" + "revision" : "f3603c7014b3b82c9df036c2e91185e1e32b2d81" } }, { diff --git a/Package.swift b/Package.swift index bf7b6557..d5111763 100644 --- a/Package.swift +++ b/Package.swift @@ -13,7 +13,10 @@ let package = Package( ], dependencies: [ .package(url: "https://github.com/argmaxinc/argmax-oss-swift.git", from: "1.0.0"), - .package(url: "https://github.com/christopherkarani/Espresso.git", from: "0.9.0"), + .package( + url: "https://github.com/IchenDEV/Espresso.git", + revision: "f3603c7014b3b82c9df036c2e91185e1e32b2d81" + ), .package(url: "https://github.com/Blaizzy/mlx-audio-swift.git", exact: "0.1.3"), .package(url: "https://github.com/huggingface/swift-transformers", from: "1.3.3"), .package(url: "https://github.com/ml-explore/mlx-swift-lm", exact: "3.31.4"), diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md b/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md index a1930287..881f86b2 100644 --- a/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md @@ -10,6 +10,7 @@ - [x] Exercise a prepared GPT-2 bundle on the available ANE host. - [x] Merge current `origin/main` and preserve both dependency sets. - [x] Resolve independent-review findings for error visibility, stale preload state, and path privacy. +- [x] Diagnose the Xcode 26.6 Release linker failure and pin the minimal Espresso metadata fix. ## Verification plan @@ -18,6 +19,7 @@ - [x] Targeted `ConfigurationTests` - [x] Release-style app build - [x] Real GPT-2 `.esp` inspection and generation attempt +- [ ] GitHub Xcode 26.6 release-style app build after the dependency pin ## Human gates diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/spec.md b/docs/sdlc/changes/2026-08-29-espresso-ane/spec.md index f3476860..b7551a28 100644 --- a/docs/sdlc/changes/2026-08-29-espresso-ane/spec.md +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/spec.md @@ -21,6 +21,12 @@ engine, verifies that the bundle resolves to the private ANE backend, applies a Qwen chat template when appropriate, and returns generated text through the existing processing pipeline. Remote LLM behavior is unchanged. +Pin Espresso to the reviewed `v0.9.0` source plus a three-line Swift 6.2 +compatibility patch. The patch gives three internal compiled-kernel holder +types package visibility so Xcode 26.6 can resolve metadata references emitted +across the `RealModelInference`, `ESPRuntime`, and app modules. It changes no +runtime logic or public API and avoids taking unrelated upstream `main` changes. + ## Safety and failure modes The model and prompt remain on-device. Espresso depends on private ANE APIs, so @@ -33,6 +39,11 @@ The UI warning explicitly states the private-API and App Store limitation. No attempt is made to silently fall back from Espresso to MLX because that would misrepresent the selected execution backend. +The patched dependency is pinned by commit rather than a moving branch. Rollback +returns the package URL and version requirement to upstream `v0.9.0` once an +equivalent fix is released there, or removes Espresso with the rest of this +experimental backend. + ## Test strategy Persistence and prompt formatting have focused unit tests. The complete Swift diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md b/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md index 4c93db5f..a4717c42 100644 --- a/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md @@ -5,9 +5,11 @@ | Check | Result | Evidence | |---|---|---| | `bash scripts/ci-basic-checks.sh` | Pass | Repository basic checks completed after merging `origin/main` | -| `swift test` | Pass | 567 XCTest tests passed, 8 skipped, plus 1 Swift Testing test passed | +| `DEVELOPER_DIR=/Applications/Xcode-beta.app/Contents/Developer swift test` | Pass | 567 XCTest tests passed, 8 skipped, plus 1 Swift Testing test passed after the dependency pin | | Targeted `ConfigurationTests` | Pass | 36 tests passed, 0 failures | -| `DEVELOPER_DIR=/Applications/Xcode-beta.app/Contents/Developer bash scripts/build-app.sh --app-only --sign=-` | Pass | Release app and CLI built, assembled, ad-hoc signed, and passed release artifact verification | +| `DEVELOPER_DIR=/Applications/Xcode-beta.app/Contents/Developer bash scripts/build-app.sh --app-only --sign=-` | Pass | Pinned-fork Release app and CLI built, assembled, ad-hoc signed, and passed release artifact verification | +| GitHub run `33296463655`, Xcode 26.6 Release build | Fail; diagnosed | Swift 6.2 emitted cross-module references to three internal `RealModelInferenceEngine.Compiled*` metadata symbols, then failed final arm64 linking | +| Pinned Espresso commit `f3603c7` symbol probe | Pass | The three metadata symbols are emitted as external after changing only the holder types from internal to package visibility | | Independent high-risk review | Completed; findings addressed | Review found generic ANE errors, stale preload publication, public path logging, and stale loading status; fixes surface localized guidance, use a preload generation token, clear owned loading state, and keep path-bearing details private | | Espresso 0.9.0 GPT-2 generation | Fail | M5 Max/macOS 27 ANE compiler returned code 10, `verifyBundleAtPath: invalid model`, while compiling layer 0 attention | | Espresso main `eafb33d` GPT-2 generation | Fail | Latest upstream source produced the same ANE code 10 on the same host | @@ -31,6 +33,8 @@ light/dark UI pass has not yet been recorded. ## Decision -Implementation and regression checks are ready for review, but real inference -acceptance is blocked on the tested host. Do not describe the backend as runtime- -verified on M5 Max/macOS 27 or merge without explicit acceptance of this risk. +Implementation, regression checks, dependency resolution, and the Release link +are locally verified. The hosted Xcode 26.6 rerun is pending the branch push. +Real inference acceptance remains blocked on the tested host. Do not describe +the backend as runtime-verified on M5 Max/macOS 27 or merge without explicit +acceptance of this risk. From 50427a4fe6e9413ac78a102ac1a426d4b23f2e1d Mon Sep 17 00:00:00 2001 From: idevlab Date: Sun, 30 Aug 2026 14:57:05 +0800 Subject: [PATCH 05/14] Record hosted Espresso linkage verification --- docs/sdlc/changes/2026-08-29-espresso-ane/plan.md | 2 +- .../sdlc/changes/2026-08-29-espresso-ane/verification.md | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md b/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md index 881f86b2..dd24d0b3 100644 --- a/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md @@ -19,7 +19,7 @@ - [x] Targeted `ConfigurationTests` - [x] Release-style app build - [x] Real GPT-2 `.esp` inspection and generation attempt -- [ ] GitHub Xcode 26.6 release-style app build after the dependency pin +- [x] GitHub Xcode 26.6 release-style app build after the dependency pin ## Human gates diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md b/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md index a4717c42..d12b2e0a 100644 --- a/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md @@ -10,6 +10,7 @@ | `DEVELOPER_DIR=/Applications/Xcode-beta.app/Contents/Developer bash scripts/build-app.sh --app-only --sign=-` | Pass | Pinned-fork Release app and CLI built, assembled, ad-hoc signed, and passed release artifact verification | | GitHub run `33296463655`, Xcode 26.6 Release build | Fail; diagnosed | Swift 6.2 emitted cross-module references to three internal `RealModelInferenceEngine.Compiled*` metadata symbols, then failed final arm64 linking | | Pinned Espresso commit `f3603c7` symbol probe | Pass | The three metadata symbols are emitted as external after changing only the holder types from internal to package visibility | +| GitHub run `33297701825`, Xcode 26.6 | Pass | Contract & Tests, Release-style App Build, and SDLC Gate all passed; the app build completed in 9m12s | | Independent high-risk review | Completed; findings addressed | Review found generic ANE errors, stale preload publication, public path logging, and stale loading status; fixes surface localized guidance, use a preload generation token, clear owned loading state, and keep path-bearing details private | | Espresso 0.9.0 GPT-2 generation | Fail | M5 Max/macOS 27 ANE compiler returned code 10, `verifyBundleAtPath: invalid model`, while compiling layer 0 attention | | Espresso main `eafb33d` GPT-2 generation | Fail | Latest upstream source produced the same ANE code 10 on the same host | @@ -34,7 +35,7 @@ light/dark UI pass has not yet been recorded. ## Decision Implementation, regression checks, dependency resolution, and the Release link -are locally verified. The hosted Xcode 26.6 rerun is pending the branch push. -Real inference acceptance remains blocked on the tested host. Do not describe -the backend as runtime-verified on M5 Max/macOS 27 or merge without explicit -acceptance of this risk. +are verified locally and on the hosted Xcode 26.6 runner. Real inference +acceptance remains blocked on the tested host. Do not describe the backend as +runtime-verified on M5 Max/macOS 27 or merge without explicit acceptance of +this risk. From 68eaa6b3287d2bc4989465ba81595a03adc2ea7b Mon Sep 17 00:00:00 2001 From: idevlab Date: Sun, 30 Aug 2026 16:01:50 +0800 Subject: [PATCH 06/14] fix: fall back to MLX when Espresso fails --- Sources/App/VoicePipeline+EditCommands.swift | 17 +-- Sources/App/VoicePipeline+Models.swift | 17 ++- Sources/App/VoicePipeline+Processing.swift | 3 +- Sources/App/VoicePipeline+Replacement.swift | 3 +- .../InputSessionCoordinator+Output.swift | 7 ++ Sources/LLM/EspressoLLMEngine.swift | 23 +++- .../Processing/TextProcessor+Generation.swift | 68 ++++++++++-- Sources/Processing/TextProcessor.swift | 29 ++++- .../Resources/en.lproj/Localizable.strings | 4 +- .../zh-Hans.lproj/Localizable.strings | 4 +- Sources/UI/OverlayPanelContent.swift | 17 ++- Tests/OpenTypeTests/ConfigurationTests.swift | 12 +++ Tests/OpenTypeTests/OverlayLayoutTests.swift | 12 +++ .../TextProcessorFallbackTests.swift | 100 ++++++++++++++++++ .../changes/2026-08-29-espresso-ane/intent.md | 16 ++- .../changes/2026-08-29-espresso-ane/plan.md | 6 ++ .../changes/2026-08-29-espresso-ane/spec.md | 27 +++-- .../2026-08-29-espresso-ane/state.json | 6 +- .../2026-08-29-espresso-ane/verification.md | 43 +++++--- 19 files changed, 360 insertions(+), 54 deletions(-) diff --git a/Sources/App/VoicePipeline+EditCommands.swift b/Sources/App/VoicePipeline+EditCommands.swift index 6babcba1..3203a39e 100644 --- a/Sources/App/VoicePipeline+EditCommands.swift +++ b/Sources/App/VoicePipeline+EditCommands.swift @@ -24,7 +24,6 @@ extension VoicePipeline { settings: settings, targetApp: targetApp ) - return true case .replaceSelection(let replacementRaw): await replaceSelectedText( raw: raw, @@ -32,7 +31,6 @@ extension VoicePipeline { settings: settings, targetApp: targetApp ) - return true case .rewriteLast(let intent): await rewriteLastInsertion( raw: raw, @@ -40,7 +38,6 @@ extension VoicePipeline { settings: settings, targetApp: targetApp ) - return true case .rewriteSelection(let intent): await rewriteSelectedText( raw: raw, @@ -48,14 +45,22 @@ extension VoicePipeline { settings: settings, targetApp: targetApp ) - return true case .deleteSelection: await deleteSelectedText(targetApp: targetApp) - return true case .undoLastInsertion: await undoLastInsertion(targetApp: targetApp) - return true } + + if let fallbackMessage = await applyEspressoFallbackIfNeeded(settings: settings) { + if case .error = appState.phase { + Log.info("[VoicePipeline] preserving edit-command error after Espresso fallback") + } else { + appState.statusMessage = fallbackMessage + showOverlay() + hideOverlayAfterDelay() + } + } + return true } private func replaceLastInsertion( diff --git a/Sources/App/VoicePipeline+Models.swift b/Sources/App/VoicePipeline+Models.swift index 14a62ddd..0fbe2443 100644 --- a/Sources/App/VoicePipeline+Models.swift +++ b/Sources/App/VoicePipeline+Models.swift @@ -93,11 +93,18 @@ extension VoicePipeline { appState.llmModelReady = warmup.loaded && ready if appState.llmModelReady { + if warmup.fallbackMessage != nil, + !settings.useRemoteLLM, + settings.localLLMBackend == .espresso { + settings.localLLMBackend = .mlx + } if backend == .mlx { catalog.updateLLMStatus(model, status: .ready) } Log.info("[VoicePipeline] LLM model loaded into memory, ready for instant inference") - appState.statusMessage = L("status.ready") + appState.statusMessage = showFailureInStatus + ? (warmup.fallbackMessage ?? L("status.ready")) + : L("status.ready") } else { if backend == .mlx { catalog.updateLLMStatus(model, status: .error(L("pipeline.model_load_failed"))) @@ -125,6 +132,14 @@ extension VoicePipeline { } } + func applyEspressoFallbackIfNeeded(settings: AppSettings) async -> String? { + guard let message = await textProcessor.consumeEspressoFallbackMessage() else { return nil } + if !settings.useRemoteLLM, settings.localLLMBackend == .espresso { + settings.localLLMBackend = .mlx + } + return message + } + func ensureEngineLoaded(requestPermission: Bool = true) async { switch appState.settings.speechEngine { case .whisper: diff --git a/Sources/App/VoicePipeline+Processing.swift b/Sources/App/VoicePipeline+Processing.swift index b9e3489f..c71605ce 100644 --- a/Sources/App/VoicePipeline+Processing.swift +++ b/Sources/App/VoicePipeline+Processing.swift @@ -251,6 +251,7 @@ extension VoicePipeline { targetApp: NSRunningApplication? ) async { let finalText = output.text + let fallbackMessage = await applyEspressoFallbackIfNeeded(settings: settings) guard !finalText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { Log.info("[VoicePipeline] skipping empty final text") let espressoFailure = await textProcessor.consumeEspressoFailureMessage() @@ -269,7 +270,7 @@ extension VoicePipeline { Log.info("[VoicePipeline] insert stage finished in \(String(format: "%.2f", elapsed))s") appState.phase = .done - appState.statusMessage = L("status.done") + appState.statusMessage = fallbackMessage ?? L("status.done") hideOverlayAfterDelay() if case .probablyFailed(let reason) = result { diff --git a/Sources/App/VoicePipeline+Replacement.swift b/Sources/App/VoicePipeline+Replacement.swift index 418a9cf5..950b585d 100644 --- a/Sources/App/VoicePipeline+Replacement.swift +++ b/Sources/App/VoicePipeline+Replacement.swift @@ -235,6 +235,7 @@ extension VoicePipeline { allowsGuardFallback: false, dictionarySnapshot: dictionarySnapshot ) + let fallbackMessage = await applyEspressoFallbackIfNeeded(settings: appState.settings) let elapsed = CFAbsoluteTimeGetCurrent() - started appState.lastFormattingDurationSeconds = elapsed Log.info("[VoicePipeline] Smart Format completed in \(String(format: "%.2f", elapsed))s") @@ -253,7 +254,7 @@ extension VoicePipeline { replacement.formattedText = formattedText replacement.state = .ready - replacement.message = L("pipeline.formatted_ready") + replacement.message = fallbackMessage ?? L("pipeline.formatted_ready") replacement.context = inputContext appState.pendingReplacement = replacement } diff --git a/Sources/Integration/InputSessionCoordinator+Output.swift b/Sources/Integration/InputSessionCoordinator+Output.swift index a902da1b..67c0d606 100644 --- a/Sources/Integration/InputSessionCoordinator+Output.swift +++ b/Sources/Integration/InputSessionCoordinator+Output.swift @@ -63,6 +63,13 @@ extension InputSessionCoordinator { ) } + if await textProcessor.consumeEspressoFallbackMessage() != nil, + !settings.useRemoteLLM, + settings.localLLMBackend == .espresso { + settings.localLLMBackend = .mlx + Log.info("[InputSessionCoordinator] Espresso failed; selected MLX as the active backend") + } + guard !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { Log.info("[InputSessionCoordinator] refusing to complete session with empty output") throw IntegrationError.operationFailed diff --git a/Sources/LLM/EspressoLLMEngine.swift b/Sources/LLM/EspressoLLMEngine.swift index 11f63768..5c7c7239 100644 --- a/Sources/LLM/EspressoLLMEngine.swift +++ b/Sources/LLM/EspressoLLMEngine.swift @@ -17,13 +17,15 @@ actor EspressoLLMEngine { private var model: LoadedModel? private var lastFailureMessage: String? + private var lastFallbackMessage: String? func loadModel(path: String) throws { let expandedPath = NSString(string: path).expandingTildeInPath let url = URL(fileURLWithPath: expandedPath, isDirectory: true).standardizedFileURL + lastFailureMessage = nil + lastFallbackMessage = nil guard model?.path != url.path else { return } - lastFailureMessage = nil Log.info("[EspressoLLMEngine] loading bundle: \(url.lastPathComponent)") do { let bundle = try ESPRuntimeBundle.open(at: url) @@ -52,6 +54,7 @@ actor EspressoLLMEngine { ) throws -> String { guard let model else { throw EspressoLLMError.modelNotLoaded } lastFailureMessage = nil + lastFallbackMessage = nil let input = Self.formatPrompt( user: prompt, system: systemPrompt, @@ -88,6 +91,24 @@ actor EspressoLLMEngine { return lastFailureMessage } + func recordMLXFallback() { + lastFailureMessage = nil + lastFallbackMessage = L("status.espresso_fell_back_to_mlx") + Log.info("[EspressoLLMEngine] Espresso failed; using the selected MLX model") + } + + func recordMLXFallbackFailure(details: String) { + lastFallbackMessage = nil + lastFailureMessage = L("error.espresso_mlx_fallback_unavailable") + Log.sensitive("[EspressoLLMEngine] Espresso and MLX fallback failed: \(details)") + Log.error("[EspressoLLMEngine] MLX fallback unavailable") + } + + func consumeLastFallbackMessage() -> String? { + defer { lastFallbackMessage = nil } + return lastFallbackMessage + } + private func recordFailure(_ error: Error) -> EspressoLLMError { let mapped = error as? EspressoLLMError ?? .runtimeFailure Log.sensitive("[EspressoLLMEngine] ANE runtime detail: \(error.localizedDescription)") diff --git a/Sources/Processing/TextProcessor+Generation.swift b/Sources/Processing/TextProcessor+Generation.swift index 592ced43..75eb01eb 100644 --- a/Sources/Processing/TextProcessor+Generation.swift +++ b/Sources/Processing/TextProcessor+Generation.swift @@ -32,13 +32,67 @@ extension TextProcessor { temperature: temperature ) case .espresso: - try await espressoLLM.loadModel(path: options.espressoModelPath) - return try await espressoLLM.generate( - prompt: prompt, - systemPrompt: systemPrompt, - maxTokens: maxTokens, - temperature: temperature - ) + do { + let result = try await Self.runEspressoWithMLXFallback( + espresso: { + try await self.espressoLLM.loadModel(path: options.espressoModelPath) + return try await self.espressoLLM.generate( + prompt: prompt, + systemPrompt: systemPrompt, + maxTokens: maxTokens, + temperature: temperature + ) + }, + mlx: { + try await self.llm.loadModel(id: options.llmModel) + return try await self.llm.generate( + prompt: prompt, + systemPrompt: systemPrompt, + maxTokens: maxTokens, + temperature: temperature + ) + } + ) + if result.usedMLX { + await espressoLLM.recordMLXFallback() + } + return result.value + } catch let error as EspressoMLXFallbackError { + await espressoLLM.recordMLXFallbackFailure(details: error.details) + throw error + } + } + } + + static func runEspressoWithMLXFallback( + espresso: () async throws -> Value, + mlx: () async throws -> Value + ) async throws -> (value: Value, usedMLX: Bool) { + do { + return (try await espresso(), false) + } catch { + let espressoFailure = error.localizedDescription + do { + return (try await mlx(), true) + } catch { + throw EspressoMLXFallbackError( + espressoFailure: espressoFailure, + mlxFailure: error.localizedDescription + ) + } + } + } + + struct EspressoMLXFallbackError: LocalizedError { + let espressoFailure: String + let mlxFailure: String + + var errorDescription: String? { + L("error.espresso_mlx_fallback_unavailable") + } + + var details: String { + "Espresso: \(espressoFailure); MLX: \(mlxFailure)" } } diff --git a/Sources/Processing/TextProcessor.swift b/Sources/Processing/TextProcessor.swift index 83e48283..47f2fd39 100644 --- a/Sources/Processing/TextProcessor.swift +++ b/Sources/Processing/TextProcessor.swift @@ -13,7 +13,10 @@ final class TextProcessor { func isLLMReady(for backend: LocalLLMBackend) async -> Bool { switch backend { case .mlx: return await llm.isLoaded - case .espresso: return await espressoLLM.isLoaded + case .espresso: + let espressoIsLoaded = await espressoLLM.isLoaded + let mlxIsLoaded = await llm.isLoaded + return espressoIsLoaded || mlxIsLoaded } } @@ -28,21 +31,33 @@ final class TextProcessor { model: String, backend: LocalLLMBackend, espressoModelPath: String - ) async -> (loaded: Bool, errorMessage: String?) { + ) async -> (loaded: Bool, errorMessage: String?, fallbackMessage: String?) { do { switch backend { case .mlx: try await llm.loadModel(id: model) case .espresso: - try await espressoLLM.loadModel(path: espressoModelPath) + let result = try await Self.runEspressoWithMLXFallback( + espresso: { try await self.espressoLLM.loadModel(path: espressoModelPath) }, + mlx: { try await self.llm.loadModel(id: model) } + ) + if result.usedMLX { + await espressoLLM.recordMLXFallback() + let message = await espressoLLM.consumeLastFallbackMessage() + return (true, nil, message) + } } - return (true, nil) + return (true, nil, nil) + } catch let error as EspressoMLXFallbackError { + await espressoLLM.recordMLXFallbackFailure(details: error.details) + let message = await espressoLLM.consumeLastFailureMessage() + return (false, message ?? error.localizedDescription, nil) } catch { Log.error("[TextProcessor] LLM warmup failed: \(error.localizedDescription)") if backend == .espresso { _ = await espressoLLM.consumeLastFailureMessage() } - return (false, error.localizedDescription) + return (false, error.localizedDescription, nil) } } @@ -50,6 +65,10 @@ final class TextProcessor { await espressoLLM.consumeLastFailureMessage() } + func consumeEspressoFallbackMessage() async -> String? { + await espressoLLM.consumeLastFallbackMessage() + } + func basicClean( text: String, inputLanguage: InputLanguage = .auto, diff --git a/Sources/Resources/en.lproj/Localizable.strings b/Sources/Resources/en.lproj/Localizable.strings index 64f2dcbe..64905eff 100644 --- a/Sources/Resources/en.lproj/Localizable.strings +++ b/Sources/Resources/en.lproj/Localizable.strings @@ -21,6 +21,7 @@ /* ── Status ── */ "status.ready" = "Ready"; "status.done" = "Done"; +"status.espresso_fell_back_to_mlx" = "Espresso failed on this Mac. This request finished with MLX."; "status.no_speech_detected" = "No Speech Detected"; /* ── Tabs ── */ @@ -204,7 +205,7 @@ "model.espresso.description" = "Run a compatible .esp bundle with Espresso's Apple Neural Engine backend."; "model.espresso.no_bundle" = "No .esp bundle selected"; "model.espresso.choose" = "Choose .esp Bundle…"; -"model.espresso.private_api_warning" = "Espresso uses private Apple Neural Engine APIs. It may break after macOS updates and is not eligible for Mac App Store distribution."; +"model.espresso.private_api_warning" = "Espresso uses private Apple Neural Engine APIs and may break after macOS updates. If it fails, Utter switches to your installed MLX model. Espresso is not eligible for Mac App Store distribution."; "model.custom_id_placeholder" = "Custom model ID (e.g. mlx-community/…)"; "model.active" = "Active"; "model.use" = "Use"; @@ -465,6 +466,7 @@ "error.network_request_failed" = "Network request failed — please try again"; "error.operation_failed" = "Operation failed — please try again"; "error.espresso_runtime_failed" = "Espresso could not run this model on the ANE. Switch to MLX or try a supported macOS and device."; +"error.espresso_mlx_fallback_unavailable" = "Espresso failed, and the selected MLX model is unavailable. Download an MLX model in Settings → Models."; "error.volc_not_configured" = "Doubao ASR not configured — enter API credentials in Settings → Models"; "error.volc_invalid_endpoint" = "Invalid ASR endpoint URL"; "error.volc_audio_conversion" = "Audio conversion to PCM 16kHz failed"; diff --git a/Sources/Resources/zh-Hans.lproj/Localizable.strings b/Sources/Resources/zh-Hans.lproj/Localizable.strings index 0900b53a..84c44135 100644 --- a/Sources/Resources/zh-Hans.lproj/Localizable.strings +++ b/Sources/Resources/zh-Hans.lproj/Localizable.strings @@ -21,6 +21,7 @@ /* ── Status ── */ "status.ready" = "就绪"; "status.done" = "完成"; +"status.espresso_fell_back_to_mlx" = "Espresso 在这台 Mac 上失败。本次已改用 MLX 完成。"; "status.no_speech_detected" = "未检测到语音"; /* ── Tabs ── */ @@ -204,7 +205,7 @@ "model.espresso.description" = "使用 Espresso 的 Apple 神经网络引擎后端运行兼容的 .esp 模型包。"; "model.espresso.no_bundle" = "尚未选择 .esp 模型包"; "model.espresso.choose" = "选择 .esp 模型包…"; -"model.espresso.private_api_warning" = "Espresso 使用 Apple 神经网络引擎私有 API,可能在 macOS 更新后失效,也无法通过 Mac App Store 审核。"; +"model.espresso.private_api_warning" = "Espresso 使用 Apple 神经网络引擎私有 API,可能在 macOS 更新后失效。失败时,Utter 会切换到已安装的 MLX 模型。Espresso 也无法通过 Mac App Store 审核。"; "model.custom_id_placeholder" = "自定义模型 ID(如 mlx-community/…)"; "model.active" = "当前"; "model.use" = "启用"; @@ -465,6 +466,7 @@ "error.network_request_failed" = "网络请求失败,请稍后重试"; "error.operation_failed" = "操作失败,请重试"; "error.espresso_runtime_failed" = "Espresso 无法在这台设备的 ANE 上运行此模型。请切换到 MLX,或改用受支持的 macOS 与设备。"; +"error.espresso_mlx_fallback_unavailable" = "Espresso 运行失败,所选 MLX 模型也不可用。请在“设置 → 模型”中下载一个 MLX 模型。"; "error.volc_not_configured" = "豆包语音识别未配置 — 请在 设置 → 模型 中填写 API 凭据"; "error.volc_invalid_endpoint" = "语音识别接口地址无效"; "error.volc_audio_conversion" = "音频转换为 PCM 16kHz 失败"; diff --git a/Sources/UI/OverlayPanelContent.swift b/Sources/UI/OverlayPanelContent.swift index 60f8a2e5..0318a083 100644 --- a/Sources/UI/OverlayPanelContent.swift +++ b/Sources/UI/OverlayPanelContent.swift @@ -17,6 +17,8 @@ struct OverlayLayout: Equatable { @MainActor init(appState: AppState) { let hasPreview = appState.phase == .recording && !appState.rawTranscription.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + let showsEspressoFallback = appState.phase == .done + && appState.statusMessage == L("status.espresso_fell_back_to_mlx") isInteractive = appState.isRecording switch appState.phase { @@ -52,6 +54,14 @@ struct OverlayLayout: Equatable { topPadding = 8 bottomPadding = 8 stackSpacing = 6 + case .done where showsEspressoFallback: + width = 288 + height = 56 + outerCornerRadius = 18 + horizontalPadding = 12 + topPadding = 8 + bottomPadding = 8 + stackSpacing = 6 default: width = 192 height = 40 @@ -89,6 +99,11 @@ struct OverlayContentView: View { return false } + private var showsEspressoFallback: Bool { + appState.phase == .done + && appState.statusMessage == L("status.espresso_fell_back_to_mlx") + } + var body: some View { VStack(spacing: layout.stackSpacing) { if layout.isInteractive { @@ -128,7 +143,7 @@ struct OverlayContentView: View { Text(appState.statusMessage) .font(.caption.weight(.medium)) .foregroundStyle(.white.opacity(isError ? 0.94 : 0.88)) - .lineLimit(isError ? 2 : 1) + .lineLimit(isError || showsEspressoFallback ? 2 : 1) .fixedSize(horizontal: false, vertical: true) Spacer(minLength: 4) diff --git a/Tests/OpenTypeTests/ConfigurationTests.swift b/Tests/OpenTypeTests/ConfigurationTests.swift index e24736d9..e2e78e09 100644 --- a/Tests/OpenTypeTests/ConfigurationTests.swift +++ b/Tests/OpenTypeTests/ConfigurationTests.swift @@ -301,6 +301,18 @@ final class ConfigurationTests: XCTestCase { XCTAssertTrue( Loc.string("error.espresso_runtime_failed", language: .chinese).contains("MLX") ) + XCTAssertTrue( + Loc.string("status.espresso_fell_back_to_mlx", language: .english).contains("MLX") + ) + XCTAssertTrue( + Loc.string("status.espresso_fell_back_to_mlx", language: .chinese).contains("MLX") + ) + XCTAssertTrue( + Loc.string("error.espresso_mlx_fallback_unavailable", language: .english).contains("MLX") + ) + XCTAssertTrue( + Loc.string("error.espresso_mlx_fallback_unavailable", language: .chinese).contains("MLX") + ) } func testDeveloperHTTPTokenCanBeReset() { diff --git a/Tests/OpenTypeTests/OverlayLayoutTests.swift b/Tests/OpenTypeTests/OverlayLayoutTests.swift index f0f158c9..ccbf192e 100644 --- a/Tests/OpenTypeTests/OverlayLayoutTests.swift +++ b/Tests/OpenTypeTests/OverlayLayoutTests.swift @@ -48,6 +48,18 @@ final class OverlayLayoutTests: XCTestCase { } } + func testEspressoFallbackCompletionMakesRoomForTwoLineStatus() { + let appState = AppState() + appState.phase = .done + appState.statusMessage = L("status.espresso_fell_back_to_mlx") + + let layout = OverlayLayout(appState: appState) + + XCTAssertEqual(layout.width, 288) + XCTAssertEqual(layout.height, 56) + XCTAssertFalse(layout.isInteractive) + } + func testOverlayPlacementCentersAboveVisibleScreenBottom() { let visibleFrame = CGRect(x: 100, y: 80, width: 1_200, height: 760) let frame = OverlayPanelPlacement.frame( diff --git a/Tests/OpenTypeTests/TextProcessorFallbackTests.swift b/Tests/OpenTypeTests/TextProcessorFallbackTests.swift index e8673ab7..e2f3fa95 100644 --- a/Tests/OpenTypeTests/TextProcessorFallbackTests.swift +++ b/Tests/OpenTypeTests/TextProcessorFallbackTests.swift @@ -2,6 +2,18 @@ import XCTest @testable import OpenType final class TextProcessorFallbackTests: XCTestCase { + private enum StubError: LocalizedError { + case espresso + case mlx + + var errorDescription: String? { + switch self { + case .espresso: return "espresso failed" + case .mlx: return "mlx failed" + } + } + } + func testSmartFormatDoesNotUsePreparedFallbackByDefault() { XCTAssertFalse(TextProcessor.defaultAllowsPreparedFallback) } @@ -191,4 +203,92 @@ final class TextProcessorFallbackTests: XCTestCase { XCTAssertFalse(formattingPrompt.contains("A screen image is attached")) XCTAssertFalse(commandPrompt.contains("A screen image is attached")) } + + func testEspressoSuccessDoesNotRunMLXFallback() async throws { + var ranMLX = false + + let result = try await TextProcessor.runEspressoWithMLXFallback( + espresso: { "espresso output" }, + mlx: { + ranMLX = true + return "mlx output" + } + ) + + XCTAssertEqual(result.value, "espresso output") + XCTAssertFalse(result.usedMLX) + XCTAssertFalse(ranMLX) + } + + func testEspressoFailureUsesMLXFallback() async throws { + let result = try await TextProcessor.runEspressoWithMLXFallback( + espresso: { throw StubError.espresso }, + mlx: { "mlx output" } + ) + + XCTAssertEqual(result.value, "mlx output") + XCTAssertTrue(result.usedMLX) + } + + func testEspressoAndMLXFailuresPreserveBothDiagnostics() async { + do { + _ = try await TextProcessor.runEspressoWithMLXFallback( + espresso: { throw StubError.espresso }, + mlx: { throw StubError.mlx } + ) as (value: String, usedMLX: Bool) + XCTFail("Expected both local backends to fail") + } catch let error as TextProcessor.EspressoMLXFallbackError { + XCTAssertTrue(error.details.contains("espresso failed")) + XCTAssertTrue(error.details.contains("mlx failed")) + } catch { + XCTFail("Unexpected error: \(error)") + } + } + + @MainActor + func testFallbackNoticeSwitchesPersistedBackendToMLX() async { + let suiteName = "TextProcessorFallbackTests.\(UUID().uuidString)" + let defaults = UserDefaults(suiteName: suiteName)! + defer { defaults.removePersistentDomain(forName: suiteName) } + let settings = AppSettings(defaults: defaults) + settings.localLLMBackend = .espresso + let pipeline = VoicePipeline(appState: AppState()) + await pipeline.textProcessor.espressoLLM.recordMLXFallback() + + let message = await pipeline.applyEspressoFallbackIfNeeded(settings: settings) + + XCTAssertEqual(settings.localLLMBackend, .mlx) + XCTAssertEqual(defaults.string(forKey: "localLLMBackend"), LocalLLMBackend.mlx.rawValue) + XCTAssertEqual(message, L("status.espresso_fell_back_to_mlx")) + } + + func testRealEspressoFailureFallsBackToInstalledMLX() async throws { + let environment = ProcessInfo.processInfo.environment + guard environment["OPENTYPE_ESPRESSO_MLX_FALLBACK_INTEGRATION"] == "1" else { + throw XCTSkip("Set OPENTYPE_ESPRESSO_MLX_FALLBACK_INTEGRATION=1 to run") + } + guard let bundlePath = environment["OPENTYPE_ESPRESSO_BUNDLE"], + let mlxModel = environment["OPENTYPE_MLX_MODEL"] else { + throw XCTSkip("Set OPENTYPE_ESPRESSO_BUNDLE and OPENTYPE_MLX_MODEL") + } + + var options = TextProcessingOptions(settings: AppSettings.shared, inputLanguage: .english) + options.useRemoteLLM = false + options.localLLMBackend = .espresso + options.espressoModelPath = bundlePath + options.llmModel = mlxModel + + let processor = TextProcessor() + let output = try await processor.generateText( + prompt: "Reply with exactly OK.", + systemPrompt: "Return only the requested answer.", + options: options, + maxTokens: 8, + temperature: 0 + ) + + let fallbackMessage = await processor.consumeEspressoFallbackMessage() + XCTAssertFalse(output.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + XCTAssertNotNil(fallbackMessage) + } } diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/intent.md b/docs/sdlc/changes/2026-08-29-espresso-ane/intent.md index c4f1dd95..29f2e44e 100644 --- a/docs/sdlc/changes/2026-08-29-espresso-ane/intent.md +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/intent.md @@ -8,7 +8,10 @@ Apple Neural Engine runtime for local post-processing. ## Outcome The model settings let users choose Espresso, select an `.esp` model bundle, -and route local warmup and text generation through Espresso's ANE runtime. +and route local warmup and text generation through Espresso's ANE runtime. If +that private runtime fails and the selected MLX model is installed, Utter +completes the request with MLX, switches the persisted backend to MLX, and +shows which backend produced the result. ## Scope @@ -29,11 +32,16 @@ Store compatibility for Espresso's private ANE API. - MLX and Espresso are selectable local LLM backends and the choice persists. - A valid `.esp` directory can be selected while malformed bundles are rejected. - Espresso selection routes warmup and generation through the Espresso engine. +- An Espresso runtime failure falls back to an installed selected MLX model, + persists MLX as the active backend, and surfaces the fallback to the user. +- If the selected MLX model is unavailable, the Espresso failure remains + visible with guidance to install an MLX model. - Existing local MLX and remote LLM behavior remains covered by passing tests. -- Real text generation succeeds on a supported Apple Silicon/macOS combination. +- The available M5/macOS 27 host rejects Espresso's private ANE program without + preventing local formatting when a compatible MLX model is installed. ## Open questions Which macOS 27 and M5 combinations Espresso will support remains an upstream -compatibility question. Real inference on the available M5 Max/macOS 27 host is -currently blocked by the ANE compiler. +compatibility question. Direct Espresso inference on the available M5 +Max/macOS 27 host is currently blocked by the ANE compiler. diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md b/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md index dd24d0b3..63ad9121 100644 --- a/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md @@ -11,6 +11,9 @@ - [x] Merge current `origin/main` and preserve both dependency sets. - [x] Resolve independent-review findings for error visibility, stale preload state, and path privacy. - [x] Diagnose the Xcode 26.6 Release linker failure and pin the minimal Espresso metadata fix. +- [x] Fall back from failed Espresso warmup and generation to an installed MLX model. +- [x] Persist MLX after successful fallback and surface a localized completion notice. +- [x] Cover fallback ordering, success, and dual-failure behavior with focused tests. ## Verification plan @@ -20,6 +23,9 @@ - [x] Release-style app build - [x] Real GPT-2 `.esp` inspection and generation attempt - [x] GitHub Xcode 26.6 release-style app build after the dependency pin +- [x] Real M5 ANE compile matrix across deployment targets and normalization variants +- [x] Targeted Espresso fallback tests +- [x] Current full repository gates and release-style app build ## Human gates diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/spec.md b/docs/sdlc/changes/2026-08-29-espresso-ane/spec.md index b7551a28..1be27054 100644 --- a/docs/sdlc/changes/2026-08-29-espresso-ane/spec.md +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/spec.md @@ -21,6 +21,15 @@ engine, verifies that the bundle resolves to the private ANE backend, applies a Qwen chat template when appropriate, and returns generated text through the existing processing pipeline. Remote LLM behavior is unchanged. +For local Espresso warmup and generation, `TextProcessor` first attempts the +selected `.esp` bundle. If that attempt fails, it tries the already-selected +MLX model through the existing `LLMEngine`; `LLMEngine` continues to require a +complete local model and never downloads during fallback. A successful MLX +fallback records a localized notice. The main app consumes that notice, changes +the persisted backend to MLX only if Espresso is still selected, and shows the +notice in the completion state. Integration sessions also persist the backend +change and log it without changing their response schema. + Pin Espresso to the reviewed `v0.9.0` source plus a three-line Swift 6.2 compatibility patch. The patch gives three internal compiled-kernel holder types package visibility so Xcode 26.6 can resolve metadata references emitted @@ -31,13 +40,16 @@ runtime logic or public API and avoids taking unrelated upstream `main` changes. The model and prompt remain on-device. Espresso depends on private ANE APIs, so OS or hardware changes can reject generated ANE programs even when bundle -metadata is valid. Such failures propagate through the existing model-load or -generation error path; MLX remains available as the default and rollback. -Selecting a missing or malformed bundle does not replace the current setting. +metadata is valid. Such failures first use the selected, installed MLX model; +if MLX is not available or also fails, the combined failure propagates through +the existing model-load or generation error path. Selecting a missing or +malformed bundle does not replace the current setting. The UI warning explicitly states the private-API and App Store limitation. No -attempt is made to silently fall back from Espresso to MLX because that would -misrepresent the selected execution backend. +fallback is silent: successful fallback produces a localized status message +and changes the persisted backend to MLX so later requests do not repeatedly +compile a rejected private ANE program. A concurrent user change away from +Espresso is preserved. The patched dependency is pinned by commit rather than a moving branch. Rollback returns the package URL and version requirement to upstream `v0.9.0` once an @@ -46,8 +58,9 @@ experimental backend. ## Test strategy -Persistence and prompt formatting have focused unit tests. The complete Swift -suite and repository checks cover existing paths and package integration. A +Persistence, prompt formatting, and Espresso-to-MLX fallback ordering and +failure behavior have focused unit tests. The complete Swift suite and +repository checks cover existing paths and package integration. A release-style app build checks dependency and Metal/resource packaging. Real inference is exercised with a prepared GPT-2 `.esp` bundle and recorded even if the host's private ANE compiler rejects it. diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/state.json b/docs/sdlc/changes/2026-08-29-espresso-ane/state.json index a676b6c1..b0fbfa6a 100644 --- a/docs/sdlc/changes/2026-08-29-espresso-ane/state.json +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/state.json @@ -3,7 +3,7 @@ "id": "2026-08-29-espresso-ane", "title": "Add a selectable Espresso ANE inference backend", "risk": "high", - "status": "verified", + "status": "implementing", "owners": [ "repository maintainer" ], @@ -11,8 +11,10 @@ "Users can select MLX or Espresso for local LLM processing and the selection persists.", "Users can select a valid Espresso bundle and invalid bundles are rejected before loading.", "When Espresso is selected, local text generation and model warmup use its ANE runtime.", + "When Espresso fails, an installed selected MLX model completes the request, the persisted backend changes to MLX, and the user sees the fallback.", + "When both Espresso and MLX are unavailable, the user sees actionable local-model guidance.", "The existing MLX and remote LLM paths continue to pass the automated test suite.", - "A real Espresso bundle can generate text on a supported Apple Silicon and macOS combination." + "A real Espresso failure on the available M5 Max and macOS 27 host does not prevent local formatting when MLX is available." ], "governedPaths": [ "Package.swift", diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md b/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md index d12b2e0a..9cf5c3b2 100644 --- a/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md @@ -4,38 +4,49 @@ | Check | Result | Evidence | |---|---|---| -| `bash scripts/ci-basic-checks.sh` | Pass | Repository basic checks completed after merging `origin/main` | -| `DEVELOPER_DIR=/Applications/Xcode-beta.app/Contents/Developer swift test` | Pass | 567 XCTest tests passed, 8 skipped, plus 1 Swift Testing test passed after the dependency pin | -| Targeted `ConfigurationTests` | Pass | 36 tests passed, 0 failures | -| `DEVELOPER_DIR=/Applications/Xcode-beta.app/Contents/Developer bash scripts/build-app.sh --app-only --sign=-` | Pass | Pinned-fork Release app and CLI built, assembled, ad-hoc signed, and passed release artifact verification | +| `bash scripts/ci-basic-checks.sh` | Pass | Current SDLC, harness, plist, localization, resource, identifier, secret-file, and symlink checks passed | +| `DEVELOPER_DIR=/Applications/Xcode-beta.app/Contents/Developer swift test` | Pass | 573 XCTest tests passed, 9 skipped, plus 1 Swift Testing test passed after the fallback and overlay changes | +| Focused fallback and localization tests | Pass | Espresso success avoids MLX; Espresso failure uses MLX; dual failure preserves both diagnostics; successful fallback persists MLX; localized status and 288 by 56 two-line overlay layout pass | +| `DEVELOPER_DIR=/Applications/Xcode-beta.app/Contents/Developer bash scripts/build-app.sh --app-only --sign=-` | Pass | Final source without the temporary visual-QA trigger built the Release app and CLI, assembled, ad-hoc signed, and passed artifact verification | | GitHub run `33296463655`, Xcode 26.6 Release build | Fail; diagnosed | Swift 6.2 emitted cross-module references to three internal `RealModelInferenceEngine.Compiled*` metadata symbols, then failed final arm64 linking | | Pinned Espresso commit `f3603c7` symbol probe | Pass | The three metadata symbols are emitted as external after changing only the holder types from internal to package visibility | | GitHub run `33297701825`, Xcode 26.6 | Pass | Contract & Tests, Release-style App Build, and SDLC Gate all passed; the app build completed in 9m12s | | Independent high-risk review | Completed; findings addressed | Review found generic ANE errors, stale preload publication, public path logging, and stale loading status; fixes surface localized guidance, use a preload generation token, clear owned loading state, and keep path-bearing details private | | Espresso 0.9.0 GPT-2 generation | Fail | M5 Max/macOS 27 ANE compiler returned code 10, `verifyBundleAtPath: invalid model`, while compiling layer 0 attention | | Espresso main `eafb33d` GPT-2 generation | Fail | Latest upstream source produced the same ANE code 10 on the same host | +| M5 ANE compile matrix | Fail as expected | All 24 combinations failed: iOS 18, iOS 19, macOS 26, and macOS 27 MIL targets; LayerNorm and RMSNorm; spatial sizes 64, 128, and 256 | +| Real Espresso-to-MLX fallback | Pass | On the M5 Max/macOS 27 host, a real GPT-2 `.esp` bundle produced ANE code 10, then the installed `mlx-community/Qwen3.5-2B-4bit` model generated non-empty output in the same test and produced the fallback notice | +| Real-window fallback notice | Pass | The actual Release app displayed the 288 by 56 non-modal completion overlay without truncation in Chinese light and dark appearances and English dark appearance; the temporary environment-triggered QA entry was removed before the final build | ## Acceptance criteria - Backend selection and persistence — pass; focused settings test and full suite. - Bundle selection and malformed-bundle rejection — pass at metadata validation level through `ESPRuntimeBundle.open`. - Espresso warmup and generation dispatch — pass by code path and build coverage. +- Automatic MLX recovery — pass in focused control-flow tests and a real M5 + Espresso-failure-to-MLX-generation integration test. +- Persisted backend correction — pass with isolated `UserDefaults`; MLX replaces + Espresso only when Espresso remains the selected backend. +- User feedback — pass in both localizations and real Release windows without + treating the successful recovery as an error. - Existing MLX and remote behavior — pass; complete suite has no failures. -- Real generation on a supported host — blocked on the available M5 Max/macOS 27 host; both the pinned release and upstream main fail in Apple's private ANE compiler. +- Direct Espresso generation on a supported host — blocked on the available M5 + Max/macOS 27 host; both the pinned release and upstream main fail in Apple's + private ANE compiler. The new fallback prevents that failure from blocking + local formatting when the selected MLX model is installed. ## Residual risk -Espresso relies on a private ANE interface whose generated programs are rejected -on the available M5 Max/macOS 27 environment. Bundle inspection succeeds, so the -failure is only discovered during ANE kernel compilation. The repository -maintainer owns the decision to wait for upstream compatibility, constrain the -supported hardware/OS matrix, or accept the experimental backend. A real-window -light/dark UI pass has not yet been recorded. +Espresso still relies on a private ANE interface whose generated programs are +rejected on the available M5 Max/macOS 27 environment. The fallback requires an +already-installed selected MLX model and deliberately does not start a download. +This change makes Utter resilient on M5; it does not establish direct Espresso +or private-ANE compatibility on M5. A public Core ML backend remains separate +future work. ## Decision -Implementation, regression checks, dependency resolution, and the Release link -are verified locally and on the hosted Xcode 26.6 runner. Real inference -acceptance remains blocked on the tested host. Do not describe the backend as -runtime-verified on M5 Max/macOS 27 or merge without explicit acceptance of -this risk. +Automatic recovery, persistence, regression checks, real M5 fallback, localized +window behavior, dependency resolution, and the Release link are verified. +Direct Espresso inference remains blocked on the tested host. Do not describe +the private ANE backend itself as runtime-compatible with M5 Max/macOS 27. From d0e6629d95585904d951f6ad57d3eaca91357100 Mon Sep 17 00:00:00 2001 From: idevlab Date: Sun, 30 Aug 2026 17:01:19 +0800 Subject: [PATCH 07/14] fix: close local model lifecycle leaks --- Sources/App/AppDelegate+Integrations.swift | 1 + Sources/App/AppState.swift | 7 + Sources/App/OpenTypeApp.swift | 8 +- .../VoicePipeline+EditCommandHelpers.swift | 103 ++++++++ Sources/App/VoicePipeline+EditCommands.swift | 111 +-------- .../App/VoicePipeline+ModelLifecycle.swift | 48 ++++ Sources/App/VoicePipeline+Models.swift | 76 +++--- Sources/App/VoicePipeline+Processing.swift | 81 +----- Sources/App/VoicePipeline+Replacement.swift | 42 +--- .../VoicePipeline+ReplacementHelpers.swift | 37 +++ Sources/App/VoicePipeline+Status.swift | 19 ++ Sources/App/VoicePipeline.swift | 31 ++- Sources/App/VoicePipelineOutput.swift | 82 +++++++ .../InputSessionCoordinator+Output.swift | 19 +- Sources/Integration/IntegrationError.swift | 3 + .../IntegrationHTTPDispatcher.swift | 2 +- Sources/LLM/EspressoLLMEngine.swift | 21 -- .../EspressoGenerationOutcome.swift | 48 ++++ .../Processing/TextProcessor+Generation.swift | 105 ++++---- Sources/Processing/TextProcessor+Models.swift | 178 ++++++++++++++ Sources/Processing/TextProcessor.swift | 174 +++++-------- Sources/UI/ModelManagementRows.swift | 31 +-- Sources/UI/ModelManagementStatus.swift | 30 +++ Sources/UI/ModelManagementView.swift | 2 +- Sources/UI/OverlayPanelContent.swift | 4 +- Sources/UI/SettingsView.swift | 2 + Tests/OpenTypeTests/ConfigurationTests.swift | 12 - .../OpenTypeTests/EspressoFallbackTests.swift | 232 ++++++++++++++++++ .../OpenTypeTests/IntegrationHTTPTests.swift | 4 + .../IntegrationOutputTests.swift | 38 +++ .../OpenTypeTests/LocalModelAccessTests.swift | 110 +++++++++ Tests/OpenTypeTests/OverlayLayoutTests.swift | 10 + .../TextProcessorFallbackTests.swift | 99 -------- .../changes/2026-08-29-espresso-ane/intent.md | 3 + .../changes/2026-08-29-espresso-ane/plan.md | 6 + .../changes/2026-08-29-espresso-ane/spec.md | 30 ++- .../2026-08-29-espresso-ane/state.json | 3 +- .../2026-08-29-espresso-ane/verification.md | 12 +- 38 files changed, 1216 insertions(+), 608 deletions(-) create mode 100644 Sources/App/VoicePipeline+EditCommandHelpers.swift create mode 100644 Sources/App/VoicePipeline+ModelLifecycle.swift create mode 100644 Sources/App/VoicePipeline+ReplacementHelpers.swift create mode 100644 Sources/App/VoicePipelineOutput.swift create mode 100644 Sources/Processing/EspressoGenerationOutcome.swift create mode 100644 Sources/Processing/TextProcessor+Models.swift create mode 100644 Sources/UI/ModelManagementStatus.swift create mode 100644 Tests/OpenTypeTests/EspressoFallbackTests.swift create mode 100644 Tests/OpenTypeTests/LocalModelAccessTests.swift diff --git a/Sources/App/AppDelegate+Integrations.swift b/Sources/App/AppDelegate+Integrations.swift index 0b1deaaa..102a9fc1 100644 --- a/Sources/App/AppDelegate+Integrations.swift +++ b/Sources/App/AppDelegate+Integrations.swift @@ -117,6 +117,7 @@ extension AppDelegate { func makeIntegrationSessionCoordinator(service: OpenTypeService) -> InputSessionCoordinator { InputSessionCoordinator( service: service, + textProcessor: textProcessor, isUserWorkflowBusy: { [weak self] in self?.appState.isBusy ?? false } diff --git a/Sources/App/AppState.swift b/Sources/App/AppState.swift index 5d335dca..f067ec30 100644 --- a/Sources/App/AppState.swift +++ b/Sources/App/AppState.swift @@ -13,6 +13,11 @@ enum AppPhase: Equatable { case error(String) } +enum AppCompletionKind: Equatable { + case standard + case espressoFallback +} + @MainActor final class AppState: ObservableObject { @Published var phase: AppPhase = .idle @@ -32,6 +37,7 @@ final class AppState: ObservableObject { @Published var lastFormattingDurationSeconds: Double = 0 @Published var pendingReplacement: DeferredReplacement? @Published var activeInputMode: VoiceInputMode = .dictation + @Published var completionKind: AppCompletionKind = .standard let settings = AppSettings.shared @@ -54,6 +60,7 @@ final class AppState: ObservableObject { resetDownloadProgress() pendingReplacement = nil activeInputMode = .dictation + completionKind = .standard } func clearPendingReplacement() { diff --git a/Sources/App/OpenTypeApp.swift b/Sources/App/OpenTypeApp.swift index a34f3fe3..5ea41de4 100644 --- a/Sources/App/OpenTypeApp.swift +++ b/Sources/App/OpenTypeApp.swift @@ -24,6 +24,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject { private var settingsWindowDelegate: SettingsWindowDelegate? private var onboardingWindow: NSWindow? private let popoverOutsideClickMonitor = PopoverOutsideClickMonitor() + let textProcessor: TextProcessor var cancellables = Set() var iconTimer: Timer? private var previousApp: NSRunningApplication? @@ -39,6 +40,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject { let registry = IntegrationClientRegistry() integrationClientRegistry = registry integrationService = OpenTypeService(registry: registry) + textProcessor = TextProcessor() super.init() integrationSessionCoordinator = makeIntegrationSessionCoordinator(service: integrationService) } @@ -95,7 +97,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject { } private func setupPipeline() { - pipeline = VoicePipeline(appState: appState) + pipeline = VoicePipeline(appState: appState, textProcessor: textProcessor) Task { await pipeline?.warmUp() } } @@ -202,6 +204,10 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject { onUnloadWhisper: { [weak self] in self?.pipeline?.unloadWhisper() }, onUnloadLLM: { [weak self] in self?.pipeline?.unloadLLM() }, onLoadLLM: { [weak self] in self?.pipeline?.loadLLM() }, + onBenchmarkLLM: { [weak self] modelID in + guard let self else { throw CancellationError() } + return try await self.textProcessor.benchmarkLLM(modelID: modelID) + }, onUnloadLocalASR: { [weak self] in self?.pipeline?.unloadLocalASR() } ) .environmentObject(appState) diff --git a/Sources/App/VoicePipeline+EditCommandHelpers.swift b/Sources/App/VoicePipeline+EditCommandHelpers.swift new file mode 100644 index 00000000..1ee53948 --- /dev/null +++ b/Sources/App/VoicePipeline+EditCommandHelpers.swift @@ -0,0 +1,103 @@ +import AppKit +import Foundation + +@MainActor +extension VoicePipeline { + func replacementInputContext( + settings: AppSettings, + targetApp: NSRunningApplication? + ) -> InputContext { + InputContext.capture( + targetApp: targetApp, + screenContext: "", + outputMode: .command, + inputLanguage: settings.inputLanguage, + source: .menuBar + ) + } + + func finalizedReplacementText( + _ text: String, + settings: AppSettings + ) -> String { + textProcessor.cleanCommandGeneratedOutput( + text, + inputLanguage: settings.inputLanguage + ) + } + + func rewriteSelectedText( + raw: String, + intent: SelectionRewriteIntent, + settings: AppSettings, + targetApp: NSRunningApplication? + ) async { + cancelScreenContextCapture() + + guard let selectedText = await textInserter.selectedText(targetApp: targetApp), + !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + showErrorHint(L("pipeline.no_selected_text_to_replace")) + return + } + + let context = InputContext.capture( + targetApp: targetApp, + screenContext: "", + selectedTextOverride: selectedText, + outputMode: .command, + inputLanguage: settings.inputLanguage, + source: .menuBar + ) + var options = TextProcessingOptions(settings: settings) + options.llmModel = settings.llmModel + let memoryContext = VoicePipelinePolicy.memoryContext( + for: .command, + settings: settings, + currentContext: context + ) + + appState.phase = .processing + appState.statusMessage = L("pipeline.formatting") + let rewrittenText = await textProcessor.processSelectionEdit( + selectedText: selectedText, + intent: intent, + options: options, + spokenCommand: raw, + memoryContext: memoryContext, + inputContext: context + ) + guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + showNoSpeechDetected(reason: "selection rewrite returned empty text") + return + } + + appState.processedText = rewrittenText + appState.phase = .inserting + appState.statusMessage = L("pipeline.replacing") + + let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp) + appState.phase = .done + appState.statusMessage = L("status.done") + hideOverlayAfterDelay() + + if case .probablyFailed(let reason) = result { + Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)") + TextInserter.copyToClipboard(rewrittenText) + showInsertionFailedAlert(text: rewrittenText, reason: reason) + return + } + + let recordID = InputHistory.shared.addRecord( + rawText: raw, + processedText: rewrittenText, + wasProcessed: true, + context: context + ) + appState.lastInsertedText = rewrittenText + beginCorrectionCapture( + recordID: recordID, + insertedText: rewrittenText, + context: context + ) + } +} diff --git a/Sources/App/VoicePipeline+EditCommands.swift b/Sources/App/VoicePipeline+EditCommands.swift index 3203a39e..1c0d3cc0 100644 --- a/Sources/App/VoicePipeline+EditCommands.swift +++ b/Sources/App/VoicePipeline+EditCommands.swift @@ -8,6 +8,7 @@ extension VoicePipeline { settings: AppSettings, targetApp: NSRunningApplication? ) async -> Bool { + let expectedEspressoModelPath = settings.espressoModelPath guard let command = await resolvedSpokenEditCommand( raw: raw, settings: settings, @@ -51,11 +52,18 @@ extension VoicePipeline { await undoLastInsertion(targetApp: targetApp) } - if let fallbackMessage = await applyEspressoFallbackIfNeeded(settings: settings) { - if case .error = appState.phase { + guard !Task.isCancelled else { return true } + if let espressoOutcome = await consumeEspressoOutcome( + settings: settings, + expectedEspressoModelPath: expectedEspressoModelPath + ) { + if case .error = appState.phase, espressoOutcome == .fallback { Log.info("[VoicePipeline] preserving edit-command error after Espresso fallback") + } else if espressoOutcome == .unavailable { + showErrorHint(espressoOutcome.message) } else { - appState.statusMessage = fallbackMessage + appState.completionKind = .espressoFallback + appState.statusMessage = espressoOutcome.message showOverlay() hideOverlayAfterDelay() } @@ -166,29 +174,6 @@ extension VoicePipeline { ) } - private func replacementInputContext( - settings: AppSettings, - targetApp: NSRunningApplication? - ) -> InputContext { - InputContext.capture( - targetApp: targetApp, - screenContext: "", - outputMode: .command, - inputLanguage: settings.inputLanguage, - source: .menuBar - ) - } - - private func finalizedReplacementText( - _ text: String, - settings: AppSettings - ) -> String { - textProcessor.cleanCommandGeneratedOutput( - text, - inputLanguage: settings.inputLanguage - ) - } - private func deleteSelectedText(targetApp: NSRunningApplication?) async { cancelScreenContextCapture() @@ -241,78 +226,4 @@ extension VoicePipeline { hideOverlayAfterDelay() } - private func rewriteSelectedText( - raw: String, - intent: SelectionRewriteIntent, - settings: AppSettings, - targetApp: NSRunningApplication? - ) async { - cancelScreenContextCapture() - - guard let selectedText = await textInserter.selectedText(targetApp: targetApp), - !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { - showErrorHint(L("pipeline.no_selected_text_to_replace")) - return - } - - let context = InputContext.capture( - targetApp: targetApp, - screenContext: "", - selectedTextOverride: selectedText, - outputMode: .command, - inputLanguage: settings.inputLanguage, - source: .menuBar - ) - var options = TextProcessingOptions(settings: settings) - options.llmModel = settings.llmModel - let memoryContext = VoicePipelinePolicy.memoryContext( - for: .command, - settings: settings, - currentContext: context - ) - - appState.phase = .processing - appState.statusMessage = L("pipeline.formatting") - let rewrittenText = await textProcessor.processSelectionEdit( - selectedText: selectedText, - intent: intent, - options: options, - spokenCommand: raw, - memoryContext: memoryContext, - inputContext: context - ) - guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { - showNoSpeechDetected(reason: "selection rewrite returned empty text") - return - } - - appState.processedText = rewrittenText - appState.phase = .inserting - appState.statusMessage = L("pipeline.replacing") - - let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp) - appState.phase = .done - appState.statusMessage = L("status.done") - hideOverlayAfterDelay() - - if case .probablyFailed(let reason) = result { - Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)") - TextInserter.copyToClipboard(rewrittenText) - showInsertionFailedAlert(text: rewrittenText, reason: reason) - return - } - - let recordID = InputHistory.shared.addRecord( - rawText: raw, - processedText: rewrittenText, - wasProcessed: true, - context: context - ) - appState.lastInsertedText = rewrittenText - beginCorrectionCapture( - recordID: recordID, - insertedText: rewrittenText, - context: context - ) - } } diff --git a/Sources/App/VoicePipeline+ModelLifecycle.swift b/Sources/App/VoicePipeline+ModelLifecycle.swift new file mode 100644 index 00000000..e5bed5ff --- /dev/null +++ b/Sources/App/VoicePipeline+ModelLifecycle.swift @@ -0,0 +1,48 @@ +import Foundation + +@MainActor +extension VoicePipeline { + func unloadLLM() { + formattingPreloadGeneration += 1 + processingTask?.cancel() + processingTask = nil + replacementTask?.cancel() + replacementTask = nil + appState.clearPendingReplacement() + if appState.phase == .processing { + appState.phase = .idle + appState.statusMessage = L("status.ready") + } else if appState.statusMessage == L("pipeline.loading_llm") { + appState.statusMessage = L("status.ready") + } + appState.llmModelReady = false + + let precedingTask = formattingModelLifecycleTask + formattingModelLifecycleTask = Task { @MainActor [weak self] in + _ = await precedingTask?.value + guard let self else { return nil } + await self.textProcessor.unloadLLM() + return nil + } + } + + func loadLLM() { + enqueueFormattingModelPreload(showFailureInStatus: true) + } + + @discardableResult + func enqueueFormattingModelPreload( + showFailureInStatus: Bool + ) -> Task { + let precedingTask = formattingModelLifecycleTask + let task: Task = Task { @MainActor [weak self] in + _ = await precedingTask?.value + guard let self else { return nil } + return await self.preloadFormattingModelNow( + showFailureInStatus: showFailureInStatus + ) + } + formattingModelLifecycleTask = task + return task + } +} diff --git a/Sources/App/VoicePipeline+Models.swift b/Sources/App/VoicePipeline+Models.swift index 0fbe2443..968d24b8 100644 --- a/Sources/App/VoicePipeline+Models.swift +++ b/Sources/App/VoicePipeline+Models.swift @@ -9,39 +9,17 @@ extension VoicePipeline { appState.statusMessage = L("pipeline.whisper_unloaded") } - func unloadLLM() { - formattingPreloadGeneration += 1 - processingTask?.cancel() - processingTask = nil - replacementTask?.cancel() - replacementTask = nil - appState.clearPendingReplacement() - if appState.phase == .processing { - appState.phase = .idle - appState.statusMessage = L("status.ready") - } else if appState.statusMessage == L("pipeline.loading_llm") { - appState.statusMessage = L("status.ready") - } - appState.llmModelReady = false - Task { await textProcessor.unloadLLM() } - } - func unloadLocalASR() { qwenSpeechEngine = nil } - func loadLLM() { - Task { - await preloadFormattingModel(showFailureInStatus: true) - } - } - - func preloadFormattingModel(showFailureInStatus: Bool) async { + @discardableResult + func preloadFormattingModelNow(showFailureInStatus: Bool) async -> EspressoGenerationOutcome? { formattingPreloadGeneration += 1 let preloadGeneration = formattingPreloadGeneration guard !appState.settings.useRemoteLLM else { appState.llmModelReady = true - return + return nil } let settings = appState.settings @@ -49,7 +27,7 @@ extension VoicePipeline { let model = settings.llmModel.trimmingCharacters(in: .whitespacesAndNewlines) let espressoPath = settings.espressoModelPath.trimmingCharacters(in: .whitespacesAndNewlines) let selectedModel = backend == .espresso ? espressoPath : model - guard !selectedModel.isEmpty else { return } + guard !selectedModel.isEmpty else { return nil } let catalog = ModelCatalog.shared let modelIsAvailable: Bool @@ -68,7 +46,7 @@ extension VoicePipeline { catalog.updateLLMStatus(model, status: .error(message)) } appState.statusMessage = showFailureInStatus ? message : L("status.ready") - return + return nil } appState.statusMessage = L("pipeline.loading_llm") @@ -83,28 +61,28 @@ extension VoicePipeline { ) guard preloadGeneration == formattingPreloadGeneration, formattingSelectionMatches(backend: backend, model: model, espressoPath: espressoPath) else { - return + return nil } let ready = warmup.loaded ? await textProcessor.isLLMReady(for: backend) : false guard preloadGeneration == formattingPreloadGeneration, formattingSelectionMatches(backend: backend, model: model, espressoPath: espressoPath) else { - return + return nil } appState.llmModelReady = warmup.loaded && ready if appState.llmModelReady { - if warmup.fallbackMessage != nil, - !settings.useRemoteLLM, - settings.localLLMBackend == .espresso { - settings.localLLMBackend = .mlx - } + EspressoFallbackPolicy.selectMLXIfNeeded( + after: warmup.espressoOutcome, + settings: settings, + expectedEspressoModelPath: espressoPath + ) if backend == .mlx { catalog.updateLLMStatus(model, status: .ready) } Log.info("[VoicePipeline] LLM model loaded into memory, ready for instant inference") - appState.statusMessage = showFailureInStatus - ? (warmup.fallbackMessage ?? L("status.ready")) - : L("status.ready") + appState.statusMessage = warmup.espressoOutcome?.message ?? L("status.ready") + presentEspressoWarmupOutcomeIfNeeded(warmup.espressoOutcome) + return warmup.espressoOutcome } else { if backend == .mlx { catalog.updateLLMStatus(model, status: .error(L("pipeline.model_load_failed"))) @@ -113,7 +91,11 @@ extension VoicePipeline { let message = backend == .espresso ? (warmup.errorMessage ?? L("error.espresso_runtime_failed")) : L("pipeline.model_load_failed") - appState.statusMessage = showFailureInStatus ? message : L("status.ready") + appState.statusMessage = warmup.espressoOutcome != nil || showFailureInStatus + ? message + : L("status.ready") + presentEspressoWarmupOutcomeIfNeeded(warmup.espressoOutcome) + return warmup.espressoOutcome } } @@ -132,12 +114,18 @@ extension VoicePipeline { } } - func applyEspressoFallbackIfNeeded(settings: AppSettings) async -> String? { - guard let message = await textProcessor.consumeEspressoFallbackMessage() else { return nil } - if !settings.useRemoteLLM, settings.localLLMBackend == .espresso { - settings.localLLMBackend = .mlx - } - return message + func consumeEspressoOutcome( + settings: AppSettings, + expectedEspressoModelPath: String + ) async -> EspressoGenerationOutcome? { + let outcome = await textProcessor.consumeEspressoOutcome() + guard !Task.isCancelled else { return nil } + EspressoFallbackPolicy.selectMLXIfNeeded( + after: outcome, + settings: settings, + expectedEspressoModelPath: expectedEspressoModelPath + ) + return outcome } func ensureEngineLoaded(requestPermission: Bool = true) async { diff --git a/Sources/App/VoicePipeline+Processing.swift b/Sources/App/VoicePipeline+Processing.swift index c71605ce..0eb68ded 100644 --- a/Sources/App/VoicePipeline+Processing.swift +++ b/Sources/App/VoicePipeline+Processing.swift @@ -49,13 +49,13 @@ extension VoicePipeline { } } + let expectedEspressoModelPath = settings.espressoModelPath let output = await outputText( for: preparedRaw, settings: settings, inputMode: inputMode, targetApp: targetApp ) - guard !Task.isCancelled else { resetToIdle() return @@ -65,6 +65,7 @@ extension VoicePipeline { output, raw: preparedRaw, settings: settings, + expectedEspressoModelPath: expectedEspressoModelPath, inputMode: inputMode, targetApp: targetApp ) @@ -237,82 +238,4 @@ extension VoicePipeline { return VoicePipelineOutput(text: text, context: inputContext) } - func recordFormattingDuration(_ started: CFAbsoluteTime, label: String) { - let elapsed = CFAbsoluteTimeGetCurrent() - started - appState.lastFormattingDurationSeconds = elapsed - Log.info("[VoicePipeline] \(label) completed in \(String(format: "%.2f", elapsed))s") - } - - private func insertFinalText( - _ output: VoicePipelineOutput, - raw: String, - settings: AppSettings, - inputMode: VoiceInputMode, - targetApp: NSRunningApplication? - ) async { - let finalText = output.text - let fallbackMessage = await applyEspressoFallbackIfNeeded(settings: settings) - guard !finalText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { - Log.info("[VoicePipeline] skipping empty final text") - let espressoFailure = await textProcessor.consumeEspressoFailureMessage() - showErrorHint(espressoFailure ?? L("error.operation_failed")) - return - } - - appState.processedText = finalText - appState.phase = .inserting - appState.statusMessage = L("pipeline.inserting") - - Log.sensitive("[VoicePipeline] inserting \(finalText.count) chars") - let started = CFAbsoluteTimeGetCurrent() - let result = await textInserter.insert(text: finalText, targetApp: targetApp) - let elapsed = CFAbsoluteTimeGetCurrent() - started - Log.info("[VoicePipeline] insert stage finished in \(String(format: "%.2f", elapsed))s") - - appState.phase = .done - appState.statusMessage = fallbackMessage ?? L("status.done") - hideOverlayAfterDelay() - - if case .probablyFailed(let reason) = result { - Log.info("[VoicePipeline] insertion probably failed: \(reason)") - TextInserter.copyToClipboard(finalText) - showInsertionFailedAlert(text: finalText, reason: reason) - return - } - - let wasProcessed = inputMode.isTranslation - || settings.outputMode == .processed - || settings.outputMode == .command - let recordID = InputHistory.shared.addRecord( - rawText: raw, - processedText: finalText, - wasProcessed: wasProcessed, - context: output.context, - formatKind: output.formatKind - ) - appState.lastInsertedText = finalText - if !inputMode.isTranslation { - beginCorrectionCapture( - recordID: recordID, - insertedText: finalText, - context: output.context - ) - } - } -} - -struct VoicePipelineOutput { - let text: String - let context: InputContext - let formatKind: TextFormatKind? - - init(text: String, context: InputContext, formatKind: TextFormatKind? = nil) { - self.text = text - self.context = context - self.formatKind = formatKind - } -} - -private enum VoicePipelineStop: Error { - case noSpeech } diff --git a/Sources/App/VoicePipeline+Replacement.swift b/Sources/App/VoicePipeline+Replacement.swift index 950b585d..30790894 100644 --- a/Sources/App/VoicePipeline+Replacement.swift +++ b/Sources/App/VoicePipeline+Replacement.swift @@ -235,18 +235,22 @@ extension VoicePipeline { allowsGuardFallback: false, dictionarySnapshot: dictionarySnapshot ) - let fallbackMessage = await applyEspressoFallbackIfNeeded(settings: appState.settings) let elapsed = CFAbsoluteTimeGetCurrent() - started appState.lastFormattingDurationSeconds = elapsed Log.info("[VoicePipeline] Smart Format completed in \(String(format: "%.2f", elapsed))s") guard !Task.isCancelled else { return } guard var replacement = appState.pendingReplacement, replacement.id == replacementID else { return } + let espressoOutcome = await consumeEspressoOutcome( + settings: appState.settings, + expectedEspressoModelPath: processingOptions.espressoModelPath + ) + guard !Task.isCancelled else { return } guard !formattedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { Log.info("[VoicePipeline] deferred Smart Format produced no LLM output") replacement.state = .failed - replacement.message = L("pipeline.formatting_failed") + replacement.message = espressoOutcome?.message ?? L("pipeline.formatting_failed") replacement.context = inputContext appState.pendingReplacement = replacement return @@ -254,7 +258,7 @@ extension VoicePipeline { replacement.formattedText = formattedText replacement.state = .ready - replacement.message = fallbackMessage ?? L("pipeline.formatted_ready") + replacement.message = espressoOutcome?.message ?? L("pipeline.formatted_ready") replacement.context = inputContext appState.pendingReplacement = replacement } @@ -278,36 +282,4 @@ extension VoicePipeline { ) } - private func immediateInsertText( - from raw: String, - inputLanguage: InputLanguage, - dictionarySnapshot: PersonalDictionarySnapshot - ) -> String { - let cleaned = textProcessor.prepareForFormatting( - text: raw, - inputLanguage: inputLanguage, - dictionarySnapshot: dictionarySnapshot - ) - let fallback = textProcessor.basicClean( - text: raw, - inputLanguage: inputLanguage, - dictionarySnapshot: dictionarySnapshot - ) - if !cleaned.isEmpty { return cleaned } - if !fallback.isEmpty { return fallback } - return "" - } - - private func replacementCopyMessage(for reason: DeferredReplacementCopyReason) -> String { - switch reason { - case .expired: - return L("pipeline.replacement_copied_expired") - case .missingTarget: - return L("pipeline.replacement_copied_missing_target") - case .appChanged: - return L("pipeline.replacement_copied_app_changed") - case .notReady: - return L("pipeline.replacement_not_ready") - } - } } diff --git a/Sources/App/VoicePipeline+ReplacementHelpers.swift b/Sources/App/VoicePipeline+ReplacementHelpers.swift new file mode 100644 index 00000000..dccac86e --- /dev/null +++ b/Sources/App/VoicePipeline+ReplacementHelpers.swift @@ -0,0 +1,37 @@ +import Foundation + +@MainActor +extension VoicePipeline { + func immediateInsertText( + from raw: String, + inputLanguage: InputLanguage, + dictionarySnapshot: PersonalDictionarySnapshot + ) -> String { + let cleaned = textProcessor.prepareForFormatting( + text: raw, + inputLanguage: inputLanguage, + dictionarySnapshot: dictionarySnapshot + ) + let fallback = textProcessor.basicClean( + text: raw, + inputLanguage: inputLanguage, + dictionarySnapshot: dictionarySnapshot + ) + if !cleaned.isEmpty { return cleaned } + if !fallback.isEmpty { return fallback } + return "" + } + + func replacementCopyMessage(for reason: DeferredReplacementCopyReason) -> String { + switch reason { + case .expired: + return L("pipeline.replacement_copied_expired") + case .missingTarget: + return L("pipeline.replacement_copied_missing_target") + case .appChanged: + return L("pipeline.replacement_copied_app_changed") + case .notReady: + return L("pipeline.replacement_not_ready") + } + } +} diff --git a/Sources/App/VoicePipeline+Status.swift b/Sources/App/VoicePipeline+Status.swift index ad23b7c7..aa3163e0 100644 --- a/Sources/App/VoicePipeline+Status.swift +++ b/Sources/App/VoicePipeline+Status.swift @@ -38,6 +38,12 @@ extension VoicePipeline { appState.statusMessage = L("status.ready") } + func recordFormattingDuration(_ started: CFAbsoluteTime, label: String) { + let elapsed = CFAbsoluteTimeGetCurrent() - started + appState.lastFormattingDurationSeconds = elapsed + Log.info("[VoicePipeline] \(label) completed in \(String(format: "%.2f", elapsed))s") + } + func hideOverlayAfterDelay() { hideOverlayTask?.cancel() hideOverlayTask = Task { @MainActor in @@ -69,6 +75,19 @@ extension VoicePipeline { hideOverlayAfterDelay() } + func presentEspressoWarmupOutcomeIfNeeded(_ outcome: EspressoGenerationOutcome?) { + guard let outcome else { return } + if outcome == .unavailable { + showErrorHint(outcome.message) + return + } + appState.phase = .done + appState.completionKind = .espressoFallback + appState.statusMessage = outcome.message + showOverlay() + hideOverlayAfterDelay() + } + func showInsertionFailedAlert(text: String, reason: String) { let alert = NSAlert() alert.messageText = L("pipeline.insert_failed_title") diff --git a/Sources/App/VoicePipeline.swift b/Sources/App/VoicePipeline.swift index 13ed914b..1d0f811c 100644 --- a/Sources/App/VoicePipeline.swift +++ b/Sources/App/VoicePipeline.swift @@ -8,7 +8,7 @@ final class VoicePipeline { let audioCapture = AudioCaptureManager() let textInserter = TextInserter() let correctionCapture = CorrectionCaptureService() - let textProcessor = TextProcessor() + let textProcessor: TextProcessor let overlay = OverlayPanel() var whisperEngine: WhisperEngine? var appleSpeechEngine: AppleSpeechEngine? @@ -19,6 +19,7 @@ final class VoicePipeline { var processingTask: Task? var replacementTask: Task? var hideOverlayTask: Task? + var formattingModelLifecycleTask: Task? var recordingTargetApp: NSRunningApplication? var formattingPreloadGeneration = 0 @@ -32,8 +33,9 @@ final class VoicePipeline { } } - init(appState: AppState) { + init(appState: AppState, textProcessor: TextProcessor = TextProcessor()) { self.appState = appState + self.textProcessor = textProcessor } func warmUp() async { @@ -64,7 +66,12 @@ final class VoicePipeline { } if shouldLoadFormatting { - await preloadFormattingModel(showFailureInStatus: false) + let espressoOutcome = await enqueueFormattingModelPreload( + showFailureInStatus: false + ).value + if espressoOutcome != nil { + return + } } markReadyIfPossible() @@ -188,14 +195,16 @@ final class VoicePipeline { processingTask = Task { @MainActor [weak self] in guard let self else { return } - await self.processRecording( - audioURL: audioURL, - audioActivity: audioActivity, - language: language, - settings: settings, - inputMode: inputMode, - targetApp: resolvedTargetApp - ) + await TextProcessor.withEspressoOutcomeTracking { + await self.processRecording( + audioURL: audioURL, + audioActivity: audioActivity, + language: language, + settings: settings, + inputMode: inputMode, + targetApp: resolvedTargetApp + ) + } } } diff --git a/Sources/App/VoicePipelineOutput.swift b/Sources/App/VoicePipelineOutput.swift new file mode 100644 index 00000000..3730a9b1 --- /dev/null +++ b/Sources/App/VoicePipelineOutput.swift @@ -0,0 +1,82 @@ +import AppKit +import Foundation + +struct VoicePipelineOutput { + let text: String + let context: InputContext + let formatKind: TextFormatKind? + + init(text: String, context: InputContext, formatKind: TextFormatKind? = nil) { + self.text = text + self.context = context + self.formatKind = formatKind + } +} + +enum VoicePipelineStop: Error { + case noSpeech +} + +@MainActor +extension VoicePipeline { + func insertFinalText( + _ output: VoicePipelineOutput, + raw: String, + settings: AppSettings, + expectedEspressoModelPath: String, + inputMode: VoiceInputMode, + targetApp: NSRunningApplication? + ) async { + let finalText = output.text + let espressoOutcome = await consumeEspressoOutcome( + settings: settings, + expectedEspressoModelPath: expectedEspressoModelPath + ) + guard !finalText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + Log.info("[VoicePipeline] skipping empty final text") + showErrorHint(espressoOutcome?.message ?? L("error.operation_failed")) + return + } + + appState.processedText = finalText + appState.phase = .inserting + appState.statusMessage = L("pipeline.inserting") + + Log.sensitive("[VoicePipeline] inserting \(finalText.count) chars") + let started = CFAbsoluteTimeGetCurrent() + let result = await textInserter.insert(text: finalText, targetApp: targetApp) + let elapsed = CFAbsoluteTimeGetCurrent() - started + Log.info("[VoicePipeline] insert stage finished in \(String(format: "%.2f", elapsed))s") + + appState.phase = .done + appState.completionKind = espressoOutcome == .fallback ? .espressoFallback : .standard + appState.statusMessage = espressoOutcome?.message ?? L("status.done") + hideOverlayAfterDelay() + + if case .probablyFailed(let reason) = result { + Log.info("[VoicePipeline] insertion probably failed: \(reason)") + TextInserter.copyToClipboard(finalText) + showInsertionFailedAlert(text: finalText, reason: reason) + return + } + + let wasProcessed = inputMode.isTranslation + || settings.outputMode == .processed + || settings.outputMode == .command + let recordID = InputHistory.shared.addRecord( + rawText: raw, + processedText: finalText, + wasProcessed: wasProcessed, + context: output.context, + formatKind: output.formatKind + ) + appState.lastInsertedText = finalText + if !inputMode.isTranslation { + beginCorrectionCapture( + recordID: recordID, + insertedText: finalText, + context: output.context + ) + } + } +} diff --git a/Sources/Integration/InputSessionCoordinator+Output.swift b/Sources/Integration/InputSessionCoordinator+Output.swift index 67c0d606..24a427aa 100644 --- a/Sources/Integration/InputSessionCoordinator+Output.swift +++ b/Sources/Integration/InputSessionCoordinator+Output.swift @@ -3,6 +3,12 @@ import Foundation @MainActor extension InputSessionCoordinator { func outputText(for raw: String, active: ActiveSession) async throws -> String { + try await TextProcessor.withEspressoOutcomeTracking { + try await trackedOutputText(for: raw, active: active) + } + } + + private func trackedOutputText(for raw: String, active: ActiveSession) async throws -> String { let options = TextProcessingOptions(settings: settings, inputLanguage: active.inputLanguage) let dictionarySnapshot = PersonalDictionary.shared.snapshot(settings: settings) let enableMemory = settings.enableMemory @@ -63,15 +69,20 @@ extension InputSessionCoordinator { ) } - if await textProcessor.consumeEspressoFallbackMessage() != nil, - !settings.useRemoteLLM, - settings.localLLMBackend == .espresso { - settings.localLLMBackend = .mlx + let espressoOutcome = await textProcessor.consumeEspressoOutcome() + if !Task.isCancelled, EspressoFallbackPolicy.selectMLXIfNeeded( + after: espressoOutcome, + settings: settings, + expectedEspressoModelPath: options.espressoModelPath + ) { Log.info("[InputSessionCoordinator] Espresso failed; selected MLX as the active backend") } guard !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { Log.info("[InputSessionCoordinator] refusing to complete session with empty output") + if let espressoOutcome, espressoOutcome == .unavailable { + throw IntegrationError.operationFailedWithMessage(espressoOutcome.message) + } throw IntegrationError.operationFailed } diff --git a/Sources/Integration/IntegrationError.swift b/Sources/Integration/IntegrationError.swift index 3045463a..33d4d2ae 100644 --- a/Sources/Integration/IntegrationError.swift +++ b/Sources/Integration/IntegrationError.swift @@ -16,6 +16,7 @@ enum IntegrationError: Error, Equatable { case invalidSessionState case noSpeechDetected case operationFailed + case operationFailedWithMessage(String) var payload: Payload { switch self { @@ -39,6 +40,8 @@ enum IntegrationError: Error, Equatable { return Payload(error: "no_speech_detected", message: "No speech was detected in the recording.") case .operationFailed: return Payload(error: "operation_failed", message: "Utter could not complete the input session.") + case .operationFailedWithMessage(let message): + return Payload(error: "operation_failed", message: message) } } } diff --git a/Sources/Integration/IntegrationHTTPDispatcher.swift b/Sources/Integration/IntegrationHTTPDispatcher.swift index cd73da3f..04ca529e 100644 --- a/Sources/Integration/IntegrationHTTPDispatcher.swift +++ b/Sources/Integration/IntegrationHTTPDispatcher.swift @@ -114,7 +114,7 @@ struct IntegrationHTTPDispatcher { return 404 case .permissionDenied, .modelNotReady, .noSpeechDetected: return 400 - case .operationFailed: + case .operationFailed, .operationFailedWithMessage: return 500 } } diff --git a/Sources/LLM/EspressoLLMEngine.swift b/Sources/LLM/EspressoLLMEngine.swift index 5c7c7239..155a91e9 100644 --- a/Sources/LLM/EspressoLLMEngine.swift +++ b/Sources/LLM/EspressoLLMEngine.swift @@ -17,13 +17,11 @@ actor EspressoLLMEngine { private var model: LoadedModel? private var lastFailureMessage: String? - private var lastFallbackMessage: String? func loadModel(path: String) throws { let expandedPath = NSString(string: path).expandingTildeInPath let url = URL(fileURLWithPath: expandedPath, isDirectory: true).standardizedFileURL lastFailureMessage = nil - lastFallbackMessage = nil guard model?.path != url.path else { return } Log.info("[EspressoLLMEngine] loading bundle: \(url.lastPathComponent)") @@ -54,7 +52,6 @@ actor EspressoLLMEngine { ) throws -> String { guard let model else { throw EspressoLLMError.modelNotLoaded } lastFailureMessage = nil - lastFallbackMessage = nil let input = Self.formatPrompt( user: prompt, system: systemPrompt, @@ -91,24 +88,6 @@ actor EspressoLLMEngine { return lastFailureMessage } - func recordMLXFallback() { - lastFailureMessage = nil - lastFallbackMessage = L("status.espresso_fell_back_to_mlx") - Log.info("[EspressoLLMEngine] Espresso failed; using the selected MLX model") - } - - func recordMLXFallbackFailure(details: String) { - lastFallbackMessage = nil - lastFailureMessage = L("error.espresso_mlx_fallback_unavailable") - Log.sensitive("[EspressoLLMEngine] Espresso and MLX fallback failed: \(details)") - Log.error("[EspressoLLMEngine] MLX fallback unavailable") - } - - func consumeLastFallbackMessage() -> String? { - defer { lastFallbackMessage = nil } - return lastFallbackMessage - } - private func recordFailure(_ error: Error) -> EspressoLLMError { let mapped = error as? EspressoLLMError ?? .runtimeFailure Log.sensitive("[EspressoLLMEngine] ANE runtime detail: \(error.localizedDescription)") diff --git a/Sources/Processing/EspressoGenerationOutcome.swift b/Sources/Processing/EspressoGenerationOutcome.swift new file mode 100644 index 00000000..057a6a2a --- /dev/null +++ b/Sources/Processing/EspressoGenerationOutcome.swift @@ -0,0 +1,48 @@ +import Foundation + +enum EspressoGenerationOutcome: Equatable, Sendable { + case fallback + case unavailable + + var message: String { + switch self { + case .fallback: + return L("status.espresso_fell_back_to_mlx") + case .unavailable: + return L("error.espresso_mlx_fallback_unavailable") + } + } +} + +actor EspressoGenerationTracker { + private var outcome: EspressoGenerationOutcome? + + func record(_ newOutcome: EspressoGenerationOutcome) { + if outcome != .unavailable { + outcome = newOutcome + } + } + + func consume() -> EspressoGenerationOutcome? { + defer { outcome = nil } + return outcome + } +} + +enum EspressoFallbackPolicy { + @discardableResult + static func selectMLXIfNeeded( + after outcome: EspressoGenerationOutcome?, + settings: AppSettings, + expectedEspressoModelPath: String + ) -> Bool { + guard outcome == .fallback, + !settings.useRemoteLLM, + settings.localLLMBackend == .espresso, + settings.espressoModelPath == expectedEspressoModelPath else { + return false + } + settings.localLLMBackend = .mlx + return true + } +} diff --git a/Sources/Processing/TextProcessor+Generation.swift b/Sources/Processing/TextProcessor+Generation.swift index 75eb01eb..e85ee889 100644 --- a/Sources/Processing/TextProcessor+Generation.swift +++ b/Sources/Processing/TextProcessor+Generation.swift @@ -22,44 +22,52 @@ extension TextProcessor { ) } - switch options.localLLMBackend { - case .mlx: - await ensureModelLoaded(options.llmModel) - return try await llm.generate( - prompt: prompt, - systemPrompt: systemPrompt, - maxTokens: maxTokens, - temperature: temperature - ) - case .espresso: - do { - let result = try await Self.runEspressoWithMLXFallback( - espresso: { - try await self.espressoLLM.loadModel(path: options.espressoModelPath) - return try await self.espressoLLM.generate( - prompt: prompt, - systemPrompt: systemPrompt, - maxTokens: maxTokens, - temperature: temperature - ) - }, - mlx: { - try await self.llm.loadModel(id: options.llmModel) - return try await self.llm.generate( - prompt: prompt, - systemPrompt: systemPrompt, - maxTokens: maxTokens, - temperature: temperature - ) - } + return try await withLocalModelAccess { + try Task.checkCancellation() + switch options.localLLMBackend { + case .mlx: + await ensureModelLoaded(options.llmModel) + return try await llm.generate( + prompt: prompt, + systemPrompt: systemPrompt, + maxTokens: maxTokens, + temperature: temperature ) - if result.usedMLX { - await espressoLLM.recordMLXFallback() + case .espresso: + do { + let result = try await Self.runEspressoWithMLXFallback( + espresso: { + try await self.espressoLLM.loadModel(path: options.espressoModelPath) + return try await self.espressoLLM.generate( + prompt: prompt, + systemPrompt: systemPrompt, + maxTokens: maxTokens, + temperature: temperature + ) + }, + mlx: { + try await self.llm.loadModel(id: options.llmModel) + return try await self.llm.generate( + prompt: prompt, + systemPrompt: systemPrompt, + maxTokens: maxTokens, + temperature: temperature + ) + } + ) + if result.usedMLX { + _ = await espressoLLM.consumeLastFailureMessage() + await Self.recordEspressoOutcome(.fallback) + Log.info("[TextProcessor] Espresso failed; used the selected MLX model") + } + return result.value + } catch let error as EspressoMLXFallbackError { + _ = await espressoLLM.consumeLastFailureMessage() + await Self.recordEspressoOutcome(.unavailable) + Log.sensitive("[TextProcessor] Espresso and MLX fallback failed: \(error.details)") + Log.error("[TextProcessor] MLX fallback unavailable") + throw error } - return result.value - } catch let error as EspressoMLXFallbackError { - await espressoLLM.recordMLXFallbackFailure(details: error.details) - throw error } } } @@ -71,10 +79,14 @@ extension TextProcessor { do { return (try await espresso(), false) } catch { + try Task.checkCancellation() let espressoFailure = error.localizedDescription do { - return (try await mlx(), true) + let value = try await mlx() + try Task.checkCancellation() + return (value, true) } catch { + try Task.checkCancellation() throw EspressoMLXFallbackError( espressoFailure: espressoFailure, mlxFailure: error.localizedDescription @@ -104,14 +116,17 @@ extension TextProcessor { maxTokens: Int, temperature: Double ) async throws -> String { - try await vlm.loadModel(id: model) - return try await vlm.generate( - prompt: prompt, - systemPrompt: systemPrompt, - image: image, - maxTokens: maxTokens, - temperature: temperature - ) + try await withLocalModelAccess { + try Task.checkCancellation() + try await vlm.loadModel(id: model) + return try await vlm.generate( + prompt: prompt, + systemPrompt: systemPrompt, + image: image, + maxTokens: maxTokens, + temperature: temperature + ) + } } func shouldUseScreenImage(options: TextProcessingOptions, image: CGImage?) -> Bool { diff --git a/Sources/Processing/TextProcessor+Models.swift b/Sources/Processing/TextProcessor+Models.swift new file mode 100644 index 00000000..ffbca11b --- /dev/null +++ b/Sources/Processing/TextProcessor+Models.swift @@ -0,0 +1,178 @@ +import Foundation +import MLX + +actor LocalModelAccessGate { + private struct Waiter { + let id: UUID + let continuation: CheckedContinuation + } + + private var isOccupied = false + private var waiters: [Waiter] = [] + + var waitingTaskCount: Int { waiters.count } + + func acquire() async throws { + try Task.checkCancellation() + guard isOccupied else { + isOccupied = true + return + } + + let id = UUID() + try await withTaskCancellationHandler(operation: { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + guard !Task.isCancelled else { + continuation.resume(throwing: CancellationError()) + return + } + waiters.append(Waiter(id: id, continuation: continuation)) + } + }, onCancel: { + Task { await self.cancelWaiter(id: id) } + }) + } + + func release() { + guard !waiters.isEmpty else { + isOccupied = false + return + } + waiters.removeFirst().continuation.resume() + } + + private func cancelWaiter(id: UUID) { + guard let index = waiters.firstIndex(where: { $0.id == id }) else { return } + waiters.remove(at: index).continuation.resume(throwing: CancellationError()) + } +} + +extension TextProcessor { + func withLocalModelAccess( + _ operation: () async throws -> Value + ) async throws -> Value { + if Self.hasLocalModelAccess { + return try await operation() + } + + try await localModelAccessGate.acquire() + do { + let value = try await Self.$hasLocalModelAccess.withValue(true) { + try Task.checkCancellation() + return try await operation() + } + await localModelAccessGate.release() + return value + } catch { + await localModelAccessGate.release() + throw error + } + } + + static func withEspressoOutcomeTracking( + _ operation: () async throws -> Value + ) async rethrows -> Value { + if espressoGenerationTracker != nil { + return try await operation() + } + return try await $espressoGenerationTracker.withValue(EspressoGenerationTracker()) { + try await operation() + } + } + + static func recordEspressoOutcome(_ outcome: EspressoGenerationOutcome) async { + await espressoGenerationTracker?.record(outcome) + } + + func consumeEspressoOutcome() async -> EspressoGenerationOutcome? { + await Self.espressoGenerationTracker?.consume() + } + + func isLLMReady(for backend: LocalLLMBackend) async -> Bool { + do { + return try await withLocalModelAccess { + switch backend { + case .mlx: + return await llm.isLoaded + case .espresso: + let espressoIsLoaded = await espressoLLM.isLoaded + let mlxIsLoaded = await llm.isLoaded + return espressoIsLoaded || mlxIsLoaded + } + } + } catch { + return false + } + } + + func unloadLLM() async { + do { + try await withLocalModelAccess { + await llm.unload() + await benchmarkEngine.unload() + await espressoLLM.unload() + await vlm.unload() + Memory.clearCache() + } + } catch { + Log.info("[TextProcessor] local model unload cancelled") + } + } + + func benchmarkLLM(modelID: String) async throws -> LLMEngine.BenchmarkResult { + try await withLocalModelAccess { + try Task.checkCancellation() + do { + let result = try await benchmarkEngine.benchmark(modelID: modelID) + await benchmarkEngine.unload() + Memory.clearCache() + return result + } catch { + await benchmarkEngine.unload() + Memory.clearCache() + throw error + } + } + } + + @discardableResult + func warmUpLLM( + model: String, + backend: LocalLLMBackend, + espressoModelPath: String + ) async -> (loaded: Bool, errorMessage: String?, espressoOutcome: EspressoGenerationOutcome?) { + do { + return try await withLocalModelAccess { + do { + switch backend { + case .mlx: + try await llm.loadModel(id: model) + case .espresso: + let result = try await Self.runEspressoWithMLXFallback( + espresso: { try await self.espressoLLM.loadModel(path: espressoModelPath) }, + mlx: { try await self.llm.loadModel(id: model) } + ) + if result.usedMLX { + _ = await espressoLLM.consumeLastFailureMessage() + return (true, nil, .fallback) + } + } + return (true, nil, nil) + } catch let error as EspressoMLXFallbackError { + Log.sensitive("[TextProcessor] Espresso and MLX warmup failed: \(error.details)") + Log.error("[TextProcessor] MLX fallback unavailable during warmup") + _ = await espressoLLM.consumeLastFailureMessage() + return (false, EspressoGenerationOutcome.unavailable.message, .unavailable) + } catch { + Log.error("[TextProcessor] LLM warmup failed: \(error.localizedDescription)") + if backend == .espresso { + _ = await espressoLLM.consumeLastFailureMessage() + } + return (false, error.localizedDescription, nil) + } + } + } catch { + return (false, error.localizedDescription, nil) + } + } +} diff --git a/Sources/Processing/TextProcessor.swift b/Sources/Processing/TextProcessor.swift index 47f2fd39..064e2770 100644 --- a/Sources/Processing/TextProcessor.swift +++ b/Sources/Processing/TextProcessor.swift @@ -4,71 +4,17 @@ import Foundation final class TextProcessor { static let defaultAllowsPreparedFallback = false + @TaskLocal static var espressoGenerationTracker: EspressoGenerationTracker? + @TaskLocal static var hasLocalModelAccess = false + let llm = LLMEngine() + let benchmarkEngine = LLMEngine() let espressoLLM = EspressoLLMEngine() let vlm = VLMEngine() + let localModelAccessGate = LocalModelAccessGate() let remoteLLMClient = RemoteLLMClient() private let dictionary = PersonalDictionary.shared - func isLLMReady(for backend: LocalLLMBackend) async -> Bool { - switch backend { - case .mlx: return await llm.isLoaded - case .espresso: - let espressoIsLoaded = await espressoLLM.isLoaded - let mlxIsLoaded = await llm.isLoaded - return espressoIsLoaded || mlxIsLoaded - } - } - - func unloadLLM() async { - await llm.unload() - await espressoLLM.unload() - await vlm.unload() - } - - @discardableResult - func warmUpLLM( - model: String, - backend: LocalLLMBackend, - espressoModelPath: String - ) async -> (loaded: Bool, errorMessage: String?, fallbackMessage: String?) { - do { - switch backend { - case .mlx: - try await llm.loadModel(id: model) - case .espresso: - let result = try await Self.runEspressoWithMLXFallback( - espresso: { try await self.espressoLLM.loadModel(path: espressoModelPath) }, - mlx: { try await self.llm.loadModel(id: model) } - ) - if result.usedMLX { - await espressoLLM.recordMLXFallback() - let message = await espressoLLM.consumeLastFallbackMessage() - return (true, nil, message) - } - } - return (true, nil, nil) - } catch let error as EspressoMLXFallbackError { - await espressoLLM.recordMLXFallbackFailure(details: error.details) - let message = await espressoLLM.consumeLastFailureMessage() - return (false, message ?? error.localizedDescription, nil) - } catch { - Log.error("[TextProcessor] LLM warmup failed: \(error.localizedDescription)") - if backend == .espresso { - _ = await espressoLLM.consumeLastFailureMessage() - } - return (false, error.localizedDescription, nil) - } - } - - func consumeEspressoFailureMessage() async -> String? { - await espressoLLM.consumeLastFailureMessage() - } - - func consumeEspressoFallbackMessage() async -> String? { - await espressoLLM.consumeLastFallbackMessage() - } - func basicClean( text: String, inputLanguage: InputLanguage = .auto, @@ -164,33 +110,35 @@ final class TextProcessor { var result: String let llmStarted = CFAbsoluteTimeGetCurrent() if let screenImage, useScreenImage { - do { - result = try await generateWithScreenImage( - prompt: userPrompt, - systemPrompt: systemPrompt, - model: options.llmModel, - image: screenImage, - maxTokens: generationOptions.maxTokens, - temperature: generationOptions.temperature - ) - } catch { - Log.error("[TextProcessor] VLM failed, falling back to text LLM: \(error.localizedDescription)") - let textFallbackSystemPrompt = formattingSystemPrompt( - options: options, - screenContext: screenContext, - screenImageAvailable: false, - memoryContext: memoryContext, - inputContext: inputContext, - formatKind: formatKind, - dictionarySnapshot: dictionarySnapshot - ) - result = try await generateText( - prompt: userPrompt, - systemPrompt: textFallbackSystemPrompt, - options: options, - maxTokens: generationOptions.maxTokens, - temperature: generationOptions.temperature - ) + result = try await withLocalModelAccess { + do { + return try await generateWithScreenImage( + prompt: userPrompt, + systemPrompt: systemPrompt, + model: options.llmModel, + image: screenImage, + maxTokens: generationOptions.maxTokens, + temperature: generationOptions.temperature + ) + } catch { + Log.error("[TextProcessor] VLM failed, falling back to text LLM: \(error.localizedDescription)") + let textFallbackSystemPrompt = formattingSystemPrompt( + options: options, + screenContext: screenContext, + screenImageAvailable: false, + memoryContext: memoryContext, + inputContext: inputContext, + formatKind: formatKind, + dictionarySnapshot: dictionarySnapshot + ) + return try await generateText( + prompt: userPrompt, + systemPrompt: textFallbackSystemPrompt, + options: options, + maxTokens: generationOptions.maxTokens, + temperature: generationOptions.temperature + ) + } } } else { result = try await generateText( @@ -283,32 +231,34 @@ final class TextProcessor { do { var result: String if let screenImage, useScreenImage { - do { - result = try await generateWithScreenImage( - prompt: userPrompt, - systemPrompt: systemPrompt, - model: options.llmModel, - image: screenImage, - maxTokens: 4096, - temperature: 0.3 - ) - } catch { - Log.error("[TextProcessor] Command VLM failed, falling back to text LLM: \(error.localizedDescription)") - let textFallbackSystemPrompt = commandSystemPrompt( - options: options, - screenContext: screenContext, - screenImageAvailable: false, - memoryContext: memoryContext, - inputContext: inputContext, - dictionarySnapshot: dictionarySnapshot - ) - result = try await generateText( - prompt: userPrompt, - systemPrompt: textFallbackSystemPrompt, - options: options, - maxTokens: 4096, - temperature: 0.3 - ) + result = try await withLocalModelAccess { + do { + return try await generateWithScreenImage( + prompt: userPrompt, + systemPrompt: systemPrompt, + model: options.llmModel, + image: screenImage, + maxTokens: 4096, + temperature: 0.3 + ) + } catch { + Log.error("[TextProcessor] Command VLM failed, falling back to text LLM: \(error.localizedDescription)") + let textFallbackSystemPrompt = commandSystemPrompt( + options: options, + screenContext: screenContext, + screenImageAvailable: false, + memoryContext: memoryContext, + inputContext: inputContext, + dictionarySnapshot: dictionarySnapshot + ) + return try await generateText( + prompt: userPrompt, + systemPrompt: textFallbackSystemPrompt, + options: options, + maxTokens: 4096, + temperature: 0.3 + ) + } } } else { result = try await generateText( diff --git a/Sources/UI/ModelManagementRows.swift b/Sources/UI/ModelManagementRows.swift index 13f491cb..a92ddc86 100644 --- a/Sources/UI/ModelManagementRows.swift +++ b/Sources/UI/ModelManagementRows.swift @@ -263,12 +263,13 @@ extension ModelManagementView { } func runBenchmark(_ modelID: String) async { - guard let idx = catalog.llmModels.firstIndex(where: { $0.id == modelID }) else { return } + guard let idx = catalog.llmModels.firstIndex(where: { $0.id == modelID }), + let onBenchmarkLLM else { return } catalog.llmModels[idx].isBenchmarking = true catalog.llmModels[idx].benchmarkTPS = nil do { - let result = try await benchmarkEngine.benchmark(modelID: modelID) + let result = try await onBenchmarkLLM(modelID) if let i = catalog.llmModels.firstIndex(where: { $0.id == modelID }) { catalog.llmModels[i].benchmarkTPS = result.tokensPerSecond catalog.llmModels[i].isBenchmarking = false @@ -281,30 +282,4 @@ extension ModelManagementView { } } - func secondaryText(for model: ModelCatalog.ModelEntry) -> String { - switch model.status { - case .unavailable(let message), .error(let message): - return message - default: - return model.hint - } - } - - func statusDot(_ status: ModelCatalog.ModelStatus) -> some View { - Group { - switch status { - case .notDownloaded: - Circle().fill(.secondary.opacity(0.3)) - case .downloading, .compiling, .loading: - ProgressView().controlSize(.mini) - case .downloaded, .ready: - Circle().fill(.green) - case .unavailable: - Circle().fill(.orange) - case .error: - Circle().fill(.red) - } - } - .frame(width: 8, height: 8) - } } diff --git a/Sources/UI/ModelManagementStatus.swift b/Sources/UI/ModelManagementStatus.swift new file mode 100644 index 00000000..e2520818 --- /dev/null +++ b/Sources/UI/ModelManagementStatus.swift @@ -0,0 +1,30 @@ +import SwiftUI + +extension ModelManagementView { + func secondaryText(for model: ModelCatalog.ModelEntry) -> String { + switch model.status { + case .unavailable(let message), .error(let message): + return message + default: + return model.hint + } + } + + func statusDot(_ status: ModelCatalog.ModelStatus) -> some View { + Group { + switch status { + case .notDownloaded: + Circle().fill(.secondary.opacity(0.3)) + case .downloading, .compiling, .loading: + ProgressView().controlSize(.mini) + case .downloaded, .ready: + Circle().fill(.green) + case .unavailable: + Circle().fill(.orange) + case .error: + Circle().fill(.red) + } + } + .frame(width: 8, height: 8) + } +} diff --git a/Sources/UI/ModelManagementView.swift b/Sources/UI/ModelManagementView.swift index 8159b76f..9fe856ad 100644 --- a/Sources/UI/ModelManagementView.swift +++ b/Sources/UI/ModelManagementView.swift @@ -10,6 +10,7 @@ struct ModelManagementView: View { var onUnloadWhisper: (() -> Void)? var onUnloadLLM: (() -> Void)? var onLoadLLM: (() -> Void)? + var onBenchmarkLLM: ((String) async throws -> LLMEngine.BenchmarkResult)? var onUnloadLocalASR: (() -> Void)? @State var customLLMInput = "" @@ -18,7 +19,6 @@ struct ModelManagementView: View { @State var selectedModelFamily: ModelCatalog.ModelFamily? = .qwen @State var showLegacyModels = false @State var pendingModelAction: PendingModelAction? - let benchmarkEngine = LLMEngine() var body: some View { VStack(spacing: 0) { diff --git a/Sources/UI/OverlayPanelContent.swift b/Sources/UI/OverlayPanelContent.swift index 0318a083..6ec68963 100644 --- a/Sources/UI/OverlayPanelContent.swift +++ b/Sources/UI/OverlayPanelContent.swift @@ -18,7 +18,7 @@ struct OverlayLayout: Equatable { init(appState: AppState) { let hasPreview = appState.phase == .recording && !appState.rawTranscription.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty let showsEspressoFallback = appState.phase == .done - && appState.statusMessage == L("status.espresso_fell_back_to_mlx") + && appState.completionKind == .espressoFallback isInteractive = appState.isRecording switch appState.phase { @@ -101,7 +101,7 @@ struct OverlayContentView: View { private var showsEspressoFallback: Bool { appState.phase == .done - && appState.statusMessage == L("status.espresso_fell_back_to_mlx") + && appState.completionKind == .espressoFallback } var body: some View { diff --git a/Sources/UI/SettingsView.swift b/Sources/UI/SettingsView.swift index c5b5eea9..d584320c 100644 --- a/Sources/UI/SettingsView.swift +++ b/Sources/UI/SettingsView.swift @@ -23,6 +23,7 @@ struct SettingsView: View { var onUnloadWhisper: (() -> Void)? var onUnloadLLM: (() -> Void)? var onLoadLLM: (() -> Void)? + var onBenchmarkLLM: ((String) async throws -> LLMEngine.BenchmarkResult)? var onUnloadLocalASR: (() -> Void)? var body: some View { @@ -35,6 +36,7 @@ struct SettingsView: View { onUnloadWhisper: onUnloadWhisper, onUnloadLLM: onUnloadLLM, onLoadLLM: onLoadLLM, + onBenchmarkLLM: onBenchmarkLLM, onUnloadLocalASR: onUnloadLocalASR ) .tabItem { Label(L("tab.models"), systemImage: "cpu") } diff --git a/Tests/OpenTypeTests/ConfigurationTests.swift b/Tests/OpenTypeTests/ConfigurationTests.swift index e2e78e09..e24736d9 100644 --- a/Tests/OpenTypeTests/ConfigurationTests.swift +++ b/Tests/OpenTypeTests/ConfigurationTests.swift @@ -301,18 +301,6 @@ final class ConfigurationTests: XCTestCase { XCTAssertTrue( Loc.string("error.espresso_runtime_failed", language: .chinese).contains("MLX") ) - XCTAssertTrue( - Loc.string("status.espresso_fell_back_to_mlx", language: .english).contains("MLX") - ) - XCTAssertTrue( - Loc.string("status.espresso_fell_back_to_mlx", language: .chinese).contains("MLX") - ) - XCTAssertTrue( - Loc.string("error.espresso_mlx_fallback_unavailable", language: .english).contains("MLX") - ) - XCTAssertTrue( - Loc.string("error.espresso_mlx_fallback_unavailable", language: .chinese).contains("MLX") - ) } func testDeveloperHTTPTokenCanBeReset() { diff --git a/Tests/OpenTypeTests/EspressoFallbackTests.swift b/Tests/OpenTypeTests/EspressoFallbackTests.swift new file mode 100644 index 00000000..6d9cb7d5 --- /dev/null +++ b/Tests/OpenTypeTests/EspressoFallbackTests.swift @@ -0,0 +1,232 @@ +import XCTest +import MLX +@testable import OpenType + +final class EspressoFallbackTests: XCTestCase { + private final class WeakTrackerBox { + weak var value: EspressoGenerationTracker? + } + + private enum StubError: LocalizedError { + case espresso + case mlx + + var errorDescription: String? { + switch self { + case .espresso: return "espresso failed" + case .mlx: return "mlx failed" + } + } + } + + func testEspressoSuccessDoesNotRunMLXFallback() async throws { + var ranMLX = false + let result = try await TextProcessor.runEspressoWithMLXFallback( + espresso: { "espresso output" }, + mlx: { + ranMLX = true + return "mlx output" + } + ) + + XCTAssertEqual(result.value, "espresso output") + XCTAssertFalse(result.usedMLX) + XCTAssertFalse(ranMLX) + } + + func testEspressoFailureUsesMLXFallback() async throws { + let result = try await TextProcessor.runEspressoWithMLXFallback( + espresso: { throw StubError.espresso }, + mlx: { "mlx output" } + ) + + XCTAssertEqual(result.value, "mlx output") + XCTAssertTrue(result.usedMLX) + } + + func testCancellationDoesNotStartMLXFallback() async { + let espressoStarted = expectation(description: "Espresso started") + var ranMLX = false + let task = Task { + try await TextProcessor.runEspressoWithMLXFallback( + espresso: { + espressoStarted.fulfill() + try await Task.sleep(for: .seconds(10)) + return "espresso output" + }, + mlx: { + ranMLX = true + return "mlx output" + } + ) + } + + await fulfillment(of: [espressoStarted], timeout: 1) + task.cancel() + + do { + _ = try await task.value + XCTFail("Expected cancellation") + } catch is CancellationError { + XCTAssertFalse(ranMLX) + } catch { + XCTFail("Unexpected error: \(error)") + } + } + + func testEspressoAndMLXFailuresPreserveBothDiagnostics() async { + do { + _ = try await TextProcessor.runEspressoWithMLXFallback( + espresso: { throw StubError.espresso }, + mlx: { throw StubError.mlx } + ) as (value: String, usedMLX: Bool) + XCTFail("Expected both local backends to fail") + } catch let error as TextProcessor.EspressoMLXFallbackError { + XCTAssertTrue(error.details.contains("espresso failed")) + XCTAssertTrue(error.details.contains("mlx failed")) + } catch { + XCTFail("Unexpected error: \(error)") + } + } + + func testOutcomeTrackingIsRequestScoped() async { + let processor = TextProcessor() + var first: EspressoGenerationOutcome? + var second: EspressoGenerationOutcome? + + await TextProcessor.withEspressoOutcomeTracking { + await TextProcessor.recordEspressoOutcome(.fallback) + first = await processor.consumeEspressoOutcome() + } + await TextProcessor.withEspressoOutcomeTracking { + second = await processor.consumeEspressoOutcome() + } + + XCTAssertEqual(first, .fallback) + XCTAssertNil(second) + } + + func testOutcomeTrackerIsReleasedAfterRequestCompletes() async { + let box = WeakTrackerBox() + + await TextProcessor.withEspressoOutcomeTracking { + box.value = TextProcessor.espressoGenerationTracker + XCTAssertNotNil(box.value) + } + + XCTAssertNil(box.value) + } + + func testFallbackOutcomeSwitchesPersistedBackendToMLX() { + let suiteName = "EspressoFallbackTests.\(UUID().uuidString)" + let defaults = UserDefaults(suiteName: suiteName)! + defer { defaults.removePersistentDomain(forName: suiteName) } + let settings = AppSettings(defaults: defaults) + settings.localLLMBackend = .espresso + + XCTAssertTrue(EspressoFallbackPolicy.selectMLXIfNeeded( + after: .fallback, + settings: settings, + expectedEspressoModelPath: settings.espressoModelPath + )) + XCTAssertEqual(settings.localLLMBackend, .mlx) + XCTAssertEqual(defaults.string(forKey: "localLLMBackend"), LocalLLMBackend.mlx.rawValue) + } + + func testFallbackOutcomePreservesNewerEspressoSelection() { + let suiteName = "EspressoFallbackTests.\(UUID().uuidString)" + let defaults = UserDefaults(suiteName: suiteName)! + defer { defaults.removePersistentDomain(forName: suiteName) } + let settings = AppSettings(defaults: defaults) + settings.localLLMBackend = .espresso + settings.espressoModelPath = "/models/new.esp" + + XCTAssertFalse(EspressoFallbackPolicy.selectMLXIfNeeded( + after: .fallback, + settings: settings, + expectedEspressoModelPath: "/models/old.esp" + )) + XCTAssertEqual(settings.localLLMBackend, .espresso) + } + + func testLocalizedFallbackMessagesMentionMLX() { + for language in [UILanguage.english, .chinese] { + XCTAssertTrue(Loc.string("status.espresso_fell_back_to_mlx", language: language).contains("MLX")) + XCTAssertTrue(Loc.string("error.espresso_mlx_fallback_unavailable", language: language).contains("MLX")) + } + } + + func testRealEspressoFailureFallsBackToInstalledMLX() async throws { + let environment = ProcessInfo.processInfo.environment + guard environment["OPENTYPE_ESPRESSO_MLX_FALLBACK_INTEGRATION"] == "1" else { + throw XCTSkip("Set OPENTYPE_ESPRESSO_MLX_FALLBACK_INTEGRATION=1 to run") + } + guard let bundlePath = environment["OPENTYPE_ESPRESSO_BUNDLE"], + let mlxModel = environment["OPENTYPE_MLX_MODEL"] else { + throw XCTSkip("Set OPENTYPE_ESPRESSO_BUNDLE and OPENTYPE_MLX_MODEL") + } + + var options = TextProcessingOptions(settings: AppSettings.shared, inputLanguage: .english) + options.useRemoteLLM = false + options.localLLMBackend = .espresso + options.espressoModelPath = bundlePath + options.llmModel = mlxModel + + let processor = TextProcessor() + let initialFootprint = currentMemoryFootprint() + let iterations = Int(environment["OPENTYPE_FALLBACK_ITERATIONS"] ?? "1") ?? 1 + var output = "" + var outcome: EspressoGenerationOutcome? + var baselineFootprint: UInt64? + try await TextProcessor.withEspressoOutcomeTracking { + for index in 0.. 1, let baselineFootprint { + let currentFootprint = currentMemoryFootprint() + let growth = currentFootprint > baselineFootprint + ? currentFootprint - baselineFootprint + : 0 + XCTAssertLessThan(growth, 384 * 1_024 * 1_024, "Repeated MLX requests retained \(growth) bytes") + } + await processor.unloadLLM() + try await Task.sleep(for: .milliseconds(500)) + XCTAssertLessThan(Memory.activeMemory, 1 * 1_024 * 1_024) + XCTAssertEqual(Memory.cacheMemory, 0) + let footprintAfterUnload = currentMemoryFootprint() + let retainedAfterUnload = footprintAfterUnload > initialFootprint + ? footprintAfterUnload - initialFootprint + : 0 + XCTAssertLessThan( + retainedAfterUnload, + 1_024 * 1_024 * 1_024, + "Local model unload retained \(retainedAfterUnload) bytes" + ) + } +} + +private func currentMemoryFootprint() -> UInt64 { + var info = task_vm_info_data_t() + var count = mach_msg_type_number_t(MemoryLayout.size / MemoryLayout.size) + let result = withUnsafeMutablePointer(to: &info) { + $0.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { + task_info(mach_task_self_, task_flavor_t(TASK_VM_INFO), $0, &count) + } + } + return result == KERN_SUCCESS ? info.phys_footprint : 0 +} diff --git a/Tests/OpenTypeTests/IntegrationHTTPTests.swift b/Tests/OpenTypeTests/IntegrationHTTPTests.swift index 641256d1..bfbb8228 100644 --- a/Tests/OpenTypeTests/IntegrationHTTPTests.swift +++ b/Tests/OpenTypeTests/IntegrationHTTPTests.swift @@ -106,6 +106,10 @@ final class IntegrationHTTPTests: XCTestCase { XCTAssertEqual(IntegrationHTTPDispatcher.statusCode(for: IntegrationError.modelNotReady), 400) XCTAssertEqual(IntegrationHTTPDispatcher.statusCode(for: IntegrationError.noSpeechDetected), 400) XCTAssertEqual(IntegrationHTTPDispatcher.statusCode(for: IntegrationError.operationFailed), 500) + XCTAssertEqual( + IntegrationHTTPDispatcher.statusCode(for: IntegrationError.operationFailedWithMessage("details")), + 500 + ) } @MainActor diff --git a/Tests/OpenTypeTests/IntegrationOutputTests.swift b/Tests/OpenTypeTests/IntegrationOutputTests.swift index ff638ecb..fbf7733d 100644 --- a/Tests/OpenTypeTests/IntegrationOutputTests.swift +++ b/Tests/OpenTypeTests/IntegrationOutputTests.swift @@ -5,6 +5,12 @@ import XCTest @MainActor final class IntegrationOutputTests: XCTestCase { + func testAppDelegateSharesTextProcessorWithIntegrationCoordinator() { + let delegate = AppDelegate() + + XCTAssertTrue(delegate.integrationSessionCoordinator.textProcessor === delegate.textProcessor) + } + func testServiceRejectsEmptyFinalText() async throws { let store = registry() defer { store.cleanup() } @@ -41,6 +47,38 @@ final class IntegrationOutputTests: XCTestCase { _ = try await coordinator.outputText(for: " ", active: active) } } + + func testCoordinatorPreservesCombinedLocalModelFailureGuidance() async { + let store = registry() + defer { store.cleanup() } + store.registry.approve(IntegrationClient.localHTTP(tokenID: "token")) + let coordinator = InputSessionCoordinator(service: makeService(registry: store.registry)) + let active = InputSessionCoordinator.ActiveSession( + sessionID: UUID(), + clientID: clientID, + engine: TestSpeechEngine(transcript: ""), + languageCode: nil, + mode: .direct, + inputLanguage: .english, + useScreenContext: false, + streamingEnabled: false, + screenContextTask: nil, + client: IntegrationClient.localHTTP(tokenID: "token") + ) + + do { + try await TextProcessor.withEspressoOutcomeTracking { + await TextProcessor.recordEspressoOutcome(.unavailable) + _ = try await coordinator.outputText(for: " ", active: active) + } + XCTFail("Expected local model failure") + } catch let error as IntegrationError { + XCTAssertEqual(error, .operationFailedWithMessage(EspressoGenerationOutcome.unavailable.message)) + XCTAssertEqual(error.payload.error, "operation_failed") + } catch { + XCTFail("Unexpected error: \(error)") + } + } } private extension IntegrationOutputTests { diff --git a/Tests/OpenTypeTests/LocalModelAccessTests.swift b/Tests/OpenTypeTests/LocalModelAccessTests.swift new file mode 100644 index 00000000..4dd27198 --- /dev/null +++ b/Tests/OpenTypeTests/LocalModelAccessTests.swift @@ -0,0 +1,110 @@ +import XCTest +@testable import OpenType + +final class LocalModelAccessTests: XCTestCase { + private actor EventRecorder { + private var events: [String] = [] + + func append(_ event: String) { + events.append(event) + } + + func snapshot() -> [String] { + events + } + } + + func testLocalModelAccessIsReentrantWithinOneTask() async throws { + let processor = TextProcessor() + let recorder = EventRecorder() + + try await processor.withLocalModelAccess { + try await processor.withLocalModelAccess { + await recorder.append("nested") + } + } + + let events = await recorder.snapshot() + XCTAssertEqual(events, ["nested"]) + } + + func testUnloadWaitsForActiveLocalModelOperation() async { + let processor = TextProcessor() + let recorder = EventRecorder() + let operationStarted = expectation(description: "Local model operation started") + let unloadStarted = expectation(description: "Unload started") + let (releaseStream, releaseContinuation) = AsyncStream.makeStream() + + let operation = Task { + try await processor.withLocalModelAccess { + await recorder.append("operation-start") + operationStarted.fulfill() + for await _ in releaseStream { break } + await recorder.append("operation-end") + } + } + await fulfillment(of: [operationStarted], timeout: 1) + + let unload = Task { + unloadStarted.fulfill() + await processor.unloadLLM() + await recorder.append("unload-end") + } + await fulfillment(of: [unloadStarted], timeout: 1) + try? await Task.sleep(for: .milliseconds(25)) + let eventsWhileOperationIsActive = await recorder.snapshot() + XCTAssertEqual(eventsWhileOperationIsActive, ["operation-start"]) + + releaseContinuation.yield() + releaseContinuation.finish() + try? await operation.value + await unload.value + + let finalEvents = await recorder.snapshot() + XCTAssertEqual(finalEvents, ["operation-start", "operation-end", "unload-end"]) + } + + func testCancelledLocalModelWaiterDoesNotRunAfterGateOpens() async { + let processor = TextProcessor() + let recorder = EventRecorder() + let holderStarted = expectation(description: "Gate holder started") + let (releaseStream, releaseContinuation) = AsyncStream.makeStream() + + let holder = Task { + try await processor.withLocalModelAccess { + holderStarted.fulfill() + for await _ in releaseStream { break } + } + } + await fulfillment(of: [holderStarted], timeout: 1) + + let waiter = Task { + try await processor.withLocalModelAccess { + await recorder.append("waiter-ran") + } + } + for _ in 0..<1_000 { + if await processor.localModelAccessGate.waitingTaskCount == 1 { break } + await Task.yield() + } + let queuedWaiters = await processor.localModelAccessGate.waitingTaskCount + XCTAssertEqual(queuedWaiters, 1) + waiter.cancel() + + do { + try await waiter.value + XCTFail("Expected cancellation") + } catch is CancellationError { + } catch { + XCTFail("Unexpected error: \(error)") + } + let remainingWaiters = await processor.localModelAccessGate.waitingTaskCount + XCTAssertEqual(remainingWaiters, 0) + + releaseContinuation.yield() + releaseContinuation.finish() + try? await holder.value + let events = await recorder.snapshot() + XCTAssertTrue(events.isEmpty) + } +} diff --git a/Tests/OpenTypeTests/OverlayLayoutTests.swift b/Tests/OpenTypeTests/OverlayLayoutTests.swift index ccbf192e..9ab9541d 100644 --- a/Tests/OpenTypeTests/OverlayLayoutTests.swift +++ b/Tests/OpenTypeTests/OverlayLayoutTests.swift @@ -52,6 +52,7 @@ final class OverlayLayoutTests: XCTestCase { let appState = AppState() appState.phase = .done appState.statusMessage = L("status.espresso_fell_back_to_mlx") + appState.completionKind = .espressoFallback let layout = OverlayLayout(appState: appState) @@ -60,6 +61,15 @@ final class OverlayLayoutTests: XCTestCase { XCTAssertFalse(layout.isInteractive) } + func testFallbackLayoutDoesNotDependOnLocalizedMessageText() { + let appState = AppState() + appState.phase = .done + appState.statusMessage = "Changed copy" + appState.completionKind = .espressoFallback + + XCTAssertEqual(OverlayLayout(appState: appState).height, 56) + } + func testOverlayPlacementCentersAboveVisibleScreenBottom() { let visibleFrame = CGRect(x: 100, y: 80, width: 1_200, height: 760) let frame = OverlayPanelPlacement.frame( diff --git a/Tests/OpenTypeTests/TextProcessorFallbackTests.swift b/Tests/OpenTypeTests/TextProcessorFallbackTests.swift index e2f3fa95..6f0b5362 100644 --- a/Tests/OpenTypeTests/TextProcessorFallbackTests.swift +++ b/Tests/OpenTypeTests/TextProcessorFallbackTests.swift @@ -2,18 +2,6 @@ import XCTest @testable import OpenType final class TextProcessorFallbackTests: XCTestCase { - private enum StubError: LocalizedError { - case espresso - case mlx - - var errorDescription: String? { - switch self { - case .espresso: return "espresso failed" - case .mlx: return "mlx failed" - } - } - } - func testSmartFormatDoesNotUsePreparedFallbackByDefault() { XCTAssertFalse(TextProcessor.defaultAllowsPreparedFallback) } @@ -204,91 +192,4 @@ final class TextProcessorFallbackTests: XCTestCase { XCTAssertFalse(commandPrompt.contains("A screen image is attached")) } - func testEspressoSuccessDoesNotRunMLXFallback() async throws { - var ranMLX = false - - let result = try await TextProcessor.runEspressoWithMLXFallback( - espresso: { "espresso output" }, - mlx: { - ranMLX = true - return "mlx output" - } - ) - - XCTAssertEqual(result.value, "espresso output") - XCTAssertFalse(result.usedMLX) - XCTAssertFalse(ranMLX) - } - - func testEspressoFailureUsesMLXFallback() async throws { - let result = try await TextProcessor.runEspressoWithMLXFallback( - espresso: { throw StubError.espresso }, - mlx: { "mlx output" } - ) - - XCTAssertEqual(result.value, "mlx output") - XCTAssertTrue(result.usedMLX) - } - - func testEspressoAndMLXFailuresPreserveBothDiagnostics() async { - do { - _ = try await TextProcessor.runEspressoWithMLXFallback( - espresso: { throw StubError.espresso }, - mlx: { throw StubError.mlx } - ) as (value: String, usedMLX: Bool) - XCTFail("Expected both local backends to fail") - } catch let error as TextProcessor.EspressoMLXFallbackError { - XCTAssertTrue(error.details.contains("espresso failed")) - XCTAssertTrue(error.details.contains("mlx failed")) - } catch { - XCTFail("Unexpected error: \(error)") - } - } - - @MainActor - func testFallbackNoticeSwitchesPersistedBackendToMLX() async { - let suiteName = "TextProcessorFallbackTests.\(UUID().uuidString)" - let defaults = UserDefaults(suiteName: suiteName)! - defer { defaults.removePersistentDomain(forName: suiteName) } - let settings = AppSettings(defaults: defaults) - settings.localLLMBackend = .espresso - let pipeline = VoicePipeline(appState: AppState()) - await pipeline.textProcessor.espressoLLM.recordMLXFallback() - - let message = await pipeline.applyEspressoFallbackIfNeeded(settings: settings) - - XCTAssertEqual(settings.localLLMBackend, .mlx) - XCTAssertEqual(defaults.string(forKey: "localLLMBackend"), LocalLLMBackend.mlx.rawValue) - XCTAssertEqual(message, L("status.espresso_fell_back_to_mlx")) - } - - func testRealEspressoFailureFallsBackToInstalledMLX() async throws { - let environment = ProcessInfo.processInfo.environment - guard environment["OPENTYPE_ESPRESSO_MLX_FALLBACK_INTEGRATION"] == "1" else { - throw XCTSkip("Set OPENTYPE_ESPRESSO_MLX_FALLBACK_INTEGRATION=1 to run") - } - guard let bundlePath = environment["OPENTYPE_ESPRESSO_BUNDLE"], - let mlxModel = environment["OPENTYPE_MLX_MODEL"] else { - throw XCTSkip("Set OPENTYPE_ESPRESSO_BUNDLE and OPENTYPE_MLX_MODEL") - } - - var options = TextProcessingOptions(settings: AppSettings.shared, inputLanguage: .english) - options.useRemoteLLM = false - options.localLLMBackend = .espresso - options.espressoModelPath = bundlePath - options.llmModel = mlxModel - - let processor = TextProcessor() - let output = try await processor.generateText( - prompt: "Reply with exactly OK.", - systemPrompt: "Return only the requested answer.", - options: options, - maxTokens: 8, - temperature: 0 - ) - - let fallbackMessage = await processor.consumeEspressoFallbackMessage() - XCTAssertFalse(output.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) - XCTAssertNotNil(fallbackMessage) - } } diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/intent.md b/docs/sdlc/changes/2026-08-29-espresso-ane/intent.md index 29f2e44e..7c397f94 100644 --- a/docs/sdlc/changes/2026-08-29-espresso-ane/intent.md +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/intent.md @@ -36,6 +36,9 @@ Store compatibility for Espresso's private ANE API. persists MLX as the active backend, and surfaces the fallback to the user. - If the selected MLX model is unavailable, the Espresso failure remains visible with guidance to install an MLX model. +- Repeated fallback requests do not retain request-scoped state, and explicitly + unloading local models waits for active local inference, releases every + production model container, and clears MLX's reusable memory cache. - Existing local MLX and remote LLM behavior remains covered by passing tests. - The available M5/macOS 27 host rejects Espresso's private ANE program without preventing local formatting when a compatible MLX model is installed. diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md b/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md index 63ad9121..fd676d44 100644 --- a/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md @@ -14,6 +14,10 @@ - [x] Fall back from failed Espresso warmup and generation to an installed MLX model. - [x] Persist MLX after successful fallback and surface a localized completion notice. - [x] Cover fallback ordering, success, and dual-failure behavior with focused tests. +- [x] Scope fallback outcomes to one request and preserve actionable dual-failure guidance. +- [x] Reproduce retained MLX memory after unload and clear the MLX cache on explicit unload. +- [x] Share one processor across voice and integration workflows and include benchmarking in its lifecycle. +- [x] Serialize complete local-model transactions and remove cancelled waiters from the lifecycle queue. ## Verification plan @@ -26,6 +30,8 @@ - [x] Real M5 ANE compile matrix across deployment targets and normalization variants - [x] Targeted Espresso fallback tests - [x] Current full repository gates and release-style app build +- [x] Repeated real fallback requests and explicit-unload memory regression +- [x] Independent cancellation, unload-order, shared-ownership, and file-size review ## Human gates diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/spec.md b/docs/sdlc/changes/2026-08-29-espresso-ane/spec.md index 1be27054..2549ba9f 100644 --- a/docs/sdlc/changes/2026-08-29-espresso-ane/spec.md +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/spec.md @@ -25,10 +25,18 @@ For local Espresso warmup and generation, `TextProcessor` first attempts the selected `.esp` bundle. If that attempt fails, it tries the already-selected MLX model through the existing `LLMEngine`; `LLMEngine` continues to require a complete local model and never downloads during fallback. A successful MLX -fallback records a localized notice. The main app consumes that notice, changes -the persisted backend to MLX only if Espresso is still selected, and shows the -notice in the completion state. Integration sessions also persist the backend -change and log it without changing their response schema. +fallback records a request-scoped semantic outcome. The main app consumes that +outcome, changes the persisted backend to MLX only if Espresso is still +selected, and shows the notice in the completion state. Integration sessions +also persist the backend change and log it without changing their response +schema. + +The voice pipeline, integration sessions, and model benchmark UI share one +application-owned `TextProcessor`. A cancellable FIFO gate serializes complete +local-model transactions, including multimodal-to-text fallback, benchmarking, +warmup, and explicit unload. Benchmark containers are dropped after each run; +explicit unload waits for active work, drops every local container, and then +clears MLX's reusable allocation cache. Pin Espresso to the reviewed `v0.9.0` source plus a three-line Swift 6.2 compatibility patch. The patch gives three internal compiled-kernel holder @@ -49,7 +57,10 @@ The UI warning explicitly states the private-API and App Store limitation. No fallback is silent: successful fallback produces a localized status message and changes the persisted backend to MLX so later requests do not repeatedly compile a rejected private ANE program. A concurrent user change away from -Espresso is preserved. +Espresso is preserved. Outcome tracking is scoped to one processing task, so +cancellation, unload, language changes, and unrelated requests cannot consume +stale fallback state. Explicit model unload drops all local model containers and +clears MLX's reusable memory cache. The patched dependency is pinned by commit rather than a moving branch. Rollback returns the package URL and version requirement to upstream `v0.9.0` once an @@ -58,10 +69,11 @@ experimental backend. ## Test strategy -Persistence, prompt formatting, and Espresso-to-MLX fallback ordering and -failure behavior have focused unit tests. The complete Swift suite and -repository checks cover existing paths and package integration. A -release-style app build checks dependency and Metal/resource packaging. Real +Persistence, prompt formatting, request-scoped Espresso-to-MLX fallback +ordering, failure behavior, cancellable local-model serialization, shared +processor wiring, and unload memory behavior have focused tests. The complete +Swift suite and repository checks cover existing paths and package integration. +A release-style app build checks dependency and Metal/resource packaging. Real inference is exercised with a prepared GPT-2 `.esp` bundle and recorded even if the host's private ANE compiler rejects it. diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/state.json b/docs/sdlc/changes/2026-08-29-espresso-ane/state.json index b0fbfa6a..5257db0f 100644 --- a/docs/sdlc/changes/2026-08-29-espresso-ane/state.json +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/state.json @@ -3,7 +3,7 @@ "id": "2026-08-29-espresso-ane", "title": "Add a selectable Espresso ANE inference backend", "risk": "high", - "status": "implementing", + "status": "verified", "owners": [ "repository maintainer" ], @@ -13,6 +13,7 @@ "When Espresso is selected, local text generation and model warmup use its ANE runtime.", "When Espresso fails, an installed selected MLX model completes the request, the persisted backend changes to MLX, and the user sees the fallback.", "When both Espresso and MLX are unavailable, the user sees actionable local-model guidance.", + "Fallback outcome tracking is request-scoped and explicit local-model unload waits for active local work, releases every production model container, and clears the MLX memory cache.", "The existing MLX and remote LLM paths continue to pass the automated test suite.", "A real Espresso failure on the available M5 Max and macOS 27 host does not prevent local formatting when MLX is available." ], diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md b/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md index 9cf5c3b2..33672c9b 100644 --- a/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md @@ -5,17 +5,19 @@ | Check | Result | Evidence | |---|---|---| | `bash scripts/ci-basic-checks.sh` | Pass | Current SDLC, harness, plist, localization, resource, identifier, secret-file, and symlink checks passed | -| `DEVELOPER_DIR=/Applications/Xcode-beta.app/Contents/Developer swift test` | Pass | 573 XCTest tests passed, 9 skipped, plus 1 Swift Testing test passed after the fallback and overlay changes | -| Focused fallback and localization tests | Pass | Espresso success avoids MLX; Espresso failure uses MLX; dual failure preserves both diagnostics; successful fallback persists MLX; localized status and 288 by 56 two-line overlay layout pass | +| `DEVELOPER_DIR=/Applications/Xcode-beta.app/Contents/Developer swift test` | Pass | 584 XCTest tests passed, 9 skipped, plus 1 Swift Testing test passed after request-scoped fallback and unload-memory changes | +| Focused fallback and localization tests | Pass | Espresso success avoids MLX; Espresso failure uses MLX; cancellation skips fallback; dual failure preserves both diagnostics; request trackers deallocate and do not cross requests; successful fallback persists MLX; localized status and typed 288 by 56 two-line overlay layout pass | | `DEVELOPER_DIR=/Applications/Xcode-beta.app/Contents/Developer bash scripts/build-app.sh --app-only --sign=-` | Pass | Final source without the temporary visual-QA trigger built the Release app and CLI, assembled, ad-hoc signed, and passed artifact verification | | GitHub run `33296463655`, Xcode 26.6 Release build | Fail; diagnosed | Swift 6.2 emitted cross-module references to three internal `RealModelInferenceEngine.Compiled*` metadata symbols, then failed final arm64 linking | | Pinned Espresso commit `f3603c7` symbol probe | Pass | The three metadata symbols are emitted as external after changing only the holder types from internal to package visibility | | GitHub run `33297701825`, Xcode 26.6 | Pass | Contract & Tests, Release-style App Build, and SDLC Gate all passed; the app build completed in 9m12s | -| Independent high-risk review | Completed; findings addressed | Review found generic ANE errors, stale preload publication, public path logging, and stale loading status; fixes surface localized guidance, use a preload generation token, clear owned loading state, and keep path-bearing details private | +| Independent high-risk reviews | Completed; final passes clean | Successive reviews found stale state, cancellation and selection races, unordered unload/reload, independent integration and benchmark containers, multimodal fallback interleaving, and retained cancelled waiters. The final implementation uses request-scoped outcomes, selection identity checks, one shared processor, and a cancellable reentrant lifecycle gate | | Espresso 0.9.0 GPT-2 generation | Fail | M5 Max/macOS 27 ANE compiler returned code 10, `verifyBundleAtPath: invalid model`, while compiling layer 0 attention | | Espresso main `eafb33d` GPT-2 generation | Fail | Latest upstream source produced the same ANE code 10 on the same host | | M5 ANE compile matrix | Fail as expected | All 24 combinations failed: iOS 18, iOS 19, macOS 26, and macOS 27 MIL targets; LayerNorm and RMSNorm; spatial sizes 64, 128, and 256 | | Real Espresso-to-MLX fallback | Pass | On the M5 Max/macOS 27 host, a real GPT-2 `.esp` bundle produced ANE code 10, then the installed `mlx-community/Qwen3.5-2B-4bit` model generated non-empty output in the same test and produced the fallback notice | +| Repeated fallback memory loop | Pass after fix | Twenty post-fallback MLX requests grew physical footprint by about 64 KiB. Before `Memory.clearCache()`, explicit unload retained about 1.74 GiB over baseline; after the fix MLX reported about 3 KiB active and zero cache memory, and the guarded footprint check passed. `vmmap` identified most remaining delta as empty malloc regions and first-loaded framework pages rather than active model tensors | +| Production model ownership and unload ordering | Pass | AppDelegate injects one processor into voice and integration workflows; benchmarking runs through the same processor and releases its container; focused tests prove unload waits for active local work, nested multimodal transactions are reentrant, and cancelled waiters never run | | Real-window fallback notice | Pass | The actual Release app displayed the 288 by 56 non-modal completion overlay without truncation in Chinese light and dark appearances and English dark appearance; the temporary environment-triggered QA entry was removed before the final build | ## Acceptance criteria @@ -29,6 +31,10 @@ Espresso only when Espresso remains the selected backend. - User feedback — pass in both localizations and real Release windows without treating the successful recovery as an error. +- Request and memory lifecycle — pass; outcome trackers deallocate at task end, + repeated requests remain flat, every production model engine is owned by the + shared processor, and explicit unload clears MLX active/cache memory after + active local work completes. - Existing MLX and remote behavior — pass; complete suite has no failures. - Direct Espresso generation on a supported host — blocked on the available M5 Max/macOS 27 host; both the pinned release and upstream main fail in Apple's From 9b94621a50f004cc04146a20909d1fa04cf45965 Mon Sep 17 00:00:00 2001 From: idevlab Date: Sun, 30 Aug 2026 17:45:30 +0800 Subject: [PATCH 08/14] fix: skip MLX cache clear before model load --- Sources/Processing/TextProcessor+Models.swift | 13 ++++++++++--- .../changes/2026-08-29-espresso-ane/verification.md | 1 + 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Sources/Processing/TextProcessor+Models.swift b/Sources/Processing/TextProcessor+Models.swift index ffbca11b..235307af 100644 --- a/Sources/Processing/TextProcessor+Models.swift +++ b/Sources/Processing/TextProcessor+Models.swift @@ -108,11 +108,16 @@ extension TextProcessor { func unloadLLM() async { do { try await withLocalModelAccess { + let llmWasLoaded = await llm.isLoaded + let benchmarkWasLoaded = await benchmarkEngine.isLoaded + let vlmWasLoaded = await vlm.isLoaded await llm.unload() await benchmarkEngine.unload() await espressoLLM.unload() await vlm.unload() - Memory.clearCache() + if llmWasLoaded || benchmarkWasLoaded || vlmWasLoaded { + Memory.clearCache() + } } } catch { Log.info("[TextProcessor] local model unload cancelled") @@ -124,12 +129,14 @@ extension TextProcessor { try Task.checkCancellation() do { let result = try await benchmarkEngine.benchmark(modelID: modelID) + let wasLoaded = await benchmarkEngine.isLoaded await benchmarkEngine.unload() - Memory.clearCache() + if wasLoaded { Memory.clearCache() } return result } catch { + let wasLoaded = await benchmarkEngine.isLoaded await benchmarkEngine.unload() - Memory.clearCache() + if wasLoaded { Memory.clearCache() } throw error } } diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md b/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md index 9e8e1138..777c0476 100644 --- a/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md @@ -11,6 +11,7 @@ | GitHub run `33296463655`, Xcode 26.6 Release build | Fail; diagnosed | Swift 6.2 emitted cross-module references to three internal `RealModelInferenceEngine.Compiled*` metadata symbols, then failed final arm64 linking | | Pinned Espresso commit `f3603c7` symbol probe | Pass | The three metadata symbols are emitted as external after changing only the holder types from internal to package visibility | | GitHub run `33297701825`, Xcode 26.6 | Pass | Contract & Tests, Release-style App Build, and SDLC Gate all passed; the app build completed in 9m12s | +| GitHub run `33304348927`, Xcode 26.6 unit tests | Fail; diagnosed | The empty lifecycle-gate test called `Memory.clearCache()` before any MLX container existed, which made the test runner initialize MLX without an app-bundled default metallib; cache clearing is now conditional on a loaded MLX LLM, benchmark, or VLM container | | Independent high-risk reviews | Completed; final passes clean | Successive reviews found stale state, cancellation and selection races, unordered unload/reload, independent integration and benchmark containers, multimodal fallback interleaving, and retained cancelled waiters. The final implementation uses request-scoped outcomes, selection identity checks, one shared processor, and a cancellable reentrant lifecycle gate | | Espresso 0.9.0 GPT-2 generation | Fail | M5 Max/macOS 27 ANE compiler returned code 10, `verifyBundleAtPath: invalid model`, while compiling layer 0 attention | | Espresso main `eafb33d` GPT-2 generation | Fail | Latest upstream source produced the same ANE code 10 on the same host | From 3534e08efcfdb5192de45f0cf867346a6e82b887 Mon Sep 17 00:00:00 2001 From: idevlab Date: Mon, 31 Aug 2026 00:23:07 +0800 Subject: [PATCH 09/14] fix: make Espresso fallback user-selectable --- Sources/App/VoicePipeline+EditCommands.swift | 2 +- Sources/App/VoicePipeline+Models.swift | 9 +- Sources/App/VoicePipeline+Status.swift | 2 +- Sources/Config/AppSettingTypes.swift | 242 +++++++++++++++++ Sources/Config/AppSettings.swift | 250 +----------------- .../InputSessionCoordinator+Output.swift | 2 +- .../EspressoGenerationOutcome.swift | 12 +- .../Processing/TextProcessingOptions.swift | 2 + .../Processing/TextProcessor+Generation.swift | 21 +- Sources/Processing/TextProcessor+Models.swift | 15 +- .../Resources/en.lproj/Localizable.strings | 4 +- .../zh-Hans.lproj/Localizable.strings | 4 +- Sources/UI/ModelManagementSections.swift | 6 + .../OpenTypeTests/EspressoFallbackTests.swift | 90 ++++--- .../OpenTypeTests/EspressoOutcomeTests.swift | 73 +++++ .../changes/2026-08-29-espresso-ane/intent.md | 24 +- .../changes/2026-08-29-espresso-ane/plan.md | 7 + .../changes/2026-08-29-espresso-ane/spec.md | 44 +-- .../2026-08-29-espresso-ane/state.json | 4 +- .../2026-08-29-espresso-ane/verification.md | 27 +- 20 files changed, 514 insertions(+), 326 deletions(-) create mode 100644 Sources/Config/AppSettingTypes.swift create mode 100644 Tests/OpenTypeTests/EspressoOutcomeTests.swift diff --git a/Sources/App/VoicePipeline+EditCommands.swift b/Sources/App/VoicePipeline+EditCommands.swift index 1c0d3cc0..24ed2442 100644 --- a/Sources/App/VoicePipeline+EditCommands.swift +++ b/Sources/App/VoicePipeline+EditCommands.swift @@ -59,7 +59,7 @@ extension VoicePipeline { ) { if case .error = appState.phase, espressoOutcome == .fallback { Log.info("[VoicePipeline] preserving edit-command error after Espresso fallback") - } else if espressoOutcome == .unavailable { + } else if espressoOutcome != .fallback { showErrorHint(espressoOutcome.message) } else { appState.completionKind = .espressoFallback diff --git a/Sources/App/VoicePipeline+Models.swift b/Sources/App/VoicePipeline+Models.swift index 968d24b8..222a2d52 100644 --- a/Sources/App/VoicePipeline+Models.swift +++ b/Sources/App/VoicePipeline+Models.swift @@ -57,14 +57,17 @@ extension VoicePipeline { let warmup = await textProcessor.warmUpLLM( model: model, backend: backend, - espressoModelPath: espressoPath + espressoModelPath: espressoPath, + fallbackToMLXOnEspressoFailure: settings.fallbackToMLXOnEspressoFailure ) - guard preloadGeneration == formattingPreloadGeneration, + guard !Task.isCancelled, + preloadGeneration == formattingPreloadGeneration, formattingSelectionMatches(backend: backend, model: model, espressoPath: espressoPath) else { return nil } let ready = warmup.loaded ? await textProcessor.isLLMReady(for: backend) : false - guard preloadGeneration == formattingPreloadGeneration, + guard !Task.isCancelled, + preloadGeneration == formattingPreloadGeneration, formattingSelectionMatches(backend: backend, model: model, espressoPath: espressoPath) else { return nil } diff --git a/Sources/App/VoicePipeline+Status.swift b/Sources/App/VoicePipeline+Status.swift index aa3163e0..fb7a37d2 100644 --- a/Sources/App/VoicePipeline+Status.swift +++ b/Sources/App/VoicePipeline+Status.swift @@ -77,7 +77,7 @@ extension VoicePipeline { func presentEspressoWarmupOutcomeIfNeeded(_ outcome: EspressoGenerationOutcome?) { guard let outcome else { return } - if outcome == .unavailable { + if outcome != .fallback { showErrorHint(outcome.message) return } diff --git a/Sources/Config/AppSettingTypes.swift b/Sources/Config/AppSettingTypes.swift new file mode 100644 index 00000000..42dbde1c --- /dev/null +++ b/Sources/Config/AppSettingTypes.swift @@ -0,0 +1,242 @@ +import Foundation + +enum UILanguage: String, Codable, CaseIterable { + case chinese = "zh" + case english = "en" + + var displayName: String { + switch self { + case .chinese: return "中文" + case .english: return "English" + } + } +} + +enum OutputMode: String, Codable, CaseIterable { + case direct = "direct" + case processed = "processed" + case command = "command" + + var label: String { + switch self { + case .direct: return L("mode.verbatim") + case .processed: return L("mode.smart_format") + case .command: return L("mode.voice_command") + } + } +} + +enum SpeechEngineType: String, Codable, CaseIterable { + case whisper = "whisper" + case apple = "apple" + case volc = "volc" + case qwen3 = "qwen3" + case mimo = "mimo" + + static var selectableCases: [SpeechEngineType] { + [.qwen3, .whisper, .apple, .volc] + } + + var label: String { + switch self { + case .whisper: return "WhisperKit" + case .apple: return L("engine.apple_speech") + case .volc: return L("engine.volc_asr") + case .qwen3: return L("engine.qwen3_asr") + case .mimo: return L("engine.mimo_asr") + } + } +} + +enum LocalLLMBackend: String, Codable, CaseIterable { + case mlx + case espresso +} + +enum LanguageStyle: String, Codable, CaseIterable { + case casual = "casual" + case professional = "professional" + case custom = "custom" + + var label: String { + switch self { + case .casual: return L("style.casual") + case .professional: return L("style.professional") + case .custom: return L("style.custom") + } + } + + var defaultPrompt: String { + switch self { + case .casual: return L("style.prompt.casual") + case .professional: return L("style.prompt.professional") + case .custom: return L("style.prompt.custom") + } + } + + var icon: String { + switch self { + case .casual: return "bubble.left" + case .professional: return "list.number" + case .custom: return "slider.horizontal.3" + } + } + + var usesCustomPrompt: Bool { self == .custom } + + static func migrated(from savedValue: String) -> LanguageStyle { + if let style = LanguageStyle(rawValue: savedValue) { + return style + } + + let normalized = savedValue.lowercased() + if normalized.contains("casual") || savedValue.contains("口语") { + return .casual + } + if normalized.contains("custom") || savedValue.contains("自定义") { + return .custom + } + if normalized.contains("professional") + || normalized.contains("formal") + || normalized.contains("concise") + || savedValue.contains("专业") + || savedValue.contains("正式") + || savedValue.contains("简洁") { + return .professional + } + return .professional + } + + static func looksLikePresetPrompt(_ prompt: String) -> Bool { + let normalized = prompt.trimmingCharacters(in: .whitespacesAndNewlines) + let prompts = [ + L("style.prompt.casual"), + L("style.prompt.professional"), + L("style.prompt.concise"), + L("style.prompt.formal"), + ].map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + + return prompts.contains(normalized) + } +} + +enum HotkeyType: String, Codable, CaseIterable { + case ctrl = "Ctrl" + case shift = "Shift" + case option = "Option" + case fn = "Fn" +} + +enum ActivationMode: String, Codable, CaseIterable { + case longPress = "longPress" + case doubleTap = "doubleTap" + case toggle = "toggle" + + var label: String { + switch self { + case .longPress: return L("mode.hold_record") + case .doubleTap: return L("mode.double_tap") + case .toggle: return L("mode.tap_toggle") + } + } +} + +enum HistoryRetention: String, Codable, CaseIterable { + case forever = "forever" + case threeDays = "threeDays" + case sevenDays = "sevenDays" + case oneMonth = "oneMonth" + + var label: String { + switch self { + case .forever: return L("retention.forever") + case .threeDays: return L("retention.three_days") + case .sevenDays: return L("retention.seven_days") + case .oneMonth: return L("retention.one_month") + } + } + + var timeInterval: TimeInterval? { + switch self { + case .forever: return nil + case .threeDays: return 3 * 24 * 3600 + case .sevenDays: return 7 * 24 * 3600 + case .oneMonth: return 30 * 24 * 3600 + } + } +} + +enum MenuBarIcon: String, Codable, CaseIterable { + case mic = "mic" + case waveform = "waveform" + case bubble = "bubble" + + var symbolName: String { + switch self { + case .mic: return "mic.fill" + case .waveform: return "waveform" + case .bubble: return "bubble.left.fill" + } + } + + var label: String { + switch self { + case .mic: return L("icon.mic") + case .waveform: return L("icon.waveform") + case .bubble: return L("icon.bubble") + } + } +} + +enum AppIconAppearance: String, Codable, CaseIterable { + case system = "system" + case dark = "dark" + case light = "light" + + func resourceName(systemIsDark: Bool) -> String { + switch self { + case .system: return systemIsDark ? "AppIconDark" : "AppIconLight" + case .dark: return "AppIconDark" + case .light: return "AppIconLight" + } + } + + var label: String { + switch self { + case .system: return L("app_icon.system") + case .dark: return L("app_icon.dark") + case .light: return L("app_icon.light") + } + } +} + +enum InputLanguage: String, Codable, CaseIterable { + case auto = "Auto" + case chinese = "中文" + case english = "English" + case japanese = "日本語" + case korean = "한국어" + case cantonese = "粤语" + + var whisperCode: String? { + switch self { + case .auto: return nil + case .chinese: return "zh" + case .english: return "en" + case .japanese: return "ja" + case .korean: return "ko" + case .cantonese: return "yue" + } + } + + var localeIdentifier: String { + switch self { + case .auto: return Locale.current.identifier + case .chinese: return "zh-CN" + case .english: return "en-US" + case .japanese: return "ja-JP" + case .korean: return "ko-KR" + case .cantonese: return "zh-HK" + } + } +} diff --git a/Sources/Config/AppSettings.swift b/Sources/Config/AppSettings.swift index 2d31eca1..4e218b9a 100644 --- a/Sources/Config/AppSettings.swift +++ b/Sources/Config/AppSettings.swift @@ -2,247 +2,6 @@ import Foundation import Combine import Security -enum UILanguage: String, Codable, CaseIterable { - case chinese = "zh" - case english = "en" - - var displayName: String { - switch self { - case .chinese: return "中文" - case .english: return "English" - } - } -} - -enum OutputMode: String, Codable, CaseIterable { - case direct = "direct" - case processed = "processed" - case command = "command" - - var label: String { - switch self { - case .direct: return L("mode.verbatim") - case .processed: return L("mode.smart_format") - case .command: return L("mode.voice_command") - } - } -} - -enum SpeechEngineType: String, Codable, CaseIterable { - case whisper = "whisper" - case apple = "apple" - case volc = "volc" - case qwen3 = "qwen3" - case mimo = "mimo" - - static var selectableCases: [SpeechEngineType] { - [.qwen3, .whisper, .apple, .volc] - } - - var label: String { - switch self { - case .whisper: return "WhisperKit" - case .apple: return L("engine.apple_speech") - case .volc: return L("engine.volc_asr") - case .qwen3: return L("engine.qwen3_asr") - case .mimo: return L("engine.mimo_asr") - } - } -} - -enum LocalLLMBackend: String, Codable, CaseIterable { - case mlx - case espresso -} - -enum LanguageStyle: String, Codable, CaseIterable { - case casual = "casual" - case professional = "professional" - case custom = "custom" - - var label: String { - switch self { - case .casual: return L("style.casual") - case .professional: return L("style.professional") - case .custom: return L("style.custom") - } - } - - var defaultPrompt: String { - switch self { - case .casual: return L("style.prompt.casual") - case .professional: return L("style.prompt.professional") - case .custom: return L("style.prompt.custom") - } - } - - var icon: String { - switch self { - case .casual: return "bubble.left" - case .professional: return "list.number" - case .custom: return "slider.horizontal.3" - } - } - - var usesCustomPrompt: Bool { self == .custom } - - static func migrated(from savedValue: String) -> LanguageStyle { - if let style = LanguageStyle(rawValue: savedValue) { - return style - } - - let normalized = savedValue.lowercased() - if normalized.contains("casual") || savedValue.contains("口语") { - return .casual - } - if normalized.contains("custom") || savedValue.contains("自定义") { - return .custom - } - if normalized.contains("professional") - || normalized.contains("formal") - || normalized.contains("concise") - || savedValue.contains("专业") - || savedValue.contains("正式") - || savedValue.contains("简洁") { - return .professional - } - return .professional - } - - static func looksLikePresetPrompt(_ prompt: String) -> Bool { - let normalized = prompt.trimmingCharacters(in: .whitespacesAndNewlines) - let prompts = [ - L("style.prompt.casual"), - L("style.prompt.professional"), - L("style.prompt.concise"), - L("style.prompt.formal"), - ].map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } - - return prompts.contains(normalized) - } -} - -enum HotkeyType: String, Codable, CaseIterable { - case ctrl = "Ctrl" - case shift = "Shift" - case option = "Option" - case fn = "Fn" -} - -enum ActivationMode: String, Codable, CaseIterable { - case longPress = "longPress" - case doubleTap = "doubleTap" - case toggle = "toggle" - - var label: String { - switch self { - case .longPress: return L("mode.hold_record") - case .doubleTap: return L("mode.double_tap") - case .toggle: return L("mode.tap_toggle") - } - } -} - -enum HistoryRetention: String, Codable, CaseIterable { - case forever = "forever" - case threeDays = "threeDays" - case sevenDays = "sevenDays" - case oneMonth = "oneMonth" - - var label: String { - switch self { - case .forever: return L("retention.forever") - case .threeDays: return L("retention.three_days") - case .sevenDays: return L("retention.seven_days") - case .oneMonth: return L("retention.one_month") - } - } - - var timeInterval: TimeInterval? { - switch self { - case .forever: return nil - case .threeDays: return 3 * 24 * 3600 - case .sevenDays: return 7 * 24 * 3600 - case .oneMonth: return 30 * 24 * 3600 - } - } -} - -enum MenuBarIcon: String, Codable, CaseIterable { - case mic = "mic" - case waveform = "waveform" - case bubble = "bubble" - - var symbolName: String { - switch self { - case .mic: return "mic.fill" - case .waveform: return "waveform" - case .bubble: return "bubble.left.fill" - } - } - - var label: String { - switch self { - case .mic: return L("icon.mic") - case .waveform: return L("icon.waveform") - case .bubble: return L("icon.bubble") - } - } -} - -enum AppIconAppearance: String, Codable, CaseIterable { - case system = "system" - case dark = "dark" - case light = "light" - - func resourceName(systemIsDark: Bool) -> String { - switch self { - case .system: return systemIsDark ? "AppIconDark" : "AppIconLight" - case .dark: return "AppIconDark" - case .light: return "AppIconLight" - } - } - - var label: String { - switch self { - case .system: return L("app_icon.system") - case .dark: return L("app_icon.dark") - case .light: return L("app_icon.light") - } - } -} - -enum InputLanguage: String, Codable, CaseIterable { - case auto = "Auto" - case chinese = "中文" - case english = "English" - case japanese = "日本語" - case korean = "한국어" - case cantonese = "粤语" - - var whisperCode: String? { - switch self { - case .auto: return nil - case .chinese: return "zh" - case .english: return "en" - case .japanese: return "ja" - case .korean: return "ko" - case .cantonese: return "yue" - } - } - - var localeIdentifier: String { - switch self { - case .auto: return Locale.current.identifier - case .chinese: return "zh-CN" - case .english: return "en-US" - case .japanese: return "ja-JP" - case .korean: return "ko-KR" - case .cantonese: return "zh-HK" - } - } -} - final class AppSettings: ObservableObject { static let shared = AppSettings() static let defaultLLMModelID = "mlx-community/Qwen3.5-2B-4bit" @@ -291,6 +50,7 @@ final class AppSettings: ObservableObject { @Published var useRemoteLLM: Bool @Published var localLLMBackend: LocalLLMBackend @Published var espressoModelPath: String + @Published var fallbackToMLXOnEspressoFailure: Bool @Published var remoteProvider: RemoteProvider @Published var remoteAPIKey: String @Published var remoteBaseURL: String @@ -321,7 +81,7 @@ final class AppSettings: ObservableObject { case useScreenContext, screenContextMode, enableInstantInsert, hasCompletedOnboarding, uiLanguage, historyRetention case enableMemory, memoryWindowMinutes, enableCorrectionLearning, industryLexicon case useCustomSystemPrompt, customSystemPrompt - case useRemoteLLM, localLLMBackend, espressoModelPath + case useRemoteLLM, localLLMBackend, espressoModelPath, fallbackToMLXOnEspressoFailure case remoteProvider, remoteAPIKey, remoteBaseURL, remoteModel case menuBarIcon, appIconAppearance case volcAppKey, volcAccessKey, volcResourceId @@ -404,6 +164,9 @@ final class AppSettings: ObservableObject { rawValue: ud.string(forKey: Key.localLLMBackend.rawValue) ?? "" ) ?? .mlx espressoModelPath = ud.string(forKey: Key.espressoModelPath.rawValue) ?? "" + fallbackToMLXOnEspressoFailure = ud.object( + forKey: Key.fallbackToMLXOnEspressoFailure.rawValue + ) as? Bool ?? true remoteProvider = RemoteProvider(rawValue: ud.string(forKey: Key.remoteProvider.rawValue) ?? "") ?? .custom remoteAPIKey = ud.string(forKey: Key.remoteAPIKey.rawValue) ?? "" remoteBaseURL = ud.string(forKey: Key.remoteBaseURL.rawValue) ?? "" @@ -477,6 +240,9 @@ final class AppSettings: ObservableObject { $espressoModelPath.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.espressoModelPath.rawValue) }.store(in: &cancellables) + $fallbackToMLXOnEspressoFailure.dropFirst().sink { + [defaults] in defaults.set($0, forKey: Key.fallbackToMLXOnEspressoFailure.rawValue) + }.store(in: &cancellables) $remoteProvider.dropFirst().sink { [defaults] in defaults.set($0.rawValue, forKey: Key.remoteProvider.rawValue) }.store(in: &cancellables) $remoteAPIKey.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.remoteAPIKey.rawValue) }.store(in: &cancellables) $remoteBaseURL.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.remoteBaseURL.rawValue) }.store(in: &cancellables) diff --git a/Sources/Integration/InputSessionCoordinator+Output.swift b/Sources/Integration/InputSessionCoordinator+Output.swift index 24a427aa..dbc1a9ca 100644 --- a/Sources/Integration/InputSessionCoordinator+Output.swift +++ b/Sources/Integration/InputSessionCoordinator+Output.swift @@ -80,7 +80,7 @@ extension InputSessionCoordinator { guard !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { Log.info("[InputSessionCoordinator] refusing to complete session with empty output") - if let espressoOutcome, espressoOutcome == .unavailable { + if let espressoOutcome, espressoOutcome != .fallback { throw IntegrationError.operationFailedWithMessage(espressoOutcome.message) } throw IntegrationError.operationFailed diff --git a/Sources/Processing/EspressoGenerationOutcome.swift b/Sources/Processing/EspressoGenerationOutcome.swift index 057a6a2a..a26bd18c 100644 --- a/Sources/Processing/EspressoGenerationOutcome.swift +++ b/Sources/Processing/EspressoGenerationOutcome.swift @@ -2,12 +2,15 @@ import Foundation enum EspressoGenerationOutcome: Equatable, Sendable { case fallback + case failed case unavailable var message: String { switch self { case .fallback: return L("status.espresso_fell_back_to_mlx") + case .failed: + return L("error.espresso_runtime_failed") case .unavailable: return L("error.espresso_mlx_fallback_unavailable") } @@ -18,9 +21,11 @@ actor EspressoGenerationTracker { private var outcome: EspressoGenerationOutcome? func record(_ newOutcome: EspressoGenerationOutcome) { - if outcome != .unavailable { - outcome = newOutcome - } + outcome = newOutcome + } + + func clear() { + outcome = nil } func consume() -> EspressoGenerationOutcome? { @@ -38,6 +43,7 @@ enum EspressoFallbackPolicy { ) -> Bool { guard outcome == .fallback, !settings.useRemoteLLM, + settings.fallbackToMLXOnEspressoFailure, settings.localLLMBackend == .espresso, settings.espressoModelPath == expectedEspressoModelPath else { return false diff --git a/Sources/Processing/TextProcessingOptions.swift b/Sources/Processing/TextProcessingOptions.swift index d697c420..1de5d3f3 100644 --- a/Sources/Processing/TextProcessingOptions.swift +++ b/Sources/Processing/TextProcessingOptions.swift @@ -18,6 +18,7 @@ struct TextProcessingOptions { var useRemoteLLM: Bool var localLLMBackend: LocalLLMBackend var espressoModelPath: String + var fallbackToMLXOnEspressoFailure: Bool var remoteBaseURL: String var remoteAPIKey: String var remoteModel: String @@ -42,6 +43,7 @@ struct TextProcessingOptions { self.useRemoteLLM = settings.useRemoteLLM self.localLLMBackend = settings.localLLMBackend self.espressoModelPath = settings.espressoModelPath + self.fallbackToMLXOnEspressoFailure = settings.fallbackToMLXOnEspressoFailure self.remoteBaseURL = settings.remoteBaseURL self.remoteAPIKey = settings.remoteAPIKey self.remoteModel = settings.remoteModel diff --git a/Sources/Processing/TextProcessor+Generation.swift b/Sources/Processing/TextProcessor+Generation.swift index e85ee889..0dc36354 100644 --- a/Sources/Processing/TextProcessor+Generation.swift +++ b/Sources/Processing/TextProcessor+Generation.swift @@ -10,6 +10,7 @@ extension TextProcessor { temperature: Double ) async throws -> String { if options.useRemoteLLM { + await Self.clearEspressoOutcome() return try await remoteLLMClient.generate( prompt: prompt, systemPrompt: systemPrompt, @@ -26,6 +27,7 @@ extension TextProcessor { try Task.checkCancellation() switch options.localLLMBackend { case .mlx: + await Self.clearEspressoOutcome() await ensureModelLoaded(options.llmModel) return try await llm.generate( prompt: prompt, @@ -36,6 +38,7 @@ extension TextProcessor { case .espresso: do { let result = try await Self.runEspressoWithMLXFallback( + fallbackEnabled: options.fallbackToMLXOnEspressoFailure, espresso: { try await self.espressoLLM.loadModel(path: options.espressoModelPath) return try await self.espressoLLM.generate( @@ -59,27 +62,42 @@ extension TextProcessor { _ = await espressoLLM.consumeLastFailureMessage() await Self.recordEspressoOutcome(.fallback) Log.info("[TextProcessor] Espresso failed; used the selected MLX model") + } else { + await Self.clearEspressoOutcome() } return result.value + } catch is CancellationError { + throw CancellationError() } catch let error as EspressoMLXFallbackError { _ = await espressoLLM.consumeLastFailureMessage() await Self.recordEspressoOutcome(.unavailable) Log.sensitive("[TextProcessor] Espresso and MLX fallback failed: \(error.details)") Log.error("[TextProcessor] MLX fallback unavailable") throw error + } catch { + if !options.fallbackToMLXOnEspressoFailure { + _ = await espressoLLM.consumeLastFailureMessage() + await Self.recordEspressoOutcome(.failed) + Log.error("[TextProcessor] Espresso failed; MLX fallback is disabled") + } + throw error } } } } static func runEspressoWithMLXFallback( + fallbackEnabled: Bool = true, espresso: () async throws -> Value, mlx: () async throws -> Value ) async throws -> (value: Value, usedMLX: Bool) { do { - return (try await espresso(), false) + let value = try await espresso() + try Task.checkCancellation() + return (value, false) } catch { try Task.checkCancellation() + guard fallbackEnabled else { throw error } let espressoFailure = error.localizedDescription do { let value = try await mlx() @@ -118,6 +136,7 @@ extension TextProcessor { ) async throws -> String { try await withLocalModelAccess { try Task.checkCancellation() + await Self.clearEspressoOutcome() try await vlm.loadModel(id: model) return try await vlm.generate( prompt: prompt, diff --git a/Sources/Processing/TextProcessor+Models.swift b/Sources/Processing/TextProcessor+Models.swift index 235307af..c01ca11a 100644 --- a/Sources/Processing/TextProcessor+Models.swift +++ b/Sources/Processing/TextProcessor+Models.swift @@ -84,6 +84,10 @@ extension TextProcessor { await espressoGenerationTracker?.record(outcome) } + static func clearEspressoOutcome() async { + await espressoGenerationTracker?.clear() + } + func consumeEspressoOutcome() async -> EspressoGenerationOutcome? { await Self.espressoGenerationTracker?.consume() } @@ -146,7 +150,8 @@ extension TextProcessor { func warmUpLLM( model: String, backend: LocalLLMBackend, - espressoModelPath: String + espressoModelPath: String, + fallbackToMLXOnEspressoFailure: Bool ) async -> (loaded: Bool, errorMessage: String?, espressoOutcome: EspressoGenerationOutcome?) { do { return try await withLocalModelAccess { @@ -156,6 +161,7 @@ extension TextProcessor { try await llm.loadModel(id: model) case .espresso: let result = try await Self.runEspressoWithMLXFallback( + fallbackEnabled: fallbackToMLXOnEspressoFailure, espresso: { try await self.espressoLLM.loadModel(path: espressoModelPath) }, mlx: { try await self.llm.loadModel(id: model) } ) @@ -165,6 +171,8 @@ extension TextProcessor { } } return (true, nil, nil) + } catch is CancellationError { + throw CancellationError() } catch let error as EspressoMLXFallbackError { Log.sensitive("[TextProcessor] Espresso and MLX warmup failed: \(error.details)") Log.error("[TextProcessor] MLX fallback unavailable during warmup") @@ -174,10 +182,15 @@ extension TextProcessor { Log.error("[TextProcessor] LLM warmup failed: \(error.localizedDescription)") if backend == .espresso { _ = await espressoLLM.consumeLastFailureMessage() + if !fallbackToMLXOnEspressoFailure { + return (false, EspressoGenerationOutcome.failed.message, .failed) + } } return (false, error.localizedDescription, nil) } } + } catch is CancellationError { + return (false, nil, nil) } catch { return (false, error.localizedDescription, nil) } diff --git a/Sources/Resources/en.lproj/Localizable.strings b/Sources/Resources/en.lproj/Localizable.strings index 51dfcc20..481c750b 100644 --- a/Sources/Resources/en.lproj/Localizable.strings +++ b/Sources/Resources/en.lproj/Localizable.strings @@ -206,7 +206,9 @@ "model.espresso.description" = "Run a compatible .esp bundle with Espresso's Apple Neural Engine backend."; "model.espresso.no_bundle" = "No .esp bundle selected"; "model.espresso.choose" = "Choose .esp Bundle…"; -"model.espresso.private_api_warning" = "Espresso uses private Apple Neural Engine APIs and may break after macOS updates. If it fails, Utter switches to your installed MLX model. Espresso is not eligible for Mac App Store distribution."; +"model.espresso.auto_fallback" = "Automatically switch to MLX if Espresso fails"; +"model.espresso.auto_fallback_help" = "Finish the request with the selected installed MLX model and make MLX active. Turn this off to keep Espresso selected and show its error."; +"model.espresso.private_api_warning" = "Espresso uses private Apple Neural Engine APIs. Compatibility depends on the Mac and macOS version and is not guaranteed. Espresso is not eligible for Mac App Store distribution."; "model.custom_id_placeholder" = "Custom model ID (e.g. mlx-community/…)"; "model.active" = "Active"; "model.use" = "Use"; diff --git a/Sources/Resources/zh-Hans.lproj/Localizable.strings b/Sources/Resources/zh-Hans.lproj/Localizable.strings index 1e978a7e..3058dbcc 100644 --- a/Sources/Resources/zh-Hans.lproj/Localizable.strings +++ b/Sources/Resources/zh-Hans.lproj/Localizable.strings @@ -206,7 +206,9 @@ "model.espresso.description" = "使用 Espresso 的 Apple 神经网络引擎后端运行兼容的 .esp 模型包。"; "model.espresso.no_bundle" = "尚未选择 .esp 模型包"; "model.espresso.choose" = "选择 .esp 模型包…"; -"model.espresso.private_api_warning" = "Espresso 使用 Apple 神经网络引擎私有 API,可能在 macOS 更新后失效。失败时,Utter 会切换到已安装的 MLX 模型。Espresso 也无法通过 Mac App Store 审核。"; +"model.espresso.auto_fallback" = "Espresso 失败时自动切换到 MLX"; +"model.espresso.auto_fallback_help" = "使用已安装且选中的 MLX 模型完成本次请求,并将 MLX 设为当前后端。关闭后会保留 Espresso 并显示错误。"; +"model.espresso.private_api_warning" = "Espresso 使用 Apple 神经网络引擎私有 API,兼容性取决于 Mac 机型和 macOS 版本,无法保证。Espresso 也无法通过 Mac App Store 审核。"; "model.custom_id_placeholder" = "自定义模型 ID(如 mlx-community/…)"; "model.active" = "当前"; "model.use" = "启用"; diff --git a/Sources/UI/ModelManagementSections.swift b/Sources/UI/ModelManagementSections.swift index 0f0fed53..684e36fa 100644 --- a/Sources/UI/ModelManagementSections.swift +++ b/Sources/UI/ModelManagementSections.swift @@ -221,6 +221,12 @@ extension ModelManagementView { .controlSize(.small) } + Toggle( + L("model.espresso.auto_fallback"), + isOn: $settings.fallbackToMLXOnEspressoFailure + ) + .help(L("model.espresso.auto_fallback_help")) + Label(L("model.espresso.private_api_warning"), systemImage: "exclamationmark.triangle.fill") .font(.system(size: 10)) .foregroundStyle(.orange) diff --git a/Tests/OpenTypeTests/EspressoFallbackTests.swift b/Tests/OpenTypeTests/EspressoFallbackTests.swift index 6d9cb7d5..915be6d9 100644 --- a/Tests/OpenTypeTests/EspressoFallbackTests.swift +++ b/Tests/OpenTypeTests/EspressoFallbackTests.swift @@ -3,8 +3,20 @@ import MLX @testable import OpenType final class EspressoFallbackTests: XCTestCase { - private final class WeakTrackerBox { - weak var value: EspressoGenerationTracker? + private actor AsyncGate { + private var isOpen = false + private var continuation: CheckedContinuation? + + func wait() async { + guard !isOpen else { return } + await withCheckedContinuation { continuation = $0 } + } + + func open() { + isOpen = true + continuation?.resume() + continuation = nil + } } private enum StubError: LocalizedError { @@ -44,14 +56,35 @@ final class EspressoFallbackTests: XCTestCase { XCTAssertTrue(result.usedMLX) } - func testCancellationDoesNotStartMLXFallback() async { + func testDisabledFallbackDoesNotRunMLX() async { + var ranMLX = false + + do { + _ = try await TextProcessor.runEspressoWithMLXFallback( + fallbackEnabled: false, + espresso: { throw StubError.espresso }, + mlx: { + ranMLX = true + return "mlx output" + } + ) as (value: String, usedMLX: Bool) + XCTFail("Expected the Espresso failure") + } catch { + XCTAssertEqual(error.localizedDescription, "espresso failed") + XCTAssertFalse(ranMLX) + } + } + + func testDisabledFallbackCancellationDoesNotStartMLX() async { let espressoStarted = expectation(description: "Espresso started") + let gate = AsyncGate() var ranMLX = false let task = Task { try await TextProcessor.runEspressoWithMLXFallback( + fallbackEnabled: false, espresso: { espressoStarted.fulfill() - try await Task.sleep(for: .seconds(10)) + await gate.wait() return "espresso output" }, mlx: { @@ -63,6 +96,7 @@ final class EspressoFallbackTests: XCTestCase { await fulfillment(of: [espressoStarted], timeout: 1) task.cancel() + await gate.open() do { _ = try await task.value @@ -89,34 +123,6 @@ final class EspressoFallbackTests: XCTestCase { } } - func testOutcomeTrackingIsRequestScoped() async { - let processor = TextProcessor() - var first: EspressoGenerationOutcome? - var second: EspressoGenerationOutcome? - - await TextProcessor.withEspressoOutcomeTracking { - await TextProcessor.recordEspressoOutcome(.fallback) - first = await processor.consumeEspressoOutcome() - } - await TextProcessor.withEspressoOutcomeTracking { - second = await processor.consumeEspressoOutcome() - } - - XCTAssertEqual(first, .fallback) - XCTAssertNil(second) - } - - func testOutcomeTrackerIsReleasedAfterRequestCompletes() async { - let box = WeakTrackerBox() - - await TextProcessor.withEspressoOutcomeTracking { - box.value = TextProcessor.espressoGenerationTracker - XCTAssertNotNil(box.value) - } - - XCTAssertNil(box.value) - } - func testFallbackOutcomeSwitchesPersistedBackendToMLX() { let suiteName = "EspressoFallbackTests.\(UUID().uuidString)" let defaults = UserDefaults(suiteName: suiteName)! @@ -149,10 +155,30 @@ final class EspressoFallbackTests: XCTestCase { XCTAssertEqual(settings.localLLMBackend, .espresso) } + func testDisabledFallbackPreservesEspressoSelection() { + let suiteName = "EspressoFallbackTests.\(UUID().uuidString)" + let defaults = UserDefaults(suiteName: suiteName)! + defer { defaults.removePersistentDomain(forName: suiteName) } + let settings = AppSettings(defaults: defaults) + XCTAssertTrue(settings.fallbackToMLXOnEspressoFailure) + settings.localLLMBackend = .espresso + settings.fallbackToMLXOnEspressoFailure = false + let reloaded = AppSettings(defaults: defaults) + XCTAssertFalse(reloaded.fallbackToMLXOnEspressoFailure) + + XCTAssertFalse(EspressoFallbackPolicy.selectMLXIfNeeded( + after: .fallback, + settings: reloaded, + expectedEspressoModelPath: reloaded.espressoModelPath + )) + XCTAssertEqual(reloaded.localLLMBackend, .espresso) + } + func testLocalizedFallbackMessagesMentionMLX() { for language in [UILanguage.english, .chinese] { XCTAssertTrue(Loc.string("status.espresso_fell_back_to_mlx", language: language).contains("MLX")) XCTAssertTrue(Loc.string("error.espresso_mlx_fallback_unavailable", language: language).contains("MLX")) + XCTAssertFalse(Loc.string("model.espresso.auto_fallback", language: language).isEmpty) } } diff --git a/Tests/OpenTypeTests/EspressoOutcomeTests.swift b/Tests/OpenTypeTests/EspressoOutcomeTests.swift new file mode 100644 index 00000000..84356784 --- /dev/null +++ b/Tests/OpenTypeTests/EspressoOutcomeTests.swift @@ -0,0 +1,73 @@ +import XCTest +@testable import OpenType + +final class EspressoOutcomeTests: XCTestCase { + private final class WeakTrackerBox { + weak var value: EspressoGenerationTracker? + } + + func testOutcomeTrackingUsesLatestOperationAndIsRequestScoped() async { + let processor = TextProcessor() + var outcomes: [EspressoGenerationOutcome?] = [] + + await TextProcessor.withEspressoOutcomeTracking { + await TextProcessor.recordEspressoOutcome(.fallback) + await TextProcessor.recordEspressoOutcome(.failed) + outcomes.append(await processor.consumeEspressoOutcome()) + } + await TextProcessor.withEspressoOutcomeTracking { + await TextProcessor.recordEspressoOutcome(.failed) + await TextProcessor.recordEspressoOutcome(.fallback) + outcomes.append(await processor.consumeEspressoOutcome()) + } + await TextProcessor.withEspressoOutcomeTracking { + outcomes.append(await processor.consumeEspressoOutcome()) + } + for staleOutcome in [EspressoGenerationOutcome.fallback, .failed] { + await TextProcessor.withEspressoOutcomeTracking { + await TextProcessor.recordEspressoOutcome(staleOutcome) + await TextProcessor.clearEspressoOutcome() + outcomes.append(await processor.consumeEspressoOutcome()) + } + } + + XCTAssertEqual(outcomes[0], .failed) + XCTAssertEqual(outcomes[1], .fallback) + XCTAssertNil(outcomes[2]) + XCTAssertNil(outcomes[3]) + XCTAssertNil(outcomes[4]) + } + + func testRemoteAttemptClearsEarlierEspressoOutcome() async { + let processor = TextProcessor() + var options = TextProcessingOptions(settings: AppSettings.shared) + options.useRemoteLLM = true + options.remoteAPIKey = "" + var outcome: EspressoGenerationOutcome? + + await TextProcessor.withEspressoOutcomeTracking { + await TextProcessor.recordEspressoOutcome(.fallback) + _ = try? await processor.generateText( + prompt: "test", + systemPrompt: "test", + options: options, + maxTokens: 1, + temperature: 0 + ) + outcome = await processor.consumeEspressoOutcome() + } + + XCTAssertNil(outcome) + } + + func testOutcomeTrackerIsReleasedAfterRequestCompletes() async { + let box = WeakTrackerBox() + + await TextProcessor.withEspressoOutcomeTracking { + box.value = TextProcessor.espressoGenerationTracker + XCTAssertNotNil(box.value) + } + + XCTAssertNil(box.value) + } +} diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/intent.md b/docs/sdlc/changes/2026-08-29-espresso-ane/intent.md index 7c397f94..c8f7091a 100644 --- a/docs/sdlc/changes/2026-08-29-espresso-ane/intent.md +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/intent.md @@ -8,10 +8,10 @@ Apple Neural Engine runtime for local post-processing. ## Outcome The model settings let users choose Espresso, select an `.esp` model bundle, -and route local warmup and text generation through Espresso's ANE runtime. If -that private runtime fails and the selected MLX model is installed, Utter -completes the request with MLX, switches the persisted backend to MLX, and -shows which backend produced the result. +and route local warmup and text generation through Espresso's ANE runtime. +Users choose whether an Espresso failure should finish with the selected, +installed MLX model or stop with an Espresso error. Automatic fallback remains +the default and identifies which backend produced the result. ## Scope @@ -32,8 +32,12 @@ Store compatibility for Espresso's private ANE API. - MLX and Espresso are selectable local LLM backends and the choice persists. - A valid `.esp` directory can be selected while malformed bundles are rejected. - Espresso selection routes warmup and generation through the Espresso engine. -- An Espresso runtime failure falls back to an installed selected MLX model, - persists MLX as the active backend, and surfaces the fallback to the user. +- Users can disable automatic MLX fallback while keeping Espresso selected. +- When fallback is enabled, an Espresso runtime failure falls back to an + installed selected MLX model, persists MLX as the active backend, and + surfaces the fallback to the user. +- When fallback is disabled, an Espresso runtime failure does not run MLX and + surfaces an actionable Espresso error. - If the selected MLX model is unavailable, the Espresso failure remains visible with guidance to install an MLX model. - Repeated fallback requests do not retain request-scoped state, and explicitly @@ -45,6 +49,8 @@ Store compatibility for Espresso's private ANE API. ## Open questions -Which macOS 27 and M5 combinations Espresso will support remains an upstream -compatibility question. Direct Espresso inference on the available M5 -Max/macOS 27 host is currently blocked by the ANE compiler. +Upstream documents M1 through M4 as tested, but Utter has not independently +verified that matrix. M5 is absent from that matrix, and direct Espresso +inference on the available M5 Max/macOS 27 host is blocked by the ANE compiler. +Because the backend uses private APIs, the failing variable may be the SoC, the +OS, or their combination; current evidence does not prove an M5-only failure. diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md b/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md index fd676d44..2d226ebd 100644 --- a/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md @@ -18,6 +18,10 @@ - [x] Reproduce retained MLX memory after unload and clear the MLX cache on explicit unload. - [x] Share one processor across voice and integration workflows and include benchmarking in its lifecycle. - [x] Serialize complete local-model transactions and remove cancelled waiters from the lifecycle queue. +- [x] Let users disable automatic MLX fallback while preserving it as the default. +- [x] Keep Espresso selected and surface its runtime error when fallback is disabled. +- [x] Clarify in-product compatibility copy without claiming an M5-only failure. +- [x] Resolve independent-review findings for latest-operation outcomes, cancellation, and file size. ## Verification plan @@ -32,6 +36,9 @@ - [x] Current full repository gates and release-style app build - [x] Repeated real fallback requests and explicit-unload memory regression - [x] Independent cancellation, unload-order, shared-ownership, and file-size review +- [x] Focused disabled-fallback and persistence tests +- [x] Real settings-window light and dark verification for the fallback control +- [x] Current repository gates and release-style app build after the follow-up ## Human gates diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/spec.md b/docs/sdlc/changes/2026-08-29-espresso-ane/spec.md index 2549ba9f..4f5491ea 100644 --- a/docs/sdlc/changes/2026-08-29-espresso-ane/spec.md +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/spec.md @@ -10,9 +10,11 @@ models and tokenizers as `.esp` directories consumed by `ESPRuntime` and ## Design -Add a persisted `LocalLLMBackend` choice with MLX as the default and a persisted -Espresso bundle path. The models UI validates a selected directory with -`ESPRuntimeBundle.open`, displays the private-API warning, and requests warmup. +Add a persisted `LocalLLMBackend` choice with MLX as the default, a persisted +Espresso bundle path, and a persisted automatic-MLX-fallback toggle that +defaults on. The models UI validates a selected directory with +`ESPRuntimeBundle.open`, displays the private-API warning and fallback toggle, +and requests warmup. `TextProcessor` owns one `EspressoLLMEngine` actor alongside the MLX engine and dispatches warmup, readiness, unloading, and generation according to the @@ -22,14 +24,15 @@ Qwen chat template when appropriate, and returns generated text through the existing processing pipeline. Remote LLM behavior is unchanged. For local Espresso warmup and generation, `TextProcessor` first attempts the -selected `.esp` bundle. If that attempt fails, it tries the already-selected -MLX model through the existing `LLMEngine`; `LLMEngine` continues to require a -complete local model and never downloads during fallback. A successful MLX -fallback records a request-scoped semantic outcome. The main app consumes that -outcome, changes the persisted backend to MLX only if Espresso is still -selected, and shows the notice in the completion state. Integration sessions -also persist the backend change and log it without changing their response -schema. +selected `.esp` bundle. Captured processing options decide whether a failed +attempt may try the already-selected MLX model through the existing +`LLMEngine`; `LLMEngine` continues to require a complete local model and never +downloads during fallback. A successful MLX fallback records a request-scoped +semantic outcome. The main app consumes that outcome, changes the persisted +backend to MLX only if Espresso and automatic fallback are still selected, and +shows the notice in the completion state. With fallback disabled, MLX is not +called, Espresso remains selected, and the existing error surfaces identify the +runtime failure. Integration response schemas remain unchanged. The voice pipeline, integration sessions, and model benchmark UI share one application-owned `TextProcessor`. A cancellable FIFO gate serializes complete @@ -48,10 +51,12 @@ runtime logic or public API and avoids taking unrelated upstream `main` changes. The model and prompt remain on-device. Espresso depends on private ANE APIs, so OS or hardware changes can reject generated ANE programs even when bundle -metadata is valid. Such failures first use the selected, installed MLX model; -if MLX is not available or also fails, the combined failure propagates through -the existing model-load or generation error path. Selecting a missing or -malformed bundle does not replace the current setting. +metadata is valid. With automatic fallback enabled, such failures first use the +selected, installed MLX model; if MLX is not available or also fails, the +combined failure propagates through the existing model-load or generation error +path. With fallback disabled, no MLX work starts and the Espresso error follows +that path directly. Selecting a missing or malformed bundle does not replace +the current setting. The UI warning explicitly states the private-API and App Store limitation. No fallback is silent: successful fallback produces a localized status message @@ -69,13 +74,14 @@ experimental backend. ## Test strategy -Persistence, prompt formatting, request-scoped Espresso-to-MLX fallback +Persistence, prompt formatting, enabled and disabled Espresso-to-MLX fallback ordering, failure behavior, cancellable local-model serialization, shared processor wiring, and unload memory behavior have focused tests. The complete Swift suite and repository checks cover existing paths and package integration. -A release-style app build checks dependency and Metal/resource packaging. Real -inference is exercised with a prepared GPT-2 `.esp` bundle and recorded even if -the host's private ANE compiler rejects it. +A release-style app build checks dependency and Metal/resource packaging. The +fallback toggle is checked in the real settings window in both appearances. +Real inference is exercised with a prepared GPT-2 `.esp` bundle and recorded +even if the host's private ANE compiler rejects it. ## Rollout and rollback diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/state.json b/docs/sdlc/changes/2026-08-29-espresso-ane/state.json index 5257db0f..a8a8282b 100644 --- a/docs/sdlc/changes/2026-08-29-espresso-ane/state.json +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/state.json @@ -11,7 +11,9 @@ "Users can select MLX or Espresso for local LLM processing and the selection persists.", "Users can select a valid Espresso bundle and invalid bundles are rejected before loading.", "When Espresso is selected, local text generation and model warmup use its ANE runtime.", - "When Espresso fails, an installed selected MLX model completes the request, the persisted backend changes to MLX, and the user sees the fallback.", + "Users can choose whether an Espresso failure may use MLX, and that choice persists.", + "When automatic fallback is enabled and Espresso fails, an installed selected MLX model completes the request, the persisted backend changes to MLX, and the user sees the fallback.", + "When automatic fallback is disabled and Espresso fails, MLX does not run, Espresso remains selected, and the user sees the Espresso error.", "When both Espresso and MLX are unavailable, the user sees actionable local-model guidance.", "Fallback outcome tracking is request-scoped and explicit local-model unload waits for active local work, releases every production model container, and clears the MLX memory cache.", "The existing MLX and remote LLM paths continue to pass the automated test suite.", diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md b/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md index 777c0476..32ed13fa 100644 --- a/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md @@ -5,14 +5,17 @@ | Check | Result | Evidence | |---|---|---| | `bash scripts/ci-basic-checks.sh` | Pass | Current SDLC, harness, plist, localization, resource, identifier, secret-file, and symlink checks passed | -| `DEVELOPER_DIR=/Applications/Xcode-beta.app/Contents/Developer swift test` | Pass | 586 XCTest tests passed, 9 skipped, plus 1 Swift Testing test passed after merging the latest settings UI and retaining request-scoped fallback and unload-memory changes | +| `DEVELOPER_DIR=/Applications/Xcode-beta.app/Contents/Developer swift test` | Pass | 589 XCTest tests passed, 9 skipped, plus 1 Swift Testing test passed with user-selectable fallback behavior | +| Focused fallback, outcome, and settings tests | Pass | 51 tests passed with 1 hardware integration skip; disabling fallback preserves the Espresso error, never invokes MLX, and persists across `AppSettings` reloads; the last generation operation determines the request outcome, non-Espresso generation clears stale outcomes, and cancellation remains neutral | +| Fallback settings real-window matrix | Pass | The 760 by 680 Release settings view rendered Chinese and English in forced Aqua and Dark Aqua; the six model segments, fallback switch, bundle action, and one-line Chinese or two-line English private-API warning remain visible without clipping | +| Final no-QA-hook Release build | Pass | After removing the temporary screenshot entry and its captures, the app and CLI rebuilt, were ad-hoc signed with hardened runtime, and passed release artifact verification | | Focused fallback and localization tests | Pass | Espresso success avoids MLX; Espresso failure uses MLX; cancellation skips fallback; dual failure preserves both diagnostics; request trackers deallocate and do not cross requests; successful fallback persists MLX; localized status and typed 288 by 56 two-line overlay layout pass | | `DEVELOPER_DIR=/Applications/Xcode-beta.app/Contents/Developer bash scripts/build-app.sh --app-only --sign=-` | Pass | Final source without the temporary visual-QA trigger built the Release app and CLI, assembled, ad-hoc signed, and passed artifact verification | | GitHub run `33296463655`, Xcode 26.6 Release build | Fail; diagnosed | Swift 6.2 emitted cross-module references to three internal `RealModelInferenceEngine.Compiled*` metadata symbols, then failed final arm64 linking | | Pinned Espresso commit `f3603c7` symbol probe | Pass | The three metadata symbols are emitted as external after changing only the holder types from internal to package visibility | | GitHub run `33297701825`, Xcode 26.6 | Pass | Contract & Tests, Release-style App Build, and SDLC Gate all passed; the app build completed in 9m12s | | GitHub run `33304348927`, Xcode 26.6 unit tests | Fail; diagnosed | The empty lifecycle-gate test called `Memory.clearCache()` before any MLX container existed, which made the test runner initialize MLX without an app-bundled default metallib; cache clearing is now conditional on a loaded MLX LLM, benchmark, or VLM container | -| Independent high-risk reviews | Completed; final passes clean | Successive reviews found stale state, cancellation and selection races, unordered unload/reload, independent integration and benchmark containers, multimodal fallback interleaving, and retained cancelled waiters. The final implementation uses request-scoped outcomes, selection identity checks, one shared processor, and a cancellable reentrant lifecycle gate | +| Independent high-risk reviews | Completed; final code review clean | Successive reviews found stale state, cancellation and selection races, unordered unload/reload, independent integration and benchmark containers, multimodal fallback interleaving, retained cancelled waiters, stale first-operation outcomes, and cancellation misreporting. The final implementation uses latest-operation request-scoped outcomes, neutral cancellation, selection identity checks, one shared processor, and a cancellable reentrant lifecycle gate | | Espresso 0.9.0 GPT-2 generation | Fail | M5 Max/macOS 27 ANE compiler returned code 10, `verifyBundleAtPath: invalid model`, while compiling layer 0 attention | | Espresso main `eafb33d` GPT-2 generation | Fail | Latest upstream source produced the same ANE code 10 on the same host | | M5 ANE compile matrix | Fail as expected | All 24 combinations failed: iOS 18, iOS 19, macOS 26, and macOS 27 MIL targets; LayerNorm and RMSNorm; spatial sizes 64, 128, and 256 | @@ -27,16 +30,18 @@ - Backend selection and persistence — pass; focused settings test and full suite. - Bundle selection and malformed-bundle rejection — pass at metadata validation level through `ESPRuntimeBundle.open`. - Espresso warmup and generation dispatch — pass by code path and build coverage. -- Automatic MLX recovery — pass in focused control-flow tests and a real M5 - Espresso-failure-to-MLX-generation integration test. +- User-controlled MLX recovery — pass; automatic fallback defaults on and still + passes the real M5 integration test, while the disabled setting neither starts + MLX nor changes the selected Espresso backend. - Persisted backend correction — pass with isolated `UserDefaults`; MLX replaces Espresso only when Espresso remains the selected backend. - User feedback — pass in both localizations and real Release windows without treating the successful recovery as an error. - Request and memory lifecycle — pass; outcome trackers deallocate at task end, - repeated requests remain flat, every production model engine is owned by the - shared processor, and explicit unload clears MLX active/cache memory after - active local work completes. + the last local operation determines the request result, cancellation does not + manufacture an Espresso failure, repeated requests remain flat, every + production model engine is owned by the shared processor, and explicit unload + clears MLX active/cache memory after active local work completes. - Existing MLX and remote behavior — pass; complete suite has no failures. - Direct Espresso generation on a supported host — blocked on the available M5 Max/macOS 27 host; both the pinned release and upstream main fail in Apple's @@ -46,7 +51,9 @@ ## Residual risk Espresso still relies on a private ANE interface whose generated programs are -rejected on the available M5 Max/macOS 27 environment. The fallback requires an +rejected on the available M5 Max/macOS 27 environment. Upstream documents M1 +through M4 as tested, but Utter has not independently verified those devices and +private-API behavior may also change with macOS. The fallback requires an already-installed selected MLX model and deliberately does not start a download. This change makes Utter resilient on M5; it does not establish direct Espresso or private-ANE compatibility on M5. A public Core ML backend remains separate @@ -54,7 +61,7 @@ future work. ## Decision -Automatic recovery, persistence, regression checks, real M5 fallback, localized -window behavior, dependency resolution, and the Release link are verified. +User-controlled recovery, persistence, regression checks, real M5 fallback, +localized window behavior, dependency resolution, and the Release link are verified. Direct Espresso inference remains blocked on the tested host. Do not describe the private ANE backend itself as runtime-compatible with M5 Max/macOS 27. From d8307eb7e31da95f88f1b674f304bc89f0de00a0 Mon Sep 17 00:00:00 2001 From: idevlab Date: Mon, 31 Aug 2026 05:27:58 +0800 Subject: [PATCH 10/14] feat: replace Espresso with hardened ANE-LM runtime --- Package.resolved | 16 +- Package.swift | 7 +- Sources/App/VoicePipeline+EditCommands.swift | 2 +- .../InputSessionCoordinator+Output.swift | 2 +- Sources/LLM/ANELMTokenizerValidation.swift | 130 ++++++++++ Sources/LLM/EspressoLLMEngine.swift | 224 +++++++++++++---- .../Processing/TextProcessor+Generation.swift | 8 +- Sources/Processing/TextProcessor+Models.swift | 2 +- .../Resources/en.lproj/Localizable.strings | 26 +- .../zh-Hans.lproj/Localizable.strings | 26 +- Sources/UI/ModelManagementSections.swift | 4 +- Sources/UI/ModelManagementView.swift | 25 +- Sources/UI/SettingsView.swift | 2 +- Sources/UI/SettingsVoiceIllustration.swift | 2 +- .../ANELMRuntimeTestFixtures.swift | 160 ++++++++++++ Tests/OpenTypeTests/ANELMRuntimeTests.swift | 228 ++++++++++++++++++ Tests/OpenTypeTests/ConfigurationTests.swift | 4 +- .../OpenTypeTests/EspressoFallbackTests.swift | 16 +- ...31-apple-neural-engine-m5-compatibility.md | 208 ++++++++++++++++ .../2026-08-31-ane-lm-runtime/intent.md | 58 +++++ .../changes/2026-08-31-ane-lm-runtime/plan.md | 35 +++ .../changes/2026-08-31-ane-lm-runtime/spec.md | 75 ++++++ .../2026-08-31-ane-lm-runtime/state.json | 32 +++ .../2026-08-31-ane-lm-runtime/verification.md | 58 +++++ .../intent.md | 35 +++ .../plan.md | 22 ++ .../spec.md | 33 +++ .../state.json | 25 ++ .../verification.md | 32 +++ 29 files changed, 1387 insertions(+), 110 deletions(-) create mode 100644 Sources/LLM/ANELMTokenizerValidation.swift create mode 100644 Tests/OpenTypeTests/ANELMRuntimeTestFixtures.swift create mode 100644 Tests/OpenTypeTests/ANELMRuntimeTests.swift create mode 100644 docs/research/2026-08-31-apple-neural-engine-m5-compatibility.md create mode 100644 docs/sdlc/changes/2026-08-31-ane-lm-runtime/intent.md create mode 100644 docs/sdlc/changes/2026-08-31-ane-lm-runtime/plan.md create mode 100644 docs/sdlc/changes/2026-08-31-ane-lm-runtime/spec.md create mode 100644 docs/sdlc/changes/2026-08-31-ane-lm-runtime/state.json create mode 100644 docs/sdlc/changes/2026-08-31-ane-lm-runtime/verification.md create mode 100644 docs/sdlc/changes/2026-08-31-settings-semantic-background/intent.md create mode 100644 docs/sdlc/changes/2026-08-31-settings-semantic-background/plan.md create mode 100644 docs/sdlc/changes/2026-08-31-settings-semantic-background/spec.md create mode 100644 docs/sdlc/changes/2026-08-31-settings-semantic-background/state.json create mode 100644 docs/sdlc/changes/2026-08-31-settings-semantic-background/verification.md diff --git a/Package.resolved b/Package.resolved index df2b5d2d..d98e88c3 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,21 +1,21 @@ { - "originHash" : "113df5803768310e85daa76109055f7d79bff777eab44985d49c77f715d46341", + "originHash" : "3283c962479741ac4c7b0cd4e71ec15275b7d24d82a2182437d9e2ba4d21b27a", "pins" : [ { - "identity" : "argmax-oss-swift", + "identity" : "ane-lm", "kind" : "remoteSourceControl", - "location" : "https://github.com/argmaxinc/argmax-oss-swift.git", + "location" : "https://github.com/IchenDEV/ANE-LM.git", "state" : { - "revision" : "25c62997041c134b03ca82731ce2f6fd2cae1eb9", - "version" : "1.0.0" + "revision" : "033472ec12ea796fc7ea4f8cefd7ed456f69900b" } }, { - "identity" : "espresso", + "identity" : "argmax-oss-swift", "kind" : "remoteSourceControl", - "location" : "https://github.com/IchenDEV/Espresso.git", + "location" : "https://github.com/argmaxinc/argmax-oss-swift.git", "state" : { - "revision" : "f3603c7014b3b82c9df036c2e91185e1e32b2d81" + "revision" : "25c62997041c134b03ca82731ce2f6fd2cae1eb9", + "version" : "1.0.0" } }, { diff --git a/Package.swift b/Package.swift index d5111763..550d0ee3 100644 --- a/Package.swift +++ b/Package.swift @@ -14,8 +14,8 @@ let package = Package( dependencies: [ .package(url: "https://github.com/argmaxinc/argmax-oss-swift.git", from: "1.0.0"), .package( - url: "https://github.com/IchenDEV/Espresso.git", - revision: "f3603c7014b3b82c9df036c2e91185e1e32b2d81" + url: "https://github.com/IchenDEV/ANE-LM.git", + revision: "033472ec12ea796fc7ea4f8cefd7ed456f69900b" ), .package(url: "https://github.com/Blaizzy/mlx-audio-swift.git", exact: "0.1.3"), .package(url: "https://github.com/huggingface/swift-transformers", from: "1.3.3"), @@ -26,8 +26,7 @@ let package = Package( name: "OpenType", dependencies: [ .product(name: "WhisperKit", package: "argmax-oss-swift"), - .product(name: "ESPRuntime", package: "Espresso"), - .product(name: "RealModelInference", package: "Espresso"), + .product(name: "ANELMRuntime", package: "ANE-LM"), .product(name: "MLXAudioCore", package: "mlx-audio-swift"), .product(name: "MLXAudioSTT", package: "mlx-audio-swift"), .product(name: "Hub", package: "swift-transformers"), diff --git a/Sources/App/VoicePipeline+EditCommands.swift b/Sources/App/VoicePipeline+EditCommands.swift index 24ed2442..b288aa9d 100644 --- a/Sources/App/VoicePipeline+EditCommands.swift +++ b/Sources/App/VoicePipeline+EditCommands.swift @@ -58,7 +58,7 @@ extension VoicePipeline { expectedEspressoModelPath: expectedEspressoModelPath ) { if case .error = appState.phase, espressoOutcome == .fallback { - Log.info("[VoicePipeline] preserving edit-command error after Espresso fallback") + Log.info("[VoicePipeline] preserving edit-command error after ANE-LM fallback") } else if espressoOutcome != .fallback { showErrorHint(espressoOutcome.message) } else { diff --git a/Sources/Integration/InputSessionCoordinator+Output.swift b/Sources/Integration/InputSessionCoordinator+Output.swift index dbc1a9ca..bb183543 100644 --- a/Sources/Integration/InputSessionCoordinator+Output.swift +++ b/Sources/Integration/InputSessionCoordinator+Output.swift @@ -75,7 +75,7 @@ extension InputSessionCoordinator { settings: settings, expectedEspressoModelPath: options.espressoModelPath ) { - Log.info("[InputSessionCoordinator] Espresso failed; selected MLX as the active backend") + Log.info("[InputSessionCoordinator] ANE-LM failed; selected MLX as the active backend") } guard !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { diff --git a/Sources/LLM/ANELMTokenizerValidation.swift b/Sources/LLM/ANELMTokenizerValidation.swift new file mode 100644 index 00000000..55de7489 --- /dev/null +++ b/Sources/LLM/ANELMTokenizerValidation.swift @@ -0,0 +1,130 @@ +import Foundation + +enum ANELMTokenizerValidation { + static func samplerVocabularySize( + tokenizer: [String: Any], + tokenizerConfig: [String: Any], + modelVocabularySize: Int + ) -> Int? { + guard supportedComponents(in: tokenizer), + let model = tokenizer["model"] as? [String: Any], + model["type"] as? String == "BPE", + let vocabulary = model["vocab"] as? [String: Any], + !vocabulary.isEmpty, + let merges = model["merges"] as? [Any], + !merges.isEmpty, + tokenizerClass(in: tokenizerConfig) == "Qwen2Tokenizer" else { + return nil + } + + let vocabularyEntries = vocabulary.compactMapValues { $0 as? Int } + let addedTokens = tokenizer["added_tokens"] as? [[String: Any]] ?? [] + let addedEntries = addedTokens.reduce(into: [String: Int]()) { result, token in + if let content = token["content"] as? String, let id = token["id"] as? Int { + result[content] = id + } + } + guard vocabularyEntries.count == vocabulary.count, + addedEntries.count == addedTokens.count else { + return nil + } + + let allEntries = vocabularyEntries.merging(addedEntries) { _, _ in -1 } + let ids = Array(allEntries.values) + guard allEntries.count == vocabularyEntries.count + addedEntries.count, + Set(ids).count == ids.count, + ids.allSatisfy({ $0 >= 0 && $0 < modelVocabularySize }), + let maximumID = ids.max(), + Set(ids) == Set(0...maximumID), + byteLevelAlphabet.allSatisfy({ vocabularyEntries[$0] != nil }), + vocabularyEntries.keys.allSatisfy(byteLevelTokenIsValid), + merges.allSatisfy({ mergeIsValid($0, vocabulary: vocabularyEntries) }), + requiredChatTokensAreMapped(tokenizerConfig, addedTokens: addedTokens), + configuredTokensAreMapped(tokenizerConfig, vocabulary: allEntries) else { + return nil + } + return maximumID + 1 + } + + private static func supportedComponents(in tokenizer: [String: Any]) -> Bool { + guard let normalizer = tokenizer["normalizer"] as? [String: Any], + normalizer["type"] as? String == "NFC", + let preTokenizer = tokenizer["pre_tokenizer"] as? [String: Any], + preTokenizer["type"] as? String == "Sequence", + let parts = preTokenizer["pretokenizers"] as? [[String: Any]], + parts.count == 2, + parts[0]["type"] as? String == "Split", + parts[1]["type"] as? String == "ByteLevel", + let pattern = parts[0]["pattern"] as? [String: Any], + pattern.values.contains(where: { ($0 as? String)?.isEmpty == false }), + let postProcessor = tokenizer["post_processor"] as? [String: Any], + postProcessor["type"] as? String == "ByteLevel", + let decoder = tokenizer["decoder"] as? [String: Any], + decoder["type"] as? String == "ByteLevel" else { + return false + } + return true + } + + private static func tokenizerClass(in config: [String: Any]) -> String? { + (config["tokenizer_class"] as? String)?.replacingOccurrences(of: "Fast", with: "") + } + + private static func configuredTokensAreMapped( + _ config: [String: Any], + vocabulary: [String: Int] + ) -> Bool { + ["unk_token", "bos_token", "eos_token", "pad_token"].allSatisfy { key in + guard let value = config[key], !(value is NSNull) else { return true } + let content = value as? String ?? (value as? [String: Any])?["content"] as? String + return content.flatMap { vocabulary[$0] } != nil + } + } + + private static func requiredChatTokensAreMapped( + _ config: [String: Any], + addedTokens: [[String: Any]] + ) -> Bool { + let specialTokens = Set(addedTokens.compactMap { token -> String? in + guard token["special"] as? Bool == true else { return nil } + return token["content"] as? String + }) + return specialTokens.contains("<|im_start|>") + && specialTokens.contains("<|im_end|>") + && configuredTokenContent(config["eos_token"]) == "<|im_end|>" + } + + private static func configuredTokenContent(_ value: Any?) -> String? { + value as? String ?? (value as? [String: Any])?["content"] as? String + } + + private static func mergeIsValid(_ value: Any, vocabulary: [String: Int]) -> Bool { + let pair: [String]? + if let array = value as? [String], array.count == 2 { + pair = array + } else if let string = value as? String { + let parts = string.split(separator: " ", omittingEmptySubsequences: false).map(String.init) + pair = parts.count == 2 ? parts : nil + } else { + pair = nil + } + guard let pair else { return false } + return vocabulary[pair[0] + pair[1]] != nil + } + + private static func byteLevelTokenIsValid(_ token: String) -> Bool { + token.unicodeScalars.allSatisfy { byteLevelScalars.contains($0) } + } + + private static let byteLevelAlphabet: [String] = { + var bytes = Array(33...126) + Array(161...172) + Array(174...255) + var codePoints = bytes + for byte in 0...255 where !bytes.contains(byte) { + bytes.append(byte) + codePoints.append(256 + codePoints.count - 188) + } + return codePoints.compactMap(UnicodeScalar.init).map(String.init) + }() + + private static let byteLevelScalars = Set(byteLevelAlphabet.compactMap { $0.unicodeScalars.first }) +} diff --git a/Sources/LLM/EspressoLLMEngine.swift b/Sources/LLM/EspressoLLMEngine.swift index 155a91e9..a63008e9 100644 --- a/Sources/LLM/EspressoLLMEngine.swift +++ b/Sources/LLM/EspressoLLMEngine.swift @@ -1,44 +1,75 @@ -import ESPRuntime +import ANELMRuntime import Foundation -import RealModelInference +import Tokenizers + +private final class ANELMGenerationContext { + var tokens: [Int32] = [] +} + +private func aneLMTokenCallback(_ token: Int32, _ context: UnsafeMutableRawPointer?) -> Int32 { + let isCancelled = withUnsafeCurrentTask { $0?.isCancelled ?? false } + guard !isCancelled, let context else { return 0 } + if token >= 0 { + Unmanaged.fromOpaque(context).takeUnretainedValue().tokens.append(token) + } + return 1 +} actor EspressoLLMEngine { private final class LoadedModel { let path: String - let name: String - var engine: RealModelInferenceEngine + let runtime: OpaquePointer + let tokenizer: any Tokenizers.Tokenizer + let samplerVocabularySize: Int - init(path: String, name: String, engine: consuming RealModelInferenceEngine) { + init( + path: String, + runtime: OpaquePointer, + tokenizer: any Tokenizers.Tokenizer, + samplerVocabularySize: Int + ) { self.path = path - self.name = name - self.engine = engine + self.runtime = runtime + self.tokenizer = tokenizer + self.samplerVocabularySize = samplerVocabularySize + } + + deinit { + ane_lm_destroy(runtime) } } private var model: LoadedModel? private var lastFailureMessage: String? - func loadModel(path: String) throws { + func loadModel(path: String) async throws { let expandedPath = NSString(string: path).expandingTildeInPath let url = URL(fileURLWithPath: expandedPath, isDirectory: true).standardizedFileURL lastFailureMessage = nil guard model?.path != url.path else { return } - Log.info("[EspressoLLMEngine] loading bundle: \(url.lastPathComponent)") + Log.info("[ANELMEngine] loading Qwen3 model: \(url.lastPathComponent)") do { - let bundle = try ESPRuntimeBundle.open(at: url) - let selection = try ESPRuntimeRunner.resolve(bundle: bundle) - guard selection.backend == .anePrivate else { - throw EspressoLLMError.aneBackendUnavailable + let validated = try await Self.makeValidatedTokenizer(at: url) + model = nil + var nativeError: UnsafeMutablePointer? + let runtime = url.path.withCString { ane_lm_create($0, &nativeError) } + guard let runtime else { + throw ANELMNativeError(Self.consumeNativeError(&nativeError)) } - - let engine = try RealModelInferenceEngine.build( - config: bundle.config, - weightDir: bundle.archive.weightsURL.path, - tokenizerDir: bundle.archive.tokenizerURL.path + guard !Task.isCancelled else { + ane_lm_destroy(runtime) + throw CancellationError() + } + model = LoadedModel( + path: url.path, + runtime: runtime, + tokenizer: validated.tokenizer, + samplerVocabularySize: validated.samplerVocabularySize ) - model = LoadedModel(path: url.path, name: bundle.config.name, engine: engine) - Log.info("[EspressoLLMEngine] bundle ready for ANE inference") + Log.info("[ANELMEngine] Qwen3 model ready for ANE inference") + } catch is CancellationError { + throw CancellationError() } catch { throw recordFailure(error) } @@ -52,28 +83,58 @@ actor EspressoLLMEngine { ) throws -> String { guard let model else { throw EspressoLLMError.modelNotLoaded } lastFailureMessage = nil - let input = Self.formatPrompt( - user: prompt, - system: systemPrompt, - modelName: model.name - ) + + let input = Self.formatPrompt(user: prompt, system: systemPrompt, modelName: "Qwen3") + let promptTokens = model.tokenizer + .encode(text: input, addSpecialTokens: false) + .map { Int32(clamping: $0) } + guard !promptTokens.isEmpty else { throw EspressoLLMError.runtimeFailure } + + let context = ANELMGenerationContext() + context.tokens.reserveCapacity(max(0, maxTokens)) + let contextPointer = Unmanaged.passUnretained(context).toOpaque() + let eosToken = Int32(clamping: model.tokenizer.eosTokenId ?? -1) + let stopToken = Int32(clamping: model.tokenizer.convertTokenToId("<|im_end|>") ?? -1) + var nativeError: UnsafeMutablePointer? let started = CFAbsoluteTimeGetCurrent() - let result: GenerationResult - do { - result = try model.engine.generate( - prompt: input, - maxTokens: maxTokens, - temperature: Float(temperature) + + let status = promptTokens.withUnsafeBufferPointer { tokens in + ane_lm_generate( + model.runtime, + tokens.baseAddress, + tokens.count, + Int32(clamping: max(1, maxTokens)), + Float(temperature), + 1.2, + Int32(clamping: model.samplerVocabularySize), + eosToken, + stopToken, + aneLMTokenCallback, + contextPointer, + &nativeError ) - } catch { + } + + if status == ANE_LM_STATUS_CANCELLED { + throw CancellationError() + } + guard status == ANE_LM_STATUS_OK else { + let error = ANELMNativeError(Self.consumeNativeError(&nativeError)) throw recordFailure(error) } + try Task.checkCancellation() + + let output = model.tokenizer.decode( + tokens: context.tokens.map(Int.init), + skipSpecialTokens: true + ) let elapsed = CFAbsoluteTimeGetCurrent() - started + let speed = elapsed > 0 ? Double(context.tokens.count) / elapsed : 0 Log.info( - "[EspressoLLMEngine] generated \(result.text.count) chars on ANE in " - + "\(String(format: "%.1f", elapsed))s (\(String(format: "%.1f", result.tokensPerSecond)) tok/s)" + "[ANELMEngine] generated \(context.tokens.count) tokens on ANE in " + + "\(String(format: "%.1f", elapsed))s (\(String(format: "%.1f", speed)) tok/s)" ) - return result.text + return output } var isLoaded: Bool { model != nil } @@ -88,12 +149,62 @@ actor EspressoLLMEngine { return lastFailureMessage } - private func recordFailure(_ error: Error) -> EspressoLLMError { - let mapped = error as? EspressoLLMError ?? .runtimeFailure - Log.sensitive("[EspressoLLMEngine] ANE runtime detail: \(error.localizedDescription)") - Log.error("[EspressoLLMEngine] \(mapped.localizedDescription)") - lastFailureMessage = mapped.localizedDescription - return mapped + static func validateModelDirectory(at url: URL) async throws { + _ = try await makeValidatedTokenizer(at: url) + } + + private struct ValidatedTokenizer { + let tokenizer: any Tokenizers.Tokenizer + let samplerVocabularySize: Int + } + + private static func makeValidatedTokenizer(at url: URL) async throws -> ValidatedTokenizer { + guard ModelStorage.llmRepoIsComplete(at: url) else { + throw EspressoLLMError.invalidModelDirectory + } + let configURL = url.appendingPathComponent("config.json") + guard let data = try? Data(contentsOf: configURL), + let config = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + config["model_type"] as? String == "qwen3" else { + throw EspressoLLMError.unsupportedModel + } + let textConfig = config["text_config"] as? [String: Any] ?? config + guard let modelVocabularySize = textConfig["vocab_size"] as? Int, + modelVocabularySize > 0 else { + throw EspressoLLMError.invalidModelDirectory + } + let tokenizerURL = url.appendingPathComponent("tokenizer.json") + let tokenizerConfigURL = url.appendingPathComponent("tokenizer_config.json") + guard let tokenizerData = try? Data(contentsOf: tokenizerURL), + let tokenizerJSON = try? JSONSerialization.jsonObject(with: tokenizerData) as? [String: Any], + let tokenizerConfigData = try? Data(contentsOf: tokenizerConfigURL), + let tokenizerConfig = try? JSONSerialization.jsonObject( + with: tokenizerConfigData + ) as? [String: Any], + let samplerVocabularySize = ANELMTokenizerValidation.samplerVocabularySize( + tokenizer: tokenizerJSON, + tokenizerConfig: tokenizerConfig, + modelVocabularySize: modelVocabularySize + ) else { + throw EspressoLLMError.invalidModelDirectory + } + var nativeError: UnsafeMutablePointer? + let status = url.path.withCString { ane_lm_validate_model($0, &nativeError) } + guard status == ANE_LM_STATUS_OK else { + let detail = consumeNativeError(&nativeError) + Log.sensitive("[ANELMEngine] rejected model directory: \(detail)") + throw EspressoLLMError.invalidModelDirectory + } + do { + let tokenizer = try await AutoTokenizer.from(modelFolder: url) + return ValidatedTokenizer( + tokenizer: tokenizer, + samplerVocabularySize: samplerVocabularySize + ) + } catch { + Log.sensitive("[ANELMEngine] rejected tokenizer: \(error.localizedDescription)") + throw EspressoLLMError.invalidModelDirectory + } } static func formatPrompt(user: String, system: String, modelName: String) -> String { @@ -104,17 +215,48 @@ actor EspressoLLMEngine { } return "System:\n\(system)\n\nUser:\n\(user)\n\nAssistant:\n" } + + private static func consumeNativeError(_ pointer: inout UnsafeMutablePointer?) -> String { + guard let allocated = pointer else { return "ANE-LM failed without an error message" } + defer { + ane_lm_free_string(allocated) + pointer = nil + } + return String(cString: allocated) + } + + private func recordFailure(_ error: Error) -> EspressoLLMError { + let mapped = error as? EspressoLLMError ?? .runtimeFailure + Log.sensitive("[ANELMEngine] runtime detail: \(error.localizedDescription)") + Log.error("[ANELMEngine] \(mapped.localizedDescription)") + lastFailureMessage = mapped.localizedDescription + return mapped + } +} + +private struct ANELMNativeError: LocalizedError { + let message: String + + init(_ message: String) { + self.message = message + } + + var errorDescription: String? { message } } enum EspressoLLMError: LocalizedError { case modelNotLoaded case aneBackendUnavailable + case invalidModelDirectory + case unsupportedModel case runtimeFailure var errorDescription: String? { switch self { case .modelNotLoaded: return L("error.espresso_not_loaded") case .aneBackendUnavailable: return L("error.espresso_ane_unavailable") + case .invalidModelDirectory: return L("error.espresso_invalid_model") + case .unsupportedModel: return L("error.espresso_unsupported_model") case .runtimeFailure: return L("error.espresso_runtime_failed") } } diff --git a/Sources/Processing/TextProcessor+Generation.swift b/Sources/Processing/TextProcessor+Generation.swift index 0dc36354..0a3d4aee 100644 --- a/Sources/Processing/TextProcessor+Generation.swift +++ b/Sources/Processing/TextProcessor+Generation.swift @@ -61,7 +61,7 @@ extension TextProcessor { if result.usedMLX { _ = await espressoLLM.consumeLastFailureMessage() await Self.recordEspressoOutcome(.fallback) - Log.info("[TextProcessor] Espresso failed; used the selected MLX model") + Log.info("[TextProcessor] ANE-LM failed; used the selected MLX model") } else { await Self.clearEspressoOutcome() } @@ -71,14 +71,14 @@ extension TextProcessor { } catch let error as EspressoMLXFallbackError { _ = await espressoLLM.consumeLastFailureMessage() await Self.recordEspressoOutcome(.unavailable) - Log.sensitive("[TextProcessor] Espresso and MLX fallback failed: \(error.details)") + Log.sensitive("[TextProcessor] ANE-LM and MLX fallback failed: \(error.details)") Log.error("[TextProcessor] MLX fallback unavailable") throw error } catch { if !options.fallbackToMLXOnEspressoFailure { _ = await espressoLLM.consumeLastFailureMessage() await Self.recordEspressoOutcome(.failed) - Log.error("[TextProcessor] Espresso failed; MLX fallback is disabled") + Log.error("[TextProcessor] ANE-LM failed; MLX fallback is disabled") } throw error } @@ -122,7 +122,7 @@ extension TextProcessor { } var details: String { - "Espresso: \(espressoFailure); MLX: \(mlxFailure)" + "ANE-LM: \(espressoFailure); MLX: \(mlxFailure)" } } diff --git a/Sources/Processing/TextProcessor+Models.swift b/Sources/Processing/TextProcessor+Models.swift index c01ca11a..29820e26 100644 --- a/Sources/Processing/TextProcessor+Models.swift +++ b/Sources/Processing/TextProcessor+Models.swift @@ -174,7 +174,7 @@ extension TextProcessor { } catch is CancellationError { throw CancellationError() } catch let error as EspressoMLXFallbackError { - Log.sensitive("[TextProcessor] Espresso and MLX warmup failed: \(error.details)") + Log.sensitive("[TextProcessor] ANE-LM and MLX warmup failed: \(error.details)") Log.error("[TextProcessor] MLX fallback unavailable during warmup") _ = await espressoLLM.consumeLastFailureMessage() return (false, EspressoGenerationOutcome.unavailable.message, .unavailable) diff --git a/Sources/Resources/en.lproj/Localizable.strings b/Sources/Resources/en.lproj/Localizable.strings index 481c750b..7b12e957 100644 --- a/Sources/Resources/en.lproj/Localizable.strings +++ b/Sources/Resources/en.lproj/Localizable.strings @@ -22,7 +22,7 @@ /* ── Status ── */ "status.ready" = "Ready"; "status.done" = "Done"; -"status.espresso_fell_back_to_mlx" = "Espresso failed on this Mac. This request finished with MLX."; +"status.espresso_fell_back_to_mlx" = "ANE-LM failed on this Mac. This request finished with MLX."; "status.no_speech_detected" = "No Speech Detected"; /* ── Tabs ── */ @@ -194,7 +194,7 @@ "model.preload.speech" = "Preload dictation model on launch"; "model.preload.speech_help" = "Applies to already downloaded WhisperKit models. Use the Download button in Models first."; "model.preload.formatting" = "Preload formatting model on launch"; -"model.preload.formatting_help" = "Applies to local MLX and Espresso models. Remote LLM providers are called when formatting starts."; +"model.preload.formatting_help" = "Applies to local MLX and ANE models. Remote LLM providers are called when formatting starts."; "model.speech_recognition" = "Speech Recognition"; "model.apple_managed_by_system" = "Apple Speech uses the language and recognition services managed by macOS."; "model.text_formatting" = "Text Formatting (LLM)"; @@ -203,12 +203,12 @@ "model.family.gemma" = "Google Gemma - Lightweight"; "model.family.llama" = "Meta Llama - General Purpose"; "model.family.remote" = "Remote"; -"model.espresso.description" = "Run a compatible .esp bundle with Espresso's Apple Neural Engine backend."; -"model.espresso.no_bundle" = "No .esp bundle selected"; -"model.espresso.choose" = "Choose .esp Bundle…"; -"model.espresso.auto_fallback" = "Automatically switch to MLX if Espresso fails"; -"model.espresso.auto_fallback_help" = "Finish the request with the selected installed MLX model and make MLX active. Turn this off to keep Espresso selected and show its error."; -"model.espresso.private_api_warning" = "Espresso uses private Apple Neural Engine APIs. Compatibility depends on the Mac and macOS version and is not guaranteed. Espresso is not eligible for Mac App Store distribution."; +"model.espresso.description" = "Run a local Qwen3 safetensors model with the experimental ANE-LM runtime."; +"model.espresso.no_bundle" = "No Qwen3 model selected"; +"model.espresso.choose" = "Choose Qwen3 Model…"; +"model.espresso.auto_fallback" = "Automatically switch to MLX if ANE-LM fails"; +"model.espresso.auto_fallback_help" = "Finish the request with the selected installed MLX model and make MLX active. Turn this off to keep ANE selected and show its error."; +"model.espresso.private_api_warning" = "ANE-LM uses private Apple Neural Engine APIs. Compatibility depends on the Mac and macOS version and is not guaranteed. It is not eligible for Mac App Store distribution. Models use substantial memory; macOS may temporarily retain some released memory after switching away."; "model.custom_id_placeholder" = "Custom model ID (e.g. mlx-community/…)"; "model.active" = "Active"; "model.use" = "Use"; @@ -468,8 +468,8 @@ "error.load_failed" = "Model loading failed: %@"; "error.network_request_failed" = "Network request failed — please try again"; "error.operation_failed" = "Operation failed — please try again"; -"error.espresso_runtime_failed" = "Espresso could not run this model on the ANE. Switch to MLX or try a supported macOS and device."; -"error.espresso_mlx_fallback_unavailable" = "Espresso failed, and the selected MLX model is unavailable. Download an MLX model in Settings → Models."; +"error.espresso_runtime_failed" = "ANE-LM could not run this model on the ANE. Switch to MLX or try a supported macOS and device."; +"error.espresso_mlx_fallback_unavailable" = "ANE-LM failed, and the selected MLX model is unavailable. Download an MLX model in Settings → Models."; "error.volc_not_configured" = "Doubao ASR not configured — enter API credentials in Settings → Models"; "error.volc_invalid_endpoint" = "Invalid ASR endpoint URL"; "error.volc_audio_conversion" = "Audio conversion to PCM 16kHz failed"; @@ -493,8 +493,10 @@ "model.asr_incomplete" = "Only part of the model was downloaded. Select Resume to finish."; "error.llm_not_loaded" = "The model is stored locally but is not loaded into memory. Run the action again and Utter will retry."; "error.llm_not_downloaded" = "The model files have not been downloaded. Open Settings → Models and confirm the data usage first."; -"error.espresso_not_loaded" = "The Espresso model bundle is not loaded. Select a valid .esp bundle in Settings → Models."; -"error.espresso_ane_unavailable" = "This Espresso bundle does not provide an Apple Neural Engine backend on this Mac."; +"error.espresso_not_loaded" = "The ANE model is not loaded. Select a valid local Qwen3 model in Settings → Models."; +"error.espresso_ane_unavailable" = "The Apple Neural Engine runtime is unavailable on this Mac."; +"error.espresso_invalid_model" = "This folder is not a complete local Qwen3 model. Select a folder containing config, tokenizer, and safetensors files."; +"error.espresso_unsupported_model" = "ANE-LM currently supports Qwen3 models only. Choose a Qwen3 model or use MLX."; "onboarding.download_notice" = "No download starts automatically. Review the size, then confirm if you want this local model."; "onboarding.download_size" = "Download (%@)"; diff --git a/Sources/Resources/zh-Hans.lproj/Localizable.strings b/Sources/Resources/zh-Hans.lproj/Localizable.strings index 3058dbcc..fc37af8b 100644 --- a/Sources/Resources/zh-Hans.lproj/Localizable.strings +++ b/Sources/Resources/zh-Hans.lproj/Localizable.strings @@ -22,7 +22,7 @@ /* ── Status ── */ "status.ready" = "就绪"; "status.done" = "完成"; -"status.espresso_fell_back_to_mlx" = "Espresso 在这台 Mac 上失败。本次已改用 MLX 完成。"; +"status.espresso_fell_back_to_mlx" = "ANE-LM 在这台 Mac 上失败。本次已改用 MLX 完成。"; "status.no_speech_detected" = "未检测到语音"; /* ── Tabs ── */ @@ -194,7 +194,7 @@ "model.preload.speech" = "启动时预加载听写模型"; "model.preload.speech_help" = "仅对已下载的 WhisperKit 模型生效。请先在模型页点击下载按钮。"; "model.preload.formatting" = "启动时预加载修稿模型"; -"model.preload.formatting_help" = "对本地 MLX 和 Espresso 模型生效。远程 LLM 会在整理开始时调用。"; +"model.preload.formatting_help" = "对本地 MLX 和 ANE 模型生效。远程 LLM 会在整理开始时调用。"; "model.speech_recognition" = "语音识别"; "model.apple_managed_by_system" = "Apple 语音使用由 macOS 管理的语言与识别服务。"; "model.text_formatting" = "文本整理 (LLM)"; @@ -203,12 +203,12 @@ "model.family.gemma" = "Google Gemma - 轻量"; "model.family.llama" = "Meta Llama - 通用"; "model.family.remote" = "远程"; -"model.espresso.description" = "使用 Espresso 的 Apple 神经网络引擎后端运行兼容的 .esp 模型包。"; -"model.espresso.no_bundle" = "尚未选择 .esp 模型包"; -"model.espresso.choose" = "选择 .esp 模型包…"; -"model.espresso.auto_fallback" = "Espresso 失败时自动切换到 MLX"; -"model.espresso.auto_fallback_help" = "使用已安装且选中的 MLX 模型完成本次请求,并将 MLX 设为当前后端。关闭后会保留 Espresso 并显示错误。"; -"model.espresso.private_api_warning" = "Espresso 使用 Apple 神经网络引擎私有 API,兼容性取决于 Mac 机型和 macOS 版本,无法保证。Espresso 也无法通过 Mac App Store 审核。"; +"model.espresso.description" = "使用实验性的 ANE-LM 运行本地 Qwen3 safetensors 模型。"; +"model.espresso.no_bundle" = "尚未选择 Qwen3 模型"; +"model.espresso.choose" = "选择 Qwen3 模型…"; +"model.espresso.auto_fallback" = "ANE-LM 失败时自动切换到 MLX"; +"model.espresso.auto_fallback_help" = "使用已安装且选中的 MLX 模型完成本次请求,并将 MLX 设为当前后端。关闭后会保留 ANE 并显示错误。"; +"model.espresso.private_api_warning" = "ANE-LM 使用 Apple 神经网络引擎私有 API,兼容性取决于 Mac 机型和 macOS 版本,无法保证,也无法通过 Mac App Store 审核。模型会占用较多内存;切换后 macOS 仍可能暂时保留一部分已释放内存。"; "model.custom_id_placeholder" = "自定义模型 ID(如 mlx-community/…)"; "model.active" = "当前"; "model.use" = "启用"; @@ -468,8 +468,8 @@ "error.load_failed" = "模型加载失败: %@"; "error.network_request_failed" = "网络请求失败,请稍后重试"; "error.operation_failed" = "操作失败,请重试"; -"error.espresso_runtime_failed" = "Espresso 无法在这台设备的 ANE 上运行此模型。请切换到 MLX,或改用受支持的 macOS 与设备。"; -"error.espresso_mlx_fallback_unavailable" = "Espresso 运行失败,所选 MLX 模型也不可用。请在“设置 → 模型”中下载一个 MLX 模型。"; +"error.espresso_runtime_failed" = "ANE-LM 无法在这台设备的 ANE 上运行此模型。请切换到 MLX,或改用受支持的 macOS 与设备。"; +"error.espresso_mlx_fallback_unavailable" = "ANE-LM 运行失败,所选 MLX 模型也不可用。请在“设置 → 模型”中下载一个 MLX 模型。"; "error.volc_not_configured" = "豆包语音识别未配置 — 请在 设置 → 模型 中填写 API 凭据"; "error.volc_invalid_endpoint" = "语音识别接口地址无效"; "error.volc_audio_conversion" = "音频转换为 PCM 16kHz 失败"; @@ -493,8 +493,10 @@ "model.asr_incomplete" = "模型只下载了一部分。点击“继续下载”即可接着完成"; "error.llm_not_loaded" = "模型文件已在本地,但当前尚未加载到内存。请重新执行;Utter 会再次尝试加载"; "error.llm_not_downloaded" = "模型文件尚未下载。请前往 设置 → 模型,确认流量后下载"; -"error.espresso_not_loaded" = "Espresso 模型包尚未加载。请在 设置 → 模型 中选择有效的 .esp 模型包。"; -"error.espresso_ane_unavailable" = "这个 Espresso 模型包无法在此 Mac 上使用 Apple 神经网络引擎后端。"; +"error.espresso_not_loaded" = "ANE 模型尚未加载。请在“设置 → 模型”中选择有效的本地 Qwen3 模型。"; +"error.espresso_ane_unavailable" = "这台 Mac 无法使用 Apple 神经网络引擎运行时。"; +"error.espresso_invalid_model" = "这个文件夹不是完整的本地 Qwen3 模型。请选择包含配置、分词器和 safetensors 权重的文件夹。"; +"error.espresso_unsupported_model" = "ANE-LM 目前只支持 Qwen3 模型。请选择 Qwen3 模型,或改用 MLX。"; "onboarding.download_notice" = "这里不会自动下载。请先确认体积,再决定是否下载这个本地模型。"; "onboarding.download_size" = "下载(%@)"; diff --git a/Sources/UI/ModelManagementSections.swift b/Sources/UI/ModelManagementSections.swift index 684e36fa..a5212ea4 100644 --- a/Sources/UI/ModelManagementSections.swift +++ b/Sources/UI/ModelManagementSections.swift @@ -199,7 +199,7 @@ extension ModelManagementView { var espressoLLMSection: some View { VStack(alignment: .leading, spacing: 10) { - Label("Espresso", systemImage: "neural.engine") + Label("ANE-LM", systemImage: "neural.engine") .font(.system(size: 12, weight: .semibold)) Text(L("model.espresso.description")) @@ -216,7 +216,7 @@ extension ModelManagementView { .textSelection(.enabled) Spacer() Button(L("model.espresso.choose")) { - chooseEspressoBundle() + chooseANEModel() } .controlSize(.small) } diff --git a/Sources/UI/ModelManagementView.swift b/Sources/UI/ModelManagementView.swift index 2d0564bf..77227f8e 100644 --- a/Sources/UI/ModelManagementView.swift +++ b/Sources/UI/ModelManagementView.swift @@ -1,6 +1,5 @@ import SwiftUI import AppKit -import ESPRuntime struct ModelManagementView: View { @EnvironmentObject var settings: AppSettings @@ -168,7 +167,7 @@ extension ModelManagementView { } } - func chooseEspressoBundle() { + func chooseANEModel() { let panel = NSOpenPanel() panel.canChooseFiles = false panel.canChooseDirectories = true @@ -177,16 +176,18 @@ extension ModelManagementView { panel.message = L("model.espresso.choose") guard panel.runModal() == .OK, let url = panel.url else { return } - do { - _ = try ESPRuntimeBundle.open(at: url) - onUnloadLLM?() - settings.espressoModelPath = url.path - settings.localLLMBackend = .espresso - settings.useRemoteLLM = false - onLoadLLM?() - } catch { - importErrorMessage = error.localizedDescription - showImportError = true + Task { @MainActor in + do { + try await EspressoLLMEngine.validateModelDirectory(at: url) + onUnloadLLM?() + settings.espressoModelPath = url.path + settings.localLLMBackend = .espresso + settings.useRemoteLLM = false + onLoadLLM?() + } catch { + importErrorMessage = error.localizedDescription + showImportError = true + } } } diff --git a/Sources/UI/SettingsView.swift b/Sources/UI/SettingsView.swift index 91f540b0..187108fe 100644 --- a/Sources/UI/SettingsView.swift +++ b/Sources/UI/SettingsView.swift @@ -52,7 +52,7 @@ struct SettingsView: View { .tabItem { Label(L("tab.about"), systemImage: "info.circle") } } .frame(width: SettingsWindowLayout.width, height: SettingsWindowLayout.height) - .background(Color(nsColor: .windowBackgroundColor)) + .background(Color(nsColor: .underPageBackgroundColor)) .id(settings.uiLanguage) } } diff --git a/Sources/UI/SettingsVoiceIllustration.swift b/Sources/UI/SettingsVoiceIllustration.swift index 051c45a3..bd1afba9 100644 --- a/Sources/UI/SettingsVoiceIllustration.swift +++ b/Sources/UI/SettingsVoiceIllustration.swift @@ -21,6 +21,6 @@ struct SettingsCardBackground: View { extension View { func settingsPageSurface() -> some View { frame(maxWidth: .infinity, maxHeight: .infinity) - .background(Color(nsColor: .windowBackgroundColor)) + .background(Color(nsColor: .underPageBackgroundColor)) } } diff --git a/Tests/OpenTypeTests/ANELMRuntimeTestFixtures.swift b/Tests/OpenTypeTests/ANELMRuntimeTestFixtures.swift new file mode 100644 index 00000000..4a2a3e1d --- /dev/null +++ b/Tests/OpenTypeTests/ANELMRuntimeTestFixtures.swift @@ -0,0 +1,160 @@ +import Foundation + +let syntheticANEModelVocabularySize = 262 + +func makeSyntheticQwen3Directory( + dtype: String = "BF16", + omitting omittedName: String? = nil +) throws -> URL { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("utter-ane-model-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + try writeSyntheticQwen3Config(to: directory) + try writeSyntheticTokenizer(to: directory) + try writeSyntheticTokenizerConfig(to: directory) + + let tensors: [(String, [Int])] = [ + ("model.embed_tokens.weight", [syntheticANEModelVocabularySize, 2]), + ("model.norm.weight", [2]), + ("model.layers.0.input_layernorm.weight", [2]), + ("model.layers.0.post_attention_layernorm.weight", [2]), + ("model.layers.0.self_attn.q_norm.weight", [2]), + ("model.layers.0.self_attn.k_norm.weight", [2]), + ("model.layers.0.self_attn.q_proj.weight", [2, 2]), + ("model.layers.0.self_attn.k_proj.weight", [2, 2]), + ("model.layers.0.self_attn.v_proj.weight", [2, 2]), + ("model.layers.0.self_attn.o_proj.weight", [2, 2]), + ("model.layers.0.mlp.gate_proj.weight", [3, 2]), + ("model.layers.0.mlp.up_proj.weight", [3, 2]), + ("model.layers.0.mlp.down_proj.weight", [2, 3]), + ].filter { $0.0 != omittedName } + + var offset = 0 + var header: [String: Any] = [:] + for (name, shape) in tensors { + let byteCount = shape.reduce(1, *) * 2 + header[name] = [ + "dtype": dtype, + "shape": shape, + "data_offsets": [offset, offset + byteCount], + ] + offset += byteCount + } + var headerData = try JSONSerialization.data(withJSONObject: header, options: [.sortedKeys]) + headerData.append(contentsOf: repeatElement(0x20, count: (8 - headerData.count % 8) % 8)) + var headerLength = UInt64(headerData.count).littleEndian + var file = withUnsafeBytes(of: &headerLength) { Data($0) } + file.append(headerData) + file.append(Data(repeating: 0, count: offset)) + try file.write(to: directory.appendingPathComponent("model.safetensors")) + return directory +} + +func writeSyntheticQwen3Config( + to directory: URL, + qHeads: Int = 1, + kvHeads: Int = 1 +) throws { + let config: [String: Any] = [ + "model_type": "qwen3", + "hidden_size": 2, + "intermediate_size": 3, + "num_hidden_layers": 1, + "num_attention_heads": qHeads, + "num_key_value_heads": kvHeads, + "head_dim": 2, + "vocab_size": syntheticANEModelVocabularySize, + "max_position_embeddings": 16, + "tie_word_embeddings": true, + ] + try JSONSerialization.data(withJSONObject: config, options: [.sortedKeys]) + .write(to: directory.appendingPathComponent("config.json")) +} + +func rewriteSyntheticTokenizer( + at directory: URL, + mutate: (inout [String: Any]) -> Void +) throws { + var tokenizer = syntheticTokenizer() + mutate(&tokenizer) + try writeJSON(tokenizer, to: directory.appendingPathComponent("tokenizer.json")) +} + +func rewriteSyntheticTokenizerConfig( + at directory: URL, + mutate: (inout [String: Any]) -> Void +) throws { + var config = syntheticTokenizerConfig() + mutate(&config) + try writeJSON(config, to: directory.appendingPathComponent("tokenizer_config.json")) +} + +private func writeSyntheticTokenizer(to directory: URL) throws { + try writeJSON(syntheticTokenizer(), to: directory.appendingPathComponent("tokenizer.json")) +} + +private func writeSyntheticTokenizerConfig(to directory: URL) throws { + try writeJSON( + syntheticTokenizerConfig(), + to: directory.appendingPathComponent("tokenizer_config.json") + ) +} + +private func syntheticTokenizer() -> [String: Any] { + var vocabulary = Dictionary( + uniqueKeysWithValues: byteLevelAlphabet().enumerated().map { ($0.element, $0.offset) } + ) + vocabulary["of"] = 256 + return [ + "version": "1.0", + "added_tokens": [ + ["id": 257, "content": "", "special": true], + ["id": 258, "content": "", "special": true], + ["id": 259, "content": "<|im_start|>", "special": true], + ["id": 260, "content": "<|im_end|>", "special": true], + ["id": 261, "content": "", "special": true], + ], + "model": [ + "type": "BPE", + "vocab": vocabulary, + "merges": [["o", "f"]], + "unk_token": "", + ], + "normalizer": ["type": "NFC"], + "pre_tokenizer": [ + "type": "Sequence", + "pretokenizers": [ + ["type": "Split", "pattern": ["String": " "], "invert": false], + ["type": "ByteLevel", "add_prefix_space": false, "use_regex": false], + ], + ], + "post_processor": ["type": "ByteLevel", "use_regex": false], + "decoder": ["type": "ByteLevel", "use_regex": false], + ] +} + +private func syntheticTokenizerConfig() -> [String: Any] { + [ + "tokenizer_class": "Qwen2Tokenizer", + "bos_token": "", + "eos_token": "<|im_end|>", + "unk_token": "", + "pad_token": "", + "model_max_length": 128, + ] +} + +private func byteLevelAlphabet() -> [String] { + var bytes = Array(33...126) + Array(161...172) + Array(174...255) + var codePoints = bytes + for byte in 0...255 where !bytes.contains(byte) { + bytes.append(byte) + codePoints.append(256 + codePoints.count - 188) + } + return codePoints.compactMap(UnicodeScalar.init).map(String.init) +} + +private func writeJSON(_ object: [String: Any], to url: URL) throws { + try JSONSerialization.data(withJSONObject: object, options: [.sortedKeys]) + .write(to: url) +} diff --git a/Tests/OpenTypeTests/ANELMRuntimeTests.swift b/Tests/OpenTypeTests/ANELMRuntimeTests.swift new file mode 100644 index 00000000..40dce3e7 --- /dev/null +++ b/Tests/OpenTypeTests/ANELMRuntimeTests.swift @@ -0,0 +1,228 @@ +import Foundation +import XCTest +@testable import OpenType + +final class ANELMRuntimeTests: XCTestCase { + func testModelValidationRejectsTruncatedMissingAndWrongDtypeWeights() async throws { + let valid = try makeSyntheticQwen3Directory() + defer { try? FileManager.default.removeItem(at: valid) } + try await EspressoLLMEngine.validateModelDirectory(at: valid) + + let truncated = try makeSyntheticQwen3Directory() + defer { try? FileManager.default.removeItem(at: truncated) } + try Data([1]).write(to: truncated.appendingPathComponent("model.safetensors")) + await XCTAssertThrowsErrorAsync(try await EspressoLLMEngine.validateModelDirectory(at: truncated)) + + let missing = try makeSyntheticQwen3Directory(omitting: "model.layers.0.self_attn.q_proj.weight") + defer { try? FileManager.default.removeItem(at: missing) } + await XCTAssertThrowsErrorAsync(try await EspressoLLMEngine.validateModelDirectory(at: missing)) + + let wrongDtype = try makeSyntheticQwen3Directory(dtype: "F16") + defer { try? FileManager.default.removeItem(at: wrongDtype) } + await XCTAssertThrowsErrorAsync(try await EspressoLLMEngine.validateModelDirectory(at: wrongDtype)) + } + + func testModelValidationRejectsUnsupportedConfigAndMissingTokenizer() async throws { + let directory = try makeSyntheticQwen3Directory() + defer { try? FileManager.default.removeItem(at: directory) } + + try Data(#"{"model_type":"llama"}"#.utf8) + .write(to: directory.appendingPathComponent("config.json")) + await XCTAssertThrowsErrorAsync(try await EspressoLLMEngine.validateModelDirectory(at: directory)) + + try writeSyntheticQwen3Config(to: directory) + try FileManager.default.removeItem(at: directory.appendingPathComponent("tokenizer.json")) + await XCTAssertThrowsErrorAsync(try await EspressoLLMEngine.validateModelDirectory(at: directory)) + + let invalidTokenizer = try makeSyntheticQwen3Directory() + defer { try? FileManager.default.removeItem(at: invalidTokenizer) } + try Data("{}".utf8).write(to: invalidTokenizer.appendingPathComponent("tokenizer.json")) + await XCTAssertThrowsErrorAsync( + try await EspressoLLMEngine.validateModelDirectory(at: invalidTokenizer) + ) + + let unsafeTokenizer = try makeSyntheticQwen3Directory() + defer { try? FileManager.default.removeItem(at: unsafeTokenizer) } + try rewriteSyntheticTokenizer(at: unsafeTokenizer) { + $0["normalizer"] = ["type": "Bogus"] + } + await XCTAssertThrowsErrorAsync( + try await EspressoLLMEngine.validateModelDirectory(at: unsafeTokenizer) + ) + + let outOfRangeTokenizer = try makeSyntheticQwen3Directory() + defer { try? FileManager.default.removeItem(at: outOfRangeTokenizer) } + try rewriteSyntheticTokenizer(at: outOfRangeTokenizer) { + var addedTokens = $0["added_tokens"] as? [[String: Any]] ?? [] + addedTokens[0]["id"] = syntheticANEModelVocabularySize + $0["added_tokens"] = addedTokens + } + await XCTAssertThrowsErrorAsync( + try await EspressoLLMEngine.validateModelDirectory(at: outOfRangeTokenizer) + ) + + let missingUnknownToken = try makeSyntheticQwen3Directory() + defer { try? FileManager.default.removeItem(at: missingUnknownToken) } + try rewriteSyntheticTokenizerConfig(at: missingUnknownToken) { + $0["unk_token"] = "" + } + await XCTAssertThrowsErrorAsync( + try await EspressoLLMEngine.validateModelDirectory(at: missingUnknownToken) + ) + + let missingChatToken = try makeSyntheticQwen3Directory() + defer { try? FileManager.default.removeItem(at: missingChatToken) } + try rewriteSyntheticTokenizer(at: missingChatToken) { + var addedTokens = $0["added_tokens"] as? [[String: Any]] ?? [] + addedTokens[2]["content"] = "" + $0["added_tokens"] = addedTokens + } + await XCTAssertThrowsErrorAsync( + try await EspressoLLMEngine.validateModelDirectory(at: missingChatToken) + ) + + let nonSpecialChatToken = try makeSyntheticQwen3Directory() + defer { try? FileManager.default.removeItem(at: nonSpecialChatToken) } + try rewriteSyntheticTokenizer(at: nonSpecialChatToken) { + var addedTokens = $0["added_tokens"] as? [[String: Any]] ?? [] + addedTokens[3]["special"] = false + $0["added_tokens"] = addedTokens + } + await XCTAssertThrowsErrorAsync( + try await EspressoLLMEngine.validateModelDirectory(at: nonSpecialChatToken) + ) + + let wrongEndToken = try makeSyntheticQwen3Directory() + defer { try? FileManager.default.removeItem(at: wrongEndToken) } + try rewriteSyntheticTokenizerConfig(at: wrongEndToken) { + $0["eos_token"] = "" + } + await XCTAssertThrowsErrorAsync( + try await EspressoLLMEngine.validateModelDirectory(at: wrongEndToken) + ) + + let invalidByteToken = try makeSyntheticQwen3Directory() + defer { try? FileManager.default.removeItem(at: invalidByteToken) } + try rewriteSyntheticTokenizer(at: invalidByteToken) { + var model = $0["model"] as? [String: Any] ?? [:] + var vocabulary = model["vocab"] as? [String: Any] ?? [:] + vocabulary["☃"] = vocabulary.removeValue(forKey: "!") ?? 0 + model["vocab"] = vocabulary + $0["model"] = model + } + await XCTAssertThrowsErrorAsync( + try await EspressoLLMEngine.validateModelDirectory(at: invalidByteToken) + ) + + let invalidHeads = try makeSyntheticQwen3Directory() + defer { try? FileManager.default.removeItem(at: invalidHeads) } + try writeSyntheticQwen3Config(to: invalidHeads, qHeads: 3, kvHeads: 2) + await XCTAssertThrowsErrorAsync(try await EspressoLLMEngine.validateModelDirectory(at: invalidHeads)) + } + + func testRealGenerationLifecycleWhenModelIsProvided() async throws { + guard let modelPath = ProcessInfo.processInfo.environment["UTTER_ANE_TEST_MODEL"], + !modelPath.isEmpty else { + throw XCTSkip("Set UTTER_ANE_TEST_MODEL to a complete local Qwen3 model directory") + } + + let iterations = Int( + ProcessInfo.processInfo.environment["UTTER_ANE_TEST_ITERATIONS"] ?? "20" + ) ?? 20 + XCTAssertGreaterThanOrEqual(iterations, 20) + let baselineResidentSize = try residentSizeKB() + let engine = EspressoLLMEngine() + var residentSamples: [Int] = [] + var unloadMinima: [Int] = [] + var lifecycleGrowth: [Int] = [] + let lifecycleCount = 3 + + for lifecycle in 0.. Int { + let process = Process() + let pipe = Pipe() + process.executableURL = URL(fileURLWithPath: "/bin/ps") + process.arguments = ["-o", "rss=", "-p", String(ProcessInfo.processInfo.processIdentifier)] + process.standardOutput = pipe + try process.run() + process.waitUntilExit() + let data = pipe.fileHandleForReading.readDataToEndOfFile() + let value = String(decoding: data, as: UTF8.self) + .trimmingCharacters(in: .whitespacesAndNewlines) + guard process.terminationStatus == 0, let size = Int(value) else { + throw NSError(domain: "ANELMRuntimeTests", code: 1) + } + return size + } +} + +private func XCTAssertThrowsErrorAsync( + _ expression: @autoclosure () async throws -> Void, + file: StaticString = #filePath, + line: UInt = #line +) async { + do { + try await expression() + XCTFail("Expected async expression to throw", file: file, line: line) + } catch { } +} diff --git a/Tests/OpenTypeTests/ConfigurationTests.swift b/Tests/OpenTypeTests/ConfigurationTests.swift index d0da23ab..508faeac 100644 --- a/Tests/OpenTypeTests/ConfigurationTests.swift +++ b/Tests/OpenTypeTests/ConfigurationTests.swift @@ -323,11 +323,11 @@ final class ConfigurationTests: XCTestCase { XCTAssertTrue(settings.espressoModelPath.isEmpty) settings.localLLMBackend = .espresso - settings.espressoModelPath = "/tmp/qwen.esp" + settings.espressoModelPath = "/tmp/Qwen3-0.6B" let reloaded = AppSettings(defaults: defaults) XCTAssertEqual(reloaded.localLLMBackend, .espresso) - XCTAssertEqual(reloaded.espressoModelPath, "/tmp/qwen.esp") + XCTAssertEqual(reloaded.espressoModelPath, "/tmp/Qwen3-0.6B") } func testEspressoPromptUsesQwenChatTemplate() { diff --git a/Tests/OpenTypeTests/EspressoFallbackTests.swift b/Tests/OpenTypeTests/EspressoFallbackTests.swift index 915be6d9..f13dca4c 100644 --- a/Tests/OpenTypeTests/EspressoFallbackTests.swift +++ b/Tests/OpenTypeTests/EspressoFallbackTests.swift @@ -145,12 +145,12 @@ final class EspressoFallbackTests: XCTestCase { defer { defaults.removePersistentDomain(forName: suiteName) } let settings = AppSettings(defaults: defaults) settings.localLLMBackend = .espresso - settings.espressoModelPath = "/models/new.esp" + settings.espressoModelPath = "/models/Qwen3-new" XCTAssertFalse(EspressoFallbackPolicy.selectMLXIfNeeded( after: .fallback, settings: settings, - expectedEspressoModelPath: "/models/old.esp" + expectedEspressoModelPath: "/models/Qwen3-old" )) XCTAssertEqual(settings.localLLMBackend, .espresso) } @@ -182,20 +182,20 @@ final class EspressoFallbackTests: XCTestCase { } } - func testRealEspressoFailureFallsBackToInstalledMLX() async throws { + func testRealANEFailureFallsBackToInstalledMLX() async throws { let environment = ProcessInfo.processInfo.environment - guard environment["OPENTYPE_ESPRESSO_MLX_FALLBACK_INTEGRATION"] == "1" else { - throw XCTSkip("Set OPENTYPE_ESPRESSO_MLX_FALLBACK_INTEGRATION=1 to run") + guard environment["OPENTYPE_ANE_MLX_FALLBACK_INTEGRATION"] == "1" else { + throw XCTSkip("Set OPENTYPE_ANE_MLX_FALLBACK_INTEGRATION=1 to run") } - guard let bundlePath = environment["OPENTYPE_ESPRESSO_BUNDLE"], + guard let modelPath = environment["OPENTYPE_ANE_FAILURE_MODEL"], let mlxModel = environment["OPENTYPE_MLX_MODEL"] else { - throw XCTSkip("Set OPENTYPE_ESPRESSO_BUNDLE and OPENTYPE_MLX_MODEL") + throw XCTSkip("Set OPENTYPE_ANE_FAILURE_MODEL and OPENTYPE_MLX_MODEL") } var options = TextProcessingOptions(settings: AppSettings.shared, inputLanguage: .english) options.useRemoteLLM = false options.localLLMBackend = .espresso - options.espressoModelPath = bundlePath + options.espressoModelPath = modelPath options.llmModel = mlxModel let processor = TextProcessor() diff --git a/docs/research/2026-08-31-apple-neural-engine-m5-compatibility.md b/docs/research/2026-08-31-apple-neural-engine-m5-compatibility.md new file mode 100644 index 00000000..878e8e57 --- /dev/null +++ b/docs/research/2026-08-31-apple-neural-engine-m5-compatibility.md @@ -0,0 +1,208 @@ +# M5 / macOS 27 上的 Apple Neural Engine 兼容性调查 + +日期:2026-08-31 +范围:Utter 当前 Espresso 后端、Apple 公开的 Core ML / Core AI 路径,以及可直接访问 ANE 的开源项目。这里的“M5”包含本次实际可用的 M5 Max;没有把仅有项目自述、但没有设备和系统版本证据的支持声明当作已验证事实。 + +> 证据标记:**[事实]** 可由 Apple 文档、上游仓库或本机复现实验直接核对;**[推断]** 是基于这些事实的工程判断;**[未确认]** 仍需针对 Utter 的模型和发布形态验证。 + +## 结论 + +**Apple 没有在 M5 或 macOS 27 上彻底封堵 ANE。** 当前证据反而能排除“硬件被关闭”或“普通进程一律无权访问”这两种解释: + +- **[事实] 公开路径仍在。** Apple 当前的 [Core ML 文档](https://developer.apple.com/documentation/coreml)仍明确写明会利用 CPU、GPU 和 Neural Engine;[`MLComputeUnits`](https://developer.apple.com/documentation/coreml/mlcomputeunits)仍公开提供 `.all` 和 `.cpuAndNeuralEngine`。macOS 27 新增的 [Core AI](https://developer.apple.com/documentation/coreai)也明确面向 CPU、GPU 和 Neural Engine。 +- **[事实] 同一台 M5 Max / macOS 27.0 机器上,私有路径也能工作。** 本次从未修改的 [`maderix/ANE`](https://github.com/maderix/ANE) 源码编译运行了十组 `_ANEInMemoryModel` 程序、RMSNorm、32K classifier、softmax 和 classifier backward,全部成功。随后 [`ANEForge`](https://github.com/sbryngelson/ANEForge) 的另一条私有 `e5rt` 编译/执行路径也在同机跑通卷积、CNN 和 transformer encoder block。 +- **[事实] Utter 当前 Espresso 仍失败。** 当前 M5 Max / macOS 27 主机对 Espresso 的 24 组 MIL 组合全部返回 Apple 私有编译器 Code 10,真实 GPT-2 `.esp` 也在 layer 0 attention 失败。详见本仓库的 [验证记录](../sdlc/changes/2026-08-29-espresso-ane/verification.md)。 +- **[事实] Code 10 不是“ANE 被封”的专用错误。** [`oMLX` issue #3124](https://github.com/jundot/omlx/issues/3124)记录了一台 M5 Max / macOS 27 主机上 rc2 可编译、rc3 因给 `_ANEInMemoryModel` 调用 `setModelURL:` 而遭 bundle hash 校验 Code 10;维护者随后提交了[修复](https://github.com/jundot/omlx/commit/7d77098ed83d7abcafeeaffaa4de1491820021d7)。 + +因此,**[推断] Espresso 的失败是其生成的 MIL、权重/bundle 组织或私有运行时调用方式与 M5 Max / macOS 27 校验规则之间的具体不兼容,而不是 Apple 对 ANE 的系统性封锁。** 这也意味着它有修复空间,但不能保证私有 API 下一个系统版本仍稳定。 + +目前存在能在 M5 上直接调用 ANE 的开源实现,但**没有一个未经修改即可替换 Utter 当前 Swift 后端**。本次因此从 ANE-LM 做了窄 fork,只保留 Qwen3 加载、生成、取消、重置和卸载接口,并在 Utter 中继续保留用户可控的 MLX fallback。 + +## 先区分两个容易混淆的“神经加速器” + +- **Apple Neural Engine(ANE)**:独立的 Neural Engine。Core ML、Core AI,以及本报告中的 Espresso、maderix/ANE、ANEForge 私有路径讨论的是它。 +- **M5 GPU core 内的 Neural Accelerator**:属于 GPU/Metal 路径。Apple 的 [M5 Tech Talk](https://developer.apple.com/videos/play/tech-talks/111432/)说明 MLX、llama.cpp、PyTorch 可以通过 Metal 利用这类 GPU 加速器;这不等于它们使用独立 ANE。 + +**[事实]** Apple 官方 [`ml-explore/mlx`](https://github.com/ml-explore/mlx) README 将当前支持设备写为 CPU 和 GPU,没有 ANE backend。Utter 的 MLX fallback 在 M5 上可用,但它不是“换一种方式调用 ANE”。 + +## 证据矩阵 + +| 路径 | API 性质 | 设备 / 系统证据 | 当前结论 | +|---|---|---|---| +| Core ML | Apple 公开 API | Apple 文档持续列出 Neural Engine;FluidAudio 在 M5 Pro / macOS 26.5 做了 100 条 TTS 实测 | 可用,但系统决定每个算子落到 CPU、GPU 还是 ANE | +| Core AI | Apple 公开 API,macOS 27+ | Apple 文档明确跨 CPU、GPU、ANE;macOS 27 release notes 仍在改进大模型 ANE 加载和内存归属 | 是 macOS 27 上自定义现代模型的长期路径 | +| Utter + Espresso | 私有 `_ANEClient` / `_ANEInMemoryModel` | M5 Max / macOS 27,24/24 编译组合及真实 GPT-2 均 Code 10 | 当前不兼容;不能据此推断整个私有 API 被封 | +| maderix/ANE | 私有 `_ANEInMemoryModel` | 本机 M5 Max / macOS 27,十组峰值程序及四个 classifier 测试全部通过 | 直接反证“私有 ANE 已被全面封锁” | +| ANEForge | 私有 `e5rt` | 上游有 M5/M5 Pro + macOS 26.5 数据;本机 M5 Max / macOS 27 CNN/transformer 均通过 | M5 可直接调用 ANE,但项目是 Python 前端和研究运行时 | +| oMLX | 私有 `_ANEInMemoryModel` | issue 报告 M5 Max / macOS 27:rc2 成功,rc3 Code 10;维护者定位到 `setModelURL:` | 证明 Code 10 可由 bundle/staging 细节触发,不是芯片封禁 | +| FluidAudio | 公开 Core ML | M5 Pro/macOS 26.5 与 M5/iPadOS 27 的实际运行和分阶段路由 | ANE 可用;个别 Apple GPU/BNNS/Core ML stage 仍会有 OS 特定 bug | +| ANE-LM | 私有 `_ANEInMemoryModel` | 本机原版真实 Qwen3-0.6B 在 fused QKV Code 10;[`IchenDEV/ANE-LM@033472e`](https://github.com/IchenDEV/ANE-LM/commit/033472ec12ea796fc7ea4f8cefd7ed456f69900b) 已完成 28 层、ANE LM head 和 Utter 产品路径实测 | 已作为精确 revision 集成;仍是实验性私有 API,只确认当前 M5 Max/macOS 27 + Qwen3-0.6B | +| ane-infer | 私有 API | README 声称 M1–M5;本次未做当前主机的真实模型复测 | 可作 Rust 混合运行时参考,不能当作已验证的 drop-in | + +## 公开 API 没有被撤掉 + +### Core ML:macOS 26 及更早兼容面的公开选择 + +**[事实]** Apple 在 [WWDC25 的机器学习框架总览](https://developer.apple.com/videos/play/wwdc2025/360/?time=675)明确说明,Core ML 会在运行时跨 CPU、GPU、Neural Engine 优化执行。`MLComputeUnits.cpuAndNeuralEngine` 允许 CPU 与 ANE、排除 GPU;`.all` 允许系统选择全部可用计算单元。 + +边界也很明确: + +- **[事实]** 公开枚举没有 `neuralEngineOnly`。应用能限制允许的设备集合,不能要求整个模型或每个 op 必须落到 ANE。 +- **[事实]** Apple 公开了 `MLComputePlan` 和 `MLNeuralEngineComputeDevice`,可以检查预期的逐 op 设备使用,但模型能否编译、如何分区仍由系统决定。 +- **[事实]** Apple 自己的 [`coremltools` issue #2687](https://github.com/apple/coremltools/issues/2687)包含 M5 上某些 shape 的 `mish/softplus` 实际路由到 ANE 的报告,同时也显示路由规则和数值边界会随芯片变化。这是“ANE 在工作”与“不是每个图都稳定”的同时证据。 + +### Core AI:macOS 27 的前向路径 + +**[事实]** [Core AI](https://developer.apple.com/documentation/coreai)从 macOS 27 起可用,Apple 将其定位为运行最新模型架构和推理技术的框架,并明确写明会使用 CPU、GPU 和 Neural Engine。Apple 同时提供 [`coreai-models`](https://github.com/apple/coreai-models) 和模型准备工具,不再要求开发者依赖私有 ANE ABI 才能运行现代模型。 + +**[事实]** [macOS 27 release notes](https://developer.apple.com/documentation/macos-release-notes/macos-27-release-notes)提到大模型在 Neural Engine 上的加载改进、ANE 内存归属到应用进程,以及后台 ANE 推理的新 entitlement。它描述的是受支持框架的资源管理和后台限制,不是前台应用被禁止访问 ANE。该 release note 仍是 beta 文档,且其中一条写作 iOS 27,因此不能把每项限制无差别外推到 macOS 前台运行。 + +### 公开路径的真实开源项目证据 + +**[事实]** [`FluidAudio` issue #667](https://github.com/FluidInference/FluidAudio/issues/667)和已合并的 [PR #671](https://github.com/FluidInference/FluidAudio/pull/671)在 M5 Pro / macOS 26.5 上把大部分 Kokoro 阶段固定为 `.cpuAndNeuralEngine`,100 条英文、中文语料均完成;只是把 ANE 不接受或 Apple framework 有 bug 的 tail stage 路由到 GPU。 + +**[事实]** 已合并的 [FluidAudio PR #849](https://github.com/FluidInference/FluidAudio/pull/849)又针对 M5 iPad Pro / OS 27 把有 MPSGraph 问题的 noise/tail 路由到 CPU,RNN 和 vocoder 继续使用 `.cpuAndNeuralEngine`。这说明 OS 27 上需要逐 stage 规避 bug,但不是 ANE 整体不可用。 + +## 私有 API:同机实测排除了全面封锁 + +### Utter 当前 Espresso 的失败边界 + +**[事实]** 当前分支已在 M5 Max / macOS 27 上验证: + +- Espresso 0.9.0 和当时 upstream main 的真实 GPT-2 generation 均在 layer 0 attention 返回 `verifyBundleAtPath: invalid model`、Code 10。 +- iOS 18、iOS 19、macOS 26、macOS 27 四个 MIL target,乘 LayerNorm/RMSNorm、spatial 64/128/256,共 24 组全部失败。 +- 同一请求切换到 MLX 后可以完成,因此失败在 Espresso 私有 ANE 编译/加载路径,而不是 Utter 的通用文本处理流水线。 + +上游 [`Espresso` README](https://github.com/christopherkarani/Espresso)只把 M1–M4 列为已测试,未列 M5。另一方面,[Espresso issue #18](https://github.com/christopherkarani/Espresso/issues/18)显示 M1 Ultra / macOS 26 的真实模型问题后来通过保持 `ios18` target 得到解决,说明“某个系统上的 Code 10/compile failure”需要按 MIL 和调用细节诊断,不能直接归因为 Apple 封锁。 + +### maderix/ANE:同一私有类、同一主机成功 + +本次在 Apple M5 Max、macOS 27.0(26A5421a)上检出 [`maderix/ANE` commit `d91c984`](https://github.com/maderix/ANE/commit/d91c9845c0784dec7753048954fc6d0e8411fe29),未修改源码: + +- **[事实]** [`inmem_peak.m`](https://github.com/maderix/ANE/blob/main/inmem_peak.m) 的十组 programmatic MIL 均完成 compile、load 和 eval;两次观察到的峰值约为 12.6–12.8 TFLOPS。 +- **[事实]** [`training/test_classifier.m`](https://github.com/maderix/ANE/blob/main/training/test_classifier.m) 完成四次 ANE 编译:RMSNorm 最大误差 `0.002856`;32K classifier projection 最大误差 `0.000831`;32K softmax 最大误差约 `0.000001`;classifier backward 最大误差 `0.006952`,全部判定通过。 +- **[事实]** 这些程序与 Utter 的 Espresso 一样使用 `_ANEInMemoryModelDescriptor`、`_ANEInMemoryModel`、`compileWithQoS`、`loadWithQoS` 和 IOSurface,而不是 Core ML 的公开模型 API。 + +这组结果直接证明,同一台机器上的 `_ANEInMemoryModel` 编译、加载和执行能力仍开放给普通进程。差异落在具体图、权重、bundle 或调用参数,而不是 API 类完全消失或权限被统一拒绝。 + +### ANEForge:另一条私有编译/执行栈也成功 + +**[事实]** [`ANEForge` README](https://github.com/sbryngelson/ANEForge)自述已在 M5 Pro 与 M1 Max 验证;其仓库还保存了 [M5 Pro / macOS 26.5.1](https://github.com/sbryngelson/ANEForge/blob/main/bench/results/rooflines/roofline-apple-m5-pro-Mac17_8-c38210cfc8ea-a154b89e.json)及 [M5 / macOS 26.5.2](https://github.com/sbryngelson/ANEForge/blob/main/bench/results/rooflines/roofline-apple-m5-Mac17_3-019d3ac27339-426ca052.json) 的可机器读取实测数据。 + +本次又在同一台 M5 Max / macOS 27.0 主机上,以未修改的 `ANEForge` commit `a0be5b5` 运行: + +- 最小 1×1 conv 完成编译和执行,输出 shape `(1, 1, 4, 4)`、sum `16.0`。 +- 上游 [`examples/quickstart.py`](https://github.com/sbryngelson/ANEForge/blob/main/examples/quickstart.py) 的 CNN 在 fp16/int8 下相对误差为 `0.0023/0.0112`,均通过。 +- 同一 quickstart 的 RMSNorm + attention + FFN transformer encoder block 在 fp16/int8 下相对误差为 `0.0029/0.0087`,均通过。 + +ANEForge 的 [`ane_e5rt_dispatch.mm`](https://github.com/sbryngelson/ANEForge/blob/main/aneforge/_lib/ane_e5rt_dispatch.mm)使用私有 `e5rt_e5_compiler_*` 与 execution stream 符号。这是与 Espresso `_ANEClient` 路径不同的第二个直接 ANE 反例。 + +但它不是 Utter 的直接依赖候选:其图构建、lowering 和模型前端以 Python 为主,而且首次运行会本地编译 dylib。Utter 若复用,只适合把已验证的窄 C ABI / MIL 生成模式移植进构建期产物,不能在已签名应用内照搬运行时 Python 与动态编译流程。 + +### oMLX:Code 10 的具体回归样本 + +**[事实]** [`oMLX issue #3124`](https://github.com/jundot/omlx/issues/3124)提供了很有辨识度的矩阵: + +- M5 Max / macOS 27 上 rc2 的 ANE procedure path 成功,rc3 在所有 split size 上 Code 10。 +- M5 Pro / macOS 26.6.2 上同一 rc3 可以成功。 +- M4 Max 和 M1 Ultra / macOS 27 也能复现 rc3,故失败不跟 M5 芯片本身绑定。 +- 维护者定位到 rc3 给 `_ANEInMemoryModel` 调用 `setModelURL:`,触发 macOS 27 的 per-file bundle hash 校验;[commit `7d77098`](https://github.com/jundot/omlx/commit/7d77098ed83d7abcafeeaffaa4de1491820021d7)移除了该覆盖。 + +修复已在 macOS 26.5 验证,但 issue 中尚没有原 macOS 27 报告者的最终回测。因此“修复方向”有源码证据,“已在 macOS 27 完全收口”仍属**[未确认]**。 + +### ANE-LM:原版失败,但最小差分 PoC 跑通真实 Qwen + +本次又在同一台 M5 Max / macOS 27.0 主机上检出 [`ANE-LM` commit +`04b1af1`](https://github.com/johnmai-dev/ANE-LM/commit/04b1af12aa30b3f280f511d286398469764b4326), +使用官方 Qwen3-0.6B safetensors 做了端到端对照: + +- **[事实] 未修改上游版本失败。** 原版可以编译并加载约 1.5 GB、28 层的模型权重,但在 layer 0 + 的 `first_proj` 立即返回 `verifyBundleAtPath: invalid model` / Code 10;关闭 ANE compile + cache、将提示缩到一个 token 后仍稳定复现。 +- **[事实] 失败缩到了 fused-QKV 图。** 同机运行 maderix/ANE 原版 + `test_fused_qkv` 时,“三卷积 + concat”同样 Code 10,而三个独立 Q、K、V 卷积全部编译成功。 + 这排除了 Qwen 权重、投影本身和 M5 权限,指向 macOS 27 私有编译器对该融合图的拒绝。 +- **[事实] 全拆 kernel 可以越过 Code 10,但会在约 126 个程序附近触发 Code 54 + `no ANE resources`。** 因此把每个投影永久拆成独立 model 不是可用方案。 +- **[事实] 临时差分 PoC 最终跑通。** PoC 将 MIL 更新为本机可接受的 + `program(1.3) / ios18` 形式,并在生成 MIL 前按输出通道拼接 Q/K/V 权重,让一个单输出 + convolution 自然产生连续的 Q、K、V 三段;Gate/Up 同样预拼成一个 convolution,激活和逐元素乘法 + 留在 CPU,Down 保持独立 ANE projection。这样每层保持四个 ANE program,共 112 个 layer + kernel;151,936 词表的 LM head 再分成 10 个 ANE chunk。 +- **[事实] 完整中文生成成功。** 27 个提示 token 约为 `21.621 token/s`,22 个生成 token + 约为 `22.597 token/s`,模型初始化约 `6.16 s`。实际输出为: + “苹果神经引擎(Apple Neural Engine)支持本地语言模型的运行,但需要通过特定的方式实现。” +- **[事实] 短内存循环未出现大幅逐轮增长。** 五轮同进程聊天在第一轮后的 RSS 为 + `1,309,760 KB`,随后为 `1,309,872`、`1,309,968`、`1,310,048`、 + `1,310,144 KB`,四轮累计约 `384 KB`。这只排除了明显的每轮大块泄漏,不能替代 + 20–100 轮、取消、卸载和空闲回落验证。 + +随后已将 PoC 收敛进 [`IchenDEV/ANE-LM`](https://github.com/IchenDEV/ANE-LM) 的 SwiftPM C++ library: + +- **[事实]** Utter 固定到不可变 revision `033472ec12ea796fc7ea4f8cefd7ed456f69900b`,不跟随移动分支;Swift 负责 tokenizer/chat template,C ABI 只暴露验证、加载、生成、重置和卸载。 +- **[事实]** 模型保存前会解析 safetensors 头,校验全部 Qwen3 必需 tensor、shape 和 BF16 dtype;原生编译入口也拒绝空权重指针,避免坏目录绕过错误回退直接崩溃。 +- **[事实]** 部分 ANE kernel 初始化失败会统一 unload 私有 model、释放已创建的 IOSurface、数组和 Objective-C 对象,并删除临时目录。 +- **[事实]** Utter 产品路径完成了三次完整加载、生成和卸载,在三轮中共执行 20 次短生成,并在第三轮另行验证了一次取消;每轮生成期间的 RSS 增量分别为 `80 KB`、`32 KB` 和 `0 KB`。fork 将大块临时权重与 embedding 改为可精确 `munmap` 的映射分配后,每轮 unload 相对当轮峰值都释放超过 `512 MiB`。 +- **[事实]** 三次 unload 后采样到的 RSS 低点分别为 `636,512 KB`、`150,928 KB` 和 `662,592 KB`,最终低点仅比第一次高 `26,080 KB`,低于测试强制的 `128 MiB` 跨生命周期上限。该结果没有出现随请求或切换次数单调累积的大块泄漏,但卸载后的 RSS 也没有稳定回到加载前 `54,368 KB` 基线。 +- **[推断]** `vmmap` 和独立 1 GiB malloc 探针说明系统分配器的回收时机会造成一部分波动;现有测试尚未完全隔离 Apple framework 或私有 ANE runtime 是否也保留了内存,因此不能把全部残留确定归因于 allocator。Utter 在 ANE 选项旁只披露“切换后已释放内存可能不会立即从系统监视器中消失”,没有承诺立即回到基线。 + +这把“能不能修”从临时 PoC 升级为当前项目可选择的实验后端,但没有改变私有 API 的发布和跨版本风险。 + +## 其他开源项目能否直接替代 + +### 能证明 M5 可直连,但不能原样嵌入 Utter + +- [`maderix/ANE`](https://github.com/maderix/ANE):本机已验证 private ANE 可用;它是研究样例和训练 PoC,不是通用 Swift 模型运行时。 +- [`ANEForge`](https://github.com/sbryngelson/ANEForge):本机已验证 M5 Max / macOS 27;模型覆盖很广,但当前产品形态是 Python frontend + Objective-C++ bridge。 +- [`oMLX`](https://github.com/jundot/omlx):已有 M5 Max / macOS 27 rc2 成功证据,并给出 Code 10 的修复线索;它不是 Swift Package,也不是 Utter `.esp` bundle 的替代格式。 +- [`ANE-LM`](https://github.com/johnmai-dev/ANE-LM):原生 C++、MIT。原版 fused-QKV 在当前主机失败;本次的 [`IchenDEV fork`](https://github.com/IchenDEV/ANE-LM) 已整理出 Swift/C ABI 并完成 Qwen3-0.6B 生命周期验证。Qwen3.5 的上游代码没有接入本次 Swift product,因此不能宣称当前 Utter 支持。 +- [`ane-infer`](https://github.com/thebasedcapital/ane-infer):Rust + Objective-C + Metal 的 Qwen3.5 专用混合引擎,README 声称覆盖 M1–M5;同样缺少本次当前主机的真实模型复测,而且不是通用 Swift 库。 + +[`hollance/neural-engine`](https://github.com/hollance/neural-engine)主要是 ANE 架构与设备资料,不是运行时,不能当作替代实现。 + +### App Store 与长期稳定性边界没有改变 + +上述直接 ANE 项目都调用未公开 framework 或符号。即使当前 M5 能运行,它们仍没有 ABI 稳定承诺,也不适合 App Store 发布。Espresso 的 README 同样明确说明私有 ANE API 会触发 App Store 拒绝。公开 Core ML / Core AI 才是可发布、可获得系统兼容承诺的路径。 + +## Utter 的最小可行技术路径 + +### 已实施的实验路径:ANE-LM + 用户可控 fallback + +Utter 现在允许用户选择普通本地 Qwen3 Hugging Face 目录,由修改后的 ANE-LM 在当前 M5 主机执行;默认允许失败后切换 MLX,用户可关闭。这样不会把私有 API 的一次系统回归变成语音输入不可用,同时保留错误可见性。 + +### 本次私有修复的边界 + +实现只承诺 Qwen3 safetensors 和本次实测的 Qwen3-0.6B。它没有证明 Qwen3.5、其他尺寸、M1–M4、普通 M5/M5 Pro、macOS 26 或后续 macOS 版本可用;这些组合仍需逐项做真实生成、取消、重复内存和卸载验证。即使验证通过,这条私有路径仍只适合实验、自签名或侧载版本。 + +### 受支持的长期路径 + +- **macOS 27+:优先 Core AI PoC。** 先转换一个 Utter 实际使用的小型 Qwen 模型,验证 state/KV cache、逐 token decode、内存、首 token 延迟和设备 placement。Apple 已把现代模型架构指向 Core AI,它比新写一套私有 ANE runtime 更可持续。 +- **若必须覆盖 macOS 26 的 ANE:再做 Core ML PoC。** 需要把 decoder prefill/decode、KV cache 和 tokenizer/采样拆开;用 `MLComputePlan` 验证真实 placement,而不是只因设置 `.cpuAndNeuralEngine` 就宣称全模型在 ANE。 +- **MLX 保持稳定底座。** 它走 CPU/GPU,不是 ANE,但当前模型资产、Swift 集成和产品稳定性已经得到验证,适合作为所有系统版本的 fallback。 + +最合理的双轨是:**短期把当前 ANE-LM fork 限定为可选择的实验后端;长期在 macOS 27 上用 Core AI 建立公开、可发布的 ANE 路径。** 不应把“Qwen3-0.6B 在一台 M5 Max 能运行”包装成跨设备、跨系统兼容承诺。 + +## 来源清单 + +### Apple + +- [Core ML documentation](https://developer.apple.com/documentation/coreml) +- [MLComputeUnits](https://developer.apple.com/documentation/coreml/mlcomputeunits) +- [WWDC25: Discover machine learning & AI frameworks on Apple platforms](https://developer.apple.com/videos/play/wwdc2025/360/?time=675) +- [Core AI documentation](https://developer.apple.com/documentation/coreai) +- [macOS 27 release notes](https://developer.apple.com/documentation/macos-release-notes/macos-27-release-notes) +- [WWDC26: Meet Core AI](https://developer.apple.com/videos/play/wwdc2026/324/) +- [Apple coreai-models](https://github.com/apple/coreai-models) + +### 开源项目官方仓库 + +- [christopherkarani/Espresso](https://github.com/christopherkarani/Espresso);[macOS 26 issue #18](https://github.com/christopherkarani/Espresso/issues/18) +- [maderix/ANE](https://github.com/maderix/ANE);[`inmem_peak.m`](https://github.com/maderix/ANE/blob/main/inmem_peak.m);[`test_classifier.m`](https://github.com/maderix/ANE/blob/main/training/test_classifier.m) +- [sbryngelson/ANEForge](https://github.com/sbryngelson/ANEForge);[`e5rt` bridge](https://github.com/sbryngelson/ANEForge/blob/main/aneforge/_lib/ane_e5rt_dispatch.mm);[`quickstart.py`](https://github.com/sbryngelson/ANEForge/blob/main/examples/quickstart.py) +- [jundot/oMLX issue #3124](https://github.com/jundot/omlx/issues/3124);[修复 commit](https://github.com/jundot/omlx/commit/7d77098ed83d7abcafeeaffaa4de1491820021d7) +- [FluidInference/FluidAudio issue #667](https://github.com/FluidInference/FluidAudio/issues/667);[PR #671](https://github.com/FluidInference/FluidAudio/pull/671);[PR #849](https://github.com/FluidInference/FluidAudio/pull/849) +- [apple/coremltools issue #2687](https://github.com/apple/coremltools/issues/2687) +- [ml-explore/mlx](https://github.com/ml-explore/mlx) +- [johnmai-dev/ANE-LM](https://github.com/johnmai-dev/ANE-LM) +- [IchenDEV/ANE-LM fork](https://github.com/IchenDEV/ANE-LM);[Utter 固定 revision `033472e`](https://github.com/IchenDEV/ANE-LM/commit/033472ec12ea796fc7ea4f8cefd7ed456f69900b) +- [thebasedcapital/ane-infer](https://github.com/thebasedcapital/ane-infer) diff --git a/docs/sdlc/changes/2026-08-31-ane-lm-runtime/intent.md b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/intent.md new file mode 100644 index 00000000..d719b5e0 --- /dev/null +++ b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/intent.md @@ -0,0 +1,58 @@ +# Intent: Replace Espresso with a packaged ANE-LM runtime + +## Problem + +The selectable ANE backend currently depends on Espresso, whose bundled ANE +program is rejected by the private compiler on the available M5 Max/macOS 27 +host. The original ANE-LM project also fails there because its fused projection +programs are rejected, even though smaller single-output ANE convolutions work. + +## Outcome + +Utter uses a reviewed ANE-LM fork whose Qwen3 projection layout compiles on the +available host. Users keep the existing ANE selection and optional MLX fallback, +while selecting an ordinary local Qwen3 Hugging Face model directory instead of +an Espresso bundle. + +## Scope + +This change packages the smallest ANE-LM inference runtime as a Swift package, +integrates load, generation, cancellation, reset, and unload into the existing +local-model lifecycle, updates model validation and product copy, and verifies +real Qwen3-0.6B inference. It does not add model download management, change the +MLX or remote backends, support training, or claim support for untested model +families and Apple devices. + +## Constraints + +- All prompts, weights, and generated tokens stay on-device. +- The runtime uses private Apple ANE APIs and is not App Store compatible. +- MLX remains the default local backend and the user controls fallback. +- Existing persisted backend and model-path keys remain readable. +- The fork is pinned to an immutable commit; no moving branch is accepted. +- Runtime ownership must be explicit so unload does not retain model weights or + compiled programs. + +## Acceptance criteria + +- The app resolves an exact `IchenDEV/ANE-LM` revision and removes Espresso. +- Selection accepts a complete Qwen3 directory and rejects missing or unsupported + configuration and weight files without replacing the saved selection. +- The available M5 Max/macOS 27 host generates non-empty Qwen3-0.6B output on ANE + without invoking MLX fallback. +- Enabled fallback completes with the selected installed MLX model after an ANE + error; disabled fallback keeps ANE selected and exposes the ANE error. +- Cancellation stops token generation, and explicit unload destroys the native + runtime before clearing reusable MLX allocations. +- A repeated real-generation sample has no sustained post-warmup growth above + 16 MiB across 20 short requests; any unavoidable retained allocation is + measured and documented. +- Existing automated tests and a release-style app build pass. +- User-visible copy identifies this as experimental private-API ANE support and + does not promise compatibility beyond tested evidence. + +## Open questions + +The proof of concept covers Qwen3-0.6B on one M5 Max/macOS 27 host. Qwen3.5, +other parameter sizes, older Apple Silicon, and future operating systems remain +outside this change until independently exercised. diff --git a/docs/sdlc/changes/2026-08-31-ane-lm-runtime/plan.md b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/plan.md new file mode 100644 index 00000000..25426334 --- /dev/null +++ b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/plan.md @@ -0,0 +1,35 @@ +# Plan: Replace Espresso with a packaged ANE-LM runtime + +## Work items + +- [x] Reduce the successful M5 proof of concept to the required Qwen3 program + layout and remove exploratory kernels. +- [x] Add a narrow C ABI and SwiftPM library product to `IchenDEV/ANE-LM`. +- [x] Pin the reviewed fork revision and remove Espresso products. +- [x] Replace bundle validation with supported Qwen3 directory validation. +- [x] Connect Swift tokenization to native generation through the existing ANE + actor and local-model lifecycle. +- [x] Preserve enabled and disabled MLX fallback behavior and settings migration. +- [x] Update localized private-API and compatibility copy. +- [x] Add focused behavioral tests where the changed seams are deterministic. +- [x] Exercise real M5 generation, cancellation, unload, and 20-request memory. +- [x] Resolve and close independent verifier findings. + +## Verification plan + +- [x] Focused package build and runtime tests in the ANE-LM fork. +- [x] Real Qwen3-0.6B generation on M5 Max/macOS 27. +- [x] Same-process 20-request resident-memory sample and post-unload sample. +- [x] Focused ANE selection and fallback tests. +- [x] `python3 scripts/sdlc.py validate --worktree` +- [x] `bash scripts/ci-basic-checks.sh` +- [x] `swift test` +- [x] `bash scripts/build-app.sh` +- [x] Independent high-risk verification. + +## Human gates + +The user explicitly requested the ANE-LM modification and current-project +integration after reviewing the original-runtime failure and proof-of-concept +result. PR approval, acceptance of residual private-API risk, and any release or +publication remain separate maintainer decisions. diff --git a/docs/sdlc/changes/2026-08-31-ane-lm-runtime/spec.md b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/spec.md new file mode 100644 index 00000000..50c4e361 --- /dev/null +++ b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/spec.md @@ -0,0 +1,75 @@ +# Spec: Replace Espresso with a packaged ANE-LM runtime + +## Context + +`TextProcessor` already owns one actor for the optional ANE backend and already +serializes local-model operations with MLX fallback. `ModelManagementView` +already owns local model selection. The current Espresso package crosses both +seams. ANE-LM supplies Qwen3 tensor loading, CPU operators, sampling, and private +ANE program compilation, but its CLI tokenizer and CMake-only packaging are not +needed by Utter. + +## Design + +Fork ANE-LM under `IchenDEV` and expose a single SwiftPM C++ library product. Its +public surface is a C ABI for runtime creation/destruction and token generation. +The handle owns one Qwen3 model, resets its KV cache before each independent +request, invokes a token callback, and treats a callback stop as cancellation. +Errors cross the ABI as owned UTF-8 strings with one matching free function. + +The M5 compatibility patch uses current MIL syntax and only single-output ANE +programs. Q, K, and V weights are packed row-wise before compilation and emitted +by one projection. Gate and up weights use the same layout; SiLU and elementwise +multiplication remain on CPU, and down projection remains one ANE program. This +keeps Qwen3 at four layer programs, avoiding both the compiler's fused/multiple- +output rejection and the host's finite ANE program resource limit. + +Swift keeps prompt construction and tokenization. `AutoTokenizer` loads from the +selected model folder, the existing chat prompt is encoded without added special +tokens, native generation returns token IDs, and Swift decodes the final IDs. +This avoids bringing ANE-LM's CLI, Jinja, or tokenizers-cpp dependency into the +app. Only `model_type == qwen3` with complete single-file or HF-indexed sharded +safetensors weights is admitted in this first version; unsupported directories +fail validation before native loading. + +The existing stored `.espresso` backend value, model-path key, engine property, +and fallback outcome identifiers remain internal compatibility names for this +change. Their behavior changes to ANE-LM while user-visible text says ANE or +ANE-LM. This preserves existing installations and avoids a second migration or +parallel model-management system. The external Espresso dependency and bundle +validation are removed completely. + +## Safety and failure modes + +Model loading can fail because files are incomplete, tensors do not match the +supported Qwen3 layout, private symbols change, compilation is rejected, or ANE +resources are exhausted. Validation handles structural failures; native errors +then follow the existing explicit fallback policy. No failure silently changes +the backend unless MLX actually completes the request. + +The native handle is actor-isolated. A new path or explicit unload destroys it; +cancellation stops the callback loop; each independent request resets model +state. The app must not call through a destroyed handle. Generated error strings +and temporary token buffers have one owner on each side of the C boundary. + +The API remains private and therefore unsuitable for Mac App Store distribution. +The selection UI retains this warning. Compatibility claims are limited to real +evidence recorded in verification. + +## Test strategy + +Focused tests cover directory validation, prompt token flow, fallback enabled +and disabled behavior, and engine lifecycle seams. The exact fork revision is +built through SwiftPM. A real Qwen3-0.6B model exercises direct generation on the +available M5 host, followed by 20 same-process short requests with resident-memory +samples and an unload sample. Repository checks, the full Swift suite, and a +release-style app build cover integration and packaging. An independent verifier +reviews ABI ownership, fallback semantics, dependency provenance, and evidence. + +## Rollout and rollback + +ANE remains explicit and non-default. Stop rollout on compiler rejection, +unbounded memory, corrupted output, or a lifecycle crash. Rollback restores the +pinned Espresso dependency and engine implementation from the preceding commit; +the persisted backend and path keys require no data migration. Publishing or +releasing the app remains a separate protected human action. diff --git a/docs/sdlc/changes/2026-08-31-ane-lm-runtime/state.json b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/state.json new file mode 100644 index 00000000..d9c1c3d4 --- /dev/null +++ b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/state.json @@ -0,0 +1,32 @@ +{ + "schemaVersion": 1, + "id": "2026-08-31-ane-lm-runtime", + "title": "Replace Espresso with a packaged ANE-LM runtime", + "risk": "high", + "status": "verified", + "owners": [ + "repository maintainer" + ], + "acceptanceCriteria": [ + "The ANE backend uses an exact pinned revision of the IchenDEV ANE-LM fork and no longer links Espresso.", + "A supported local Qwen3 Hugging Face model directory is rejected before loading when required configuration or weights are missing.", + "On the available M5 Max and macOS 27 host, Qwen3-0.6B produces non-empty output through ANE without falling back to MLX.", + "The existing user-selectable MLX fallback behavior remains unchanged for ANE load and generation failures.", + "Cancellation and explicit unload release the native runtime, and repeated generation stays within a documented post-warmup memory bound.", + "Existing MLX and remote LLM behavior continues to pass the automated test suite.", + "A release-style app build succeeds with the packaged C++ runtime and model resources.", + "The product continues to disclose the private-API and App Store compatibility boundary without claiming unsupported hardware coverage." + ], + "governedPaths": [ + "Package.swift", + "Package.resolved", + "Sources/", + "Tests/" + ], + "artifacts": { + "intent": "intent.md", + "spec": "spec.md", + "plan": "plan.md", + "verification": "verification.md" + } +} diff --git a/docs/sdlc/changes/2026-08-31-ane-lm-runtime/verification.md b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/verification.md new file mode 100644 index 00000000..2fc67cf0 --- /dev/null +++ b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/verification.md @@ -0,0 +1,58 @@ +# Verification: Replace Espresso with a packaged ANE-LM runtime + +## Evidence + +| Check | Result | Evidence | +|---|---|---| +| Exact dependency | Pass | `Package.swift` and `Package.resolved` resolve `IchenDEV/ANE-LM` at immutable revision `033472ec12ea796fc7ea4f8cefd7ed456f69900b`; no Espresso package or product remains. | +| Fork package build | Pass | `swift build -c release` passed for the fork; its CMake CLI also rebuilt after parser and cleanup hardening. | +| Fork CLI smoke | Pass | Rebuilt CLI generated two tokens from the official Qwen3-0.6B model at 19.197 prompt tok/s and 18.904 generation tok/s after the mapped-allocation change. | +| Model validation | Pass | Focused tests accept a structurally complete synthetic Qwen3 model and reject truncated or missing weights, F16, unsupported config, missing or malformed tokenizer data, unsupported tokenizer components, out-of-range token IDs, missing or non-special ChatML boundary tokens, an EOS mismatch, and an invalid GQA head ratio before loading ANE. | +| Real M5 Qwen3 generation | Pass | The Utter Swift tokenizer → pinned C ABI → private ANE path generated non-empty output on M5 Max/macOS 27 without MLX fallback. | +| Three-lifecycle memory sample | Pass | 20 requests across three complete load/generate/unload lifecycles, plus cancellation in the final lifecycle, passed in 84.367s. Within-lifecycle RSS growth was 80 KB, 32 KB, and 0 KB, each below the 16 MiB bound. Unload released more than 512 MiB from every lifecycle peak; the three sampled post-unload minima were 636,512 KB, 150,928 KB, and 662,592 KB. The final minimum was 26,080 KB above the first and below the enforced 128 MiB cross-lifecycle bound. | +| Fallback behavior | Pass | Existing enabled, disabled, load-failure, generation-failure, and cancellation policy tests pass in the full suite; selection changes only after native structural validation succeeds. | +| `python3 scripts/sdlc.py validate --worktree` | Pass | Final validation passed for 10 change bundles and all 29 changed governed paths after both active bundles reached verified. | +| `bash scripts/ci-basic-checks.sh` | Pass | Package, SDLC harness, plist and localization, identifiers, vocabulary, resources, conflict, secret-bearing file, and symlink checks passed after the final tokenizer fix. | +| `swift test` | Pass | 592 XCTest cases passed, 10 environment-gated cases skipped, and the Swift Testing model-upgrade test passed after the final runtime and UI changes. | +| `bash scripts/build-app.sh` | Pass | The final no-hook `Utter.app` and `Utter-0.0.43.dmg` built; app and mounted-DMG signatures passed and the DMG checksum verified. | +| Real-window UI | Pass | Actual 760 by 680 settings window exposed the ANE selector, Qwen3 directory chooser, enabled MLX fallback, and private-API/App Store warning with no clipping. | +| Independent verification | Pass | Final P1/P2-only review returned CLEAN after confirming ChatML boundary-token validation, the three tokenizer regressions, lifecycle evidence wording, and plan/verification consistency. | + +## Acceptance criteria + +- Exact fork pin and Espresso removal — Pass. +- Qwen3 directory validation — Pass, including required tensor, shape, BF16 + dtype, safe header, and tokenizer checks before saved selection changes. +- Real M5 ANE generation — Pass for official Qwen3-0.6B on the available M5 + Max/macOS 27 host. +- User-selectable MLX fallback — Pass in existing policy tests and real UI. +- Cancellation, unload, and bounded repeated-generation memory — Pass across + three complete lifecycles with one cancellation: 80/32/0 KB within-lifecycle + growth, at least 512 MiB recovery from each peak, and 26,080 KB final growth under the enforced + 128 MiB cross-lifecycle bound. +- Existing MLX and remote behavior — Pass in the full suite. +- Release-style packaging — Pass for the final no-hook app and mounted DMG. +- Private-API and compatibility disclosure — Pass in both localizations and the + real settings window. + +## Residual risk + +Only Qwen3-0.6B on one M5 Max/macOS 27 host has real product-path evidence. +Qwen3.5, other parameter sizes, other Apple Silicon devices, macOS 26, and +future operating-system versions remain outside the compatibility claim. The +runtime uses private APIs, has no ABI stability promise, and is not suitable for +Mac App Store distribution. Loading still compiles 122 private ANE programs and +can fail under resource pressure; the corrected path now cleans partial state +and follows the explicit fallback policy instead of retaining it. Post-unload +RSS was not deterministic: the three observed minima varied between 150,928 KB +and 662,592 KB even though request-time and cross-lifecycle growth stayed +bounded. `vmmap` and a standalone allocation probe indicate that allocator +reclaim timing contributes, but this test does not completely isolate framework +or private-runtime retention. The UI therefore only states that memory may not +immediately return to its pre-load level after switching away. + +## Decision + +Verified with focused and full automated tests, the final release-style +artifact, repository governance checks, and independent P1/P2 review. No +release or human approval is recorded. diff --git a/docs/sdlc/changes/2026-08-31-settings-semantic-background/intent.md b/docs/sdlc/changes/2026-08-31-settings-semantic-background/intent.md new file mode 100644 index 00000000..dcca96e9 --- /dev/null +++ b/docs/sdlc/changes/2026-08-31-settings-semantic-background/intent.md @@ -0,0 +1,35 @@ +# Intent: Match the macOS settings background hierarchy + +## Problem + +The settings shell forces `windowBackgroundColor` behind every tab. In dark +appearance that semantic color is substantially darker than the page surface +used by macOS Settings, so Utter reads as a near-black canvas beside the system +window. + +## Outcome + +The settings window uses the macOS semantic under-page background while keeping +the existing grouped sections, cards, controls, spacing, and fixed window size. + +## Scope + +Only the shared settings shell and page surface colors change. Tab content, +layout, behavior, model selection, other windows, and overlays are out of scope. + +## Constraints + +- Use an AppKit semantic color rather than a fixed RGB value. +- Keep the existing section/card contrast and all six tab layouts unchanged. +- Do not add appearance preferences or a custom visual-effect layer. + +## Acceptance criteria + +- The shared settings shell and page surfaces use `underPageBackgroundColor`. +- The real dark-appearance settings window has the lighter system-like page + hierarchy without clipping or reducing text/control contrast. +- Existing automated tests and the release-style build continue to pass. + +## Open questions + +Final visual acceptance remains a maintainer decision. diff --git a/docs/sdlc/changes/2026-08-31-settings-semantic-background/plan.md b/docs/sdlc/changes/2026-08-31-settings-semantic-background/plan.md new file mode 100644 index 00000000..d8a78db1 --- /dev/null +++ b/docs/sdlc/changes/2026-08-31-settings-semantic-background/plan.md @@ -0,0 +1,22 @@ +# Plan: Match the macOS settings background hierarchy + +## Work items + +- [x] Identify the shared forced background used by every settings tab. +- [x] Replace it with the macOS semantic under-page background. +- [x] Inspect Activity and the ANE model pane in the real dark window. +- [x] Record current evidence and residual risk. + +## Verification plan + +- [x] Real-window dark-appearance inspection +- [x] `swift test` +- [x] `python3 scripts/sdlc.py validate --worktree` +- [x] `bash scripts/ci-basic-checks.sh` +- [x] Final `bash scripts/build-app.sh` +- [x] `git diff --check` + +## Human gates + +The user reported the mismatch and requested the system-like background. Final +visual acceptance and any release remain separate maintainer decisions. diff --git a/docs/sdlc/changes/2026-08-31-settings-semantic-background/spec.md b/docs/sdlc/changes/2026-08-31-settings-semantic-background/spec.md new file mode 100644 index 00000000..c4ddb95a --- /dev/null +++ b/docs/sdlc/changes/2026-08-31-settings-semantic-background/spec.md @@ -0,0 +1,33 @@ +# Spec: Match the macOS settings background hierarchy + +## Context + +`SettingsView` and `settingsPageSurface()` both paint +`NSColor.windowBackgroundColor`. The duplicated opaque paint makes the whole +settings window use the dark-window base color even where macOS settings pages +normally use the lighter semantic under-page surface. + +## Design + +Replace those two shared paints with `NSColor.underPageBackgroundColor`. This is +the smallest change that keeps AppKit responsible for light/dark adaptation and +changes every settings tab consistently. Existing grouped forms and custom +cards continue to own their own semantic control backgrounds. + +## Safety and failure modes + +The color remains system-provided and appearance-aware. The main risk is reduced +contrast between page and grouped sections, so the actual dark window is +inspected on the affected model page and the card-heavy Activity page. + +## Test strategy + +Inspect the real built settings window in dark appearance, including Activity +and the ANE model pane, then run repository checks, the full Swift suite, and a +release-style artifact build. No unit test is added for a two-line semantic +color choice because it would only duplicate the implementation. + +## Rollout and rollback + +Ship through normal review. Roll back the two semantic-color substitutions if +the accepted system appearance or contrast is worse; no user data is affected. diff --git a/docs/sdlc/changes/2026-08-31-settings-semantic-background/state.json b/docs/sdlc/changes/2026-08-31-settings-semantic-background/state.json new file mode 100644 index 00000000..b5ee4fbd --- /dev/null +++ b/docs/sdlc/changes/2026-08-31-settings-semantic-background/state.json @@ -0,0 +1,25 @@ +{ + "schemaVersion": 1, + "id": "2026-08-31-settings-semantic-background", + "title": "Match the macOS settings background hierarchy", + "risk": "medium", + "status": "verified", + "owners": [ + "repository maintainer" + ], + "acceptanceCriteria": [ + "The shared settings shell and page surfaces use AppKit underPageBackgroundColor.", + "The real dark-appearance settings window has the lighter system-like hierarchy without clipping or reduced text and control contrast.", + "Existing automated tests and the release-style build continue to pass." + ], + "governedPaths": [ + "Sources/UI/SettingsView.swift", + "Sources/UI/SettingsVoiceIllustration.swift" + ], + "artifacts": { + "intent": "intent.md", + "spec": "spec.md", + "plan": "plan.md", + "verification": "verification.md" + } +} diff --git a/docs/sdlc/changes/2026-08-31-settings-semantic-background/verification.md b/docs/sdlc/changes/2026-08-31-settings-semantic-background/verification.md new file mode 100644 index 00000000..a174312c --- /dev/null +++ b/docs/sdlc/changes/2026-08-31-settings-semantic-background/verification.md @@ -0,0 +1,32 @@ +# Verification: Match the macOS settings background hierarchy + +## Evidence + +| Check | Result | Evidence | +|---|---|---| +| Source inspection | Pass | Both shared settings paints now use AppKit `underPageBackgroundColor`; no fixed RGB or new material layer was added. | +| Real-window dark inspection | Pass | The built 760 by 680 window was opened through the actual AppKit/SwiftUI settings path; Activity and the ANE model pane use the lighter page surface while grouped sections, warning copy, switches, and labels remain legible. | +| `swift test` | Pass | 592 XCTest cases passed, 10 environment-gated cases skipped, and the Swift Testing model-upgrade test passed. | +| `python3 scripts/sdlc.py validate --worktree` | Pass | Final standalone validation passed for 10 change bundles and all 29 changed governed paths after the high-risk bundle was independently verified. | +| `bash scripts/ci-basic-checks.sh` | Pass | Package, 13 SDLC harness tests, plist/localization parity, identifiers, vocabulary, resources, conflict markers, secrets, and symlink checks passed. | +| Final `bash scripts/build-app.sh` | Pass | Final no-hook app and DMG rebuilt; app and mounted-DMG signatures and DMG checksum passed. | +| `git diff --check` | Pass | No whitespace errors after final evidence updates. | + +## Acceptance criteria + +- Shared semantic under-page background — pass in both shared source seams. +- Dark real-window page hierarchy and contrast — pass on Activity and ANE model + content without layout changes. +- Regression and release-style checks — full tests, basic CI, final artifact, + signature, checksum, and diff checks pass. + +## Residual risk + +Only the current dark appearance was visually compared with the reported issue. +The chosen AppKit semantic color adapts in light and accessibility appearances, +but final human visual acceptance still applies. + +## Decision + +Verified with automated checks, a final release-style build, and real-window +dark-appearance inspection. No release or human approval is recorded. From 2f477b8c9119debdf451ffcc22a95f6c704b3ee5 Mon Sep 17 00:00:00 2001 From: idevlab Date: Mon, 31 Aug 2026 13:22:13 +0800 Subject: [PATCH 11/14] docs: record intent approval --- docs/sdlc/changes/2026-08-31-ane-lm-runtime/intent.md | 6 +++--- docs/sdlc/changes/2026-08-31-ane-lm-runtime/spec.md | 2 +- .../2026-08-31-settings-semantic-background/intent.md | 6 +++--- .../changes/2026-08-31-settings-semantic-background/spec.md | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/sdlc/changes/2026-08-31-ane-lm-runtime/intent.md b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/intent.md index 9061b5c7..827c9e63 100644 --- a/docs/sdlc/changes/2026-08-31-ane-lm-runtime/intent.md +++ b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/intent.md @@ -1,8 +1,8 @@ # Intent: Replace Espresso with a packaged ANE-LM runtime -**Status:** pending approval -**Approved-by:** — -**Approved-date:** — +**Status:** approved +**Approved-by:** IchenDEV (user) +**Approved-date:** 2026-08-31 **Upstream:** User request and PR #86 ## Problem diff --git a/docs/sdlc/changes/2026-08-31-ane-lm-runtime/spec.md b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/spec.md index 018b4108..e0dd8160 100644 --- a/docs/sdlc/changes/2026-08-31-ane-lm-runtime/spec.md +++ b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/spec.md @@ -1,6 +1,6 @@ # Spec: Replace Espresso with a packaged ANE-LM runtime -**Status:** draft +**Status:** pending approval **Approved-by:** — **Approved-date:** — **Upstream:** [intent.md](intent.md) diff --git a/docs/sdlc/changes/2026-08-31-settings-semantic-background/intent.md b/docs/sdlc/changes/2026-08-31-settings-semantic-background/intent.md index 99c323ca..cf53b93a 100644 --- a/docs/sdlc/changes/2026-08-31-settings-semantic-background/intent.md +++ b/docs/sdlc/changes/2026-08-31-settings-semantic-background/intent.md @@ -1,8 +1,8 @@ # Intent: Match the macOS settings background hierarchy -**Status:** pending approval -**Approved-by:** — -**Approved-date:** — +**Status:** approved +**Approved-by:** IchenDEV (user) +**Approved-date:** 2026-08-31 **Upstream:** User report and PR #86 ## Problem diff --git a/docs/sdlc/changes/2026-08-31-settings-semantic-background/spec.md b/docs/sdlc/changes/2026-08-31-settings-semantic-background/spec.md index e001e25c..6430e66c 100644 --- a/docs/sdlc/changes/2026-08-31-settings-semantic-background/spec.md +++ b/docs/sdlc/changes/2026-08-31-settings-semantic-background/spec.md @@ -1,6 +1,6 @@ # Spec: Match the macOS settings background hierarchy -**Status:** draft +**Status:** pending approval **Approved-by:** — **Approved-date:** — **Upstream:** [intent.md](intent.md) From 41158c22d1b2506021d6833d29e4684c576ffb0b Mon Sep 17 00:00:00 2001 From: idevlab Date: Mon, 31 Aug 2026 13:32:54 +0800 Subject: [PATCH 12/14] docs: record spec approval --- docs/sdlc/changes/2026-08-31-ane-lm-runtime/plan.md | 2 +- docs/sdlc/changes/2026-08-31-ane-lm-runtime/spec.md | 6 +++--- .../changes/2026-08-31-settings-semantic-background/plan.md | 2 +- .../changes/2026-08-31-settings-semantic-background/spec.md | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/sdlc/changes/2026-08-31-ane-lm-runtime/plan.md b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/plan.md index 030dc8b8..fd9f782f 100644 --- a/docs/sdlc/changes/2026-08-31-ane-lm-runtime/plan.md +++ b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/plan.md @@ -1,6 +1,6 @@ # Plan: Replace Espresso with a packaged ANE-LM runtime -**Status:** draft +**Status:** pending approval **Approved-by:** — **Approved-date:** — **Upstream:** [spec.md](spec.md) diff --git a/docs/sdlc/changes/2026-08-31-ane-lm-runtime/spec.md b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/spec.md index e0dd8160..5e032f23 100644 --- a/docs/sdlc/changes/2026-08-31-ane-lm-runtime/spec.md +++ b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/spec.md @@ -1,8 +1,8 @@ # Spec: Replace Espresso with a packaged ANE-LM runtime -**Status:** pending approval -**Approved-by:** — -**Approved-date:** — +**Status:** approved +**Approved-by:** IchenDEV (user) +**Approved-date:** 2026-08-31 **Upstream:** [intent.md](intent.md) ## Context diff --git a/docs/sdlc/changes/2026-08-31-settings-semantic-background/plan.md b/docs/sdlc/changes/2026-08-31-settings-semantic-background/plan.md index 24c55a43..55a94742 100644 --- a/docs/sdlc/changes/2026-08-31-settings-semantic-background/plan.md +++ b/docs/sdlc/changes/2026-08-31-settings-semantic-background/plan.md @@ -1,6 +1,6 @@ # Plan: Match the macOS settings background hierarchy -**Status:** draft +**Status:** pending approval **Approved-by:** — **Approved-date:** — **Upstream:** [spec.md](spec.md) diff --git a/docs/sdlc/changes/2026-08-31-settings-semantic-background/spec.md b/docs/sdlc/changes/2026-08-31-settings-semantic-background/spec.md index 6430e66c..cf4f5906 100644 --- a/docs/sdlc/changes/2026-08-31-settings-semantic-background/spec.md +++ b/docs/sdlc/changes/2026-08-31-settings-semantic-background/spec.md @@ -1,8 +1,8 @@ # Spec: Match the macOS settings background hierarchy -**Status:** pending approval -**Approved-by:** — -**Approved-date:** — +**Status:** approved +**Approved-by:** IchenDEV (user) +**Approved-date:** 2026-08-31 **Upstream:** [intent.md](intent.md) ## Context From 61d53a6c8080680dd91ea9b8a4a3311970fe0105 Mon Sep 17 00:00:00 2001 From: idevlab Date: Mon, 31 Aug 2026 13:34:23 +0800 Subject: [PATCH 13/14] docs: record plan approval --- docs/sdlc/changes/2026-08-31-ane-lm-runtime/plan.md | 6 +++--- docs/sdlc/changes/2026-08-31-ane-lm-runtime/verification.md | 2 +- .../changes/2026-08-31-settings-semantic-background/plan.md | 6 +++--- .../2026-08-31-settings-semantic-background/verification.md | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/sdlc/changes/2026-08-31-ane-lm-runtime/plan.md b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/plan.md index fd9f782f..c5b4e6ca 100644 --- a/docs/sdlc/changes/2026-08-31-ane-lm-runtime/plan.md +++ b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/plan.md @@ -1,8 +1,8 @@ # Plan: Replace Espresso with a packaged ANE-LM runtime -**Status:** pending approval -**Approved-by:** — -**Approved-date:** — +**Status:** approved +**Approved-by:** IchenDEV (user) +**Approved-date:** 2026-08-31 **Upstream:** [spec.md](spec.md) ## Work items diff --git a/docs/sdlc/changes/2026-08-31-ane-lm-runtime/verification.md b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/verification.md index ce9fe4cd..c478f8a1 100644 --- a/docs/sdlc/changes/2026-08-31-ane-lm-runtime/verification.md +++ b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/verification.md @@ -1,6 +1,6 @@ # Verification: Replace Espresso with a packaged ANE-LM runtime -**Status:** draft +**Status:** pending approval **Approved-by:** — **Approved-date:** — **Upstream:** [plan.md](plan.md) diff --git a/docs/sdlc/changes/2026-08-31-settings-semantic-background/plan.md b/docs/sdlc/changes/2026-08-31-settings-semantic-background/plan.md index 55a94742..8cb52ac0 100644 --- a/docs/sdlc/changes/2026-08-31-settings-semantic-background/plan.md +++ b/docs/sdlc/changes/2026-08-31-settings-semantic-background/plan.md @@ -1,8 +1,8 @@ # Plan: Match the macOS settings background hierarchy -**Status:** pending approval -**Approved-by:** — -**Approved-date:** — +**Status:** approved +**Approved-by:** IchenDEV (user) +**Approved-date:** 2026-08-31 **Upstream:** [spec.md](spec.md) ## Work items diff --git a/docs/sdlc/changes/2026-08-31-settings-semantic-background/verification.md b/docs/sdlc/changes/2026-08-31-settings-semantic-background/verification.md index 6731b5c2..9f5f36bf 100644 --- a/docs/sdlc/changes/2026-08-31-settings-semantic-background/verification.md +++ b/docs/sdlc/changes/2026-08-31-settings-semantic-background/verification.md @@ -1,6 +1,6 @@ # Verification: Match the macOS settings background hierarchy -**Status:** draft +**Status:** pending approval **Approved-by:** — **Approved-date:** — **Upstream:** [plan.md](plan.md) From 829d0a53ca208aa81cfc9f54d2651a2721b524c6 Mon Sep 17 00:00:00 2001 From: idevlab Date: Mon, 31 Aug 2026 13:38:54 +0800 Subject: [PATCH 14/14] docs: record verification approval --- docs/sdlc/changes/2026-08-31-ane-lm-runtime/verification.md | 6 +++--- .../2026-08-31-settings-semantic-background/verification.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/sdlc/changes/2026-08-31-ane-lm-runtime/verification.md b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/verification.md index c478f8a1..456c1513 100644 --- a/docs/sdlc/changes/2026-08-31-ane-lm-runtime/verification.md +++ b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/verification.md @@ -1,8 +1,8 @@ # Verification: Replace Espresso with a packaged ANE-LM runtime -**Status:** pending approval -**Approved-by:** — -**Approved-date:** — +**Status:** approved +**Approved-by:** IchenDEV (user) +**Approved-date:** 2026-08-31 **Upstream:** [plan.md](plan.md) ## Evidence diff --git a/docs/sdlc/changes/2026-08-31-settings-semantic-background/verification.md b/docs/sdlc/changes/2026-08-31-settings-semantic-background/verification.md index 9f5f36bf..5b22cd50 100644 --- a/docs/sdlc/changes/2026-08-31-settings-semantic-background/verification.md +++ b/docs/sdlc/changes/2026-08-31-settings-semantic-background/verification.md @@ -1,8 +1,8 @@ # Verification: Match the macOS settings background hierarchy -**Status:** pending approval -**Approved-by:** — -**Approved-date:** — +**Status:** approved +**Approved-by:** IchenDEV (user) +**Approved-date:** 2026-08-31 **Upstream:** [plan.md](plan.md) ## Evidence