diff --git a/Package.resolved b/Package.resolved index 7659258..d98e88c 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,6 +1,14 @@ { - "originHash" : "a272f2b5fa2a7a226c6e9fdcf766dbda0afb1e7004e1b2f15f107c99cd5b662b", + "originHash" : "3283c962479741ac4c7b0cd4e71ec15275b7d24d82a2182437d9e2ba4d21b27a", "pins" : [ + { + "identity" : "ane-lm", + "kind" : "remoteSourceControl", + "location" : "https://github.com/IchenDEV/ANE-LM.git", + "state" : { + "revision" : "033472ec12ea796fc7ea4f8cefd7ed456f69900b" + } + }, { "identity" : "argmax-oss-swift", "kind" : "remoteSourceControl", diff --git a/Package.swift b/Package.swift index 2bee492..550d0ee 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 6.0 +// swift-tools-version: 6.2 import PackageDescription let package = Package( @@ -13,6 +13,10 @@ let package = Package( ], dependencies: [ .package(url: "https://github.com/argmaxinc/argmax-oss-swift.git", from: "1.0.0"), + .package( + url: "https://github.com/IchenDEV/ANE-LM.git", + revision: "033472ec12ea796fc7ea4f8cefd7ed456f69900b" + ), .package(url: "https://github.com/Blaizzy/mlx-audio-swift.git", exact: "0.1.3"), .package(url: "https://github.com/huggingface/swift-transformers", from: "1.3.3"), .package(url: "https://github.com/ml-explore/mlx-swift-lm", exact: "3.31.4"), @@ -22,6 +26,7 @@ let package = Package( name: "OpenType", dependencies: [ .product(name: "WhisperKit", package: "argmax-oss-swift"), + .product(name: "ANELMRuntime", package: "ANE-LM"), .product(name: "MLXAudioCore", package: "mlx-audio-swift"), .product(name: "MLXAudioSTT", package: "mlx-audio-swift"), .product(name: "Hub", package: "swift-transformers"), diff --git a/Sources/App/AppDelegate+Integrations.swift b/Sources/App/AppDelegate+Integrations.swift index 0b1deaa..102a9fc 100644 --- a/Sources/App/AppDelegate+Integrations.swift +++ b/Sources/App/AppDelegate+Integrations.swift @@ -117,6 +117,7 @@ extension AppDelegate { func makeIntegrationSessionCoordinator(service: OpenTypeService) -> InputSessionCoordinator { InputSessionCoordinator( service: service, + textProcessor: textProcessor, isUserWorkflowBusy: { [weak self] in self?.appState.isBusy ?? false } diff --git a/Sources/App/AppState.swift b/Sources/App/AppState.swift index 5d335dc..f067ec3 100644 --- a/Sources/App/AppState.swift +++ b/Sources/App/AppState.swift @@ -13,6 +13,11 @@ enum AppPhase: Equatable { case error(String) } +enum AppCompletionKind: Equatable { + case standard + case espressoFallback +} + @MainActor final class AppState: ObservableObject { @Published var phase: AppPhase = .idle @@ -32,6 +37,7 @@ final class AppState: ObservableObject { @Published var lastFormattingDurationSeconds: Double = 0 @Published var pendingReplacement: DeferredReplacement? @Published var activeInputMode: VoiceInputMode = .dictation + @Published var completionKind: AppCompletionKind = .standard let settings = AppSettings.shared @@ -54,6 +60,7 @@ final class AppState: ObservableObject { resetDownloadProgress() pendingReplacement = nil activeInputMode = .dictation + completionKind = .standard } func clearPendingReplacement() { diff --git a/Sources/App/OpenTypeApp.swift b/Sources/App/OpenTypeApp.swift index 8931dde..05851be 100644 --- a/Sources/App/OpenTypeApp.swift +++ b/Sources/App/OpenTypeApp.swift @@ -24,6 +24,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject { private var settingsWindowDelegate: SettingsWindowDelegate? private var onboardingWindow: NSWindow? private let popoverOutsideClickMonitor = PopoverOutsideClickMonitor() + let textProcessor: TextProcessor var cancellables = Set() var iconTimer: Timer? private var previousApp: NSRunningApplication? @@ -39,6 +40,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject { let registry = IntegrationClientRegistry() integrationClientRegistry = registry integrationService = OpenTypeService(registry: registry) + textProcessor = TextProcessor() super.init() integrationSessionCoordinator = makeIntegrationSessionCoordinator(service: integrationService) } @@ -95,7 +97,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject { } private func setupPipeline() { - pipeline = VoicePipeline(appState: appState) + pipeline = VoicePipeline(appState: appState, textProcessor: textProcessor) Task { await pipeline?.warmUp() } } @@ -202,6 +204,10 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject { onUnloadWhisper: { [weak self] in self?.pipeline?.unloadWhisper() }, onUnloadLLM: { [weak self] in self?.pipeline?.unloadLLM() }, onLoadLLM: { [weak self] in self?.pipeline?.loadLLM() }, + onBenchmarkLLM: { [weak self] modelID in + guard let self else { throw CancellationError() } + return try await self.textProcessor.benchmarkLLM(modelID: modelID) + }, onUnloadLocalASR: { [weak self] in self?.pipeline?.unloadLocalASR() } ) .environmentObject(appState) diff --git a/Sources/App/VoicePipeline+EditCommandHelpers.swift b/Sources/App/VoicePipeline+EditCommandHelpers.swift new file mode 100644 index 0000000..1ee5394 --- /dev/null +++ b/Sources/App/VoicePipeline+EditCommandHelpers.swift @@ -0,0 +1,103 @@ +import AppKit +import Foundation + +@MainActor +extension VoicePipeline { + func replacementInputContext( + settings: AppSettings, + targetApp: NSRunningApplication? + ) -> InputContext { + InputContext.capture( + targetApp: targetApp, + screenContext: "", + outputMode: .command, + inputLanguage: settings.inputLanguage, + source: .menuBar + ) + } + + func finalizedReplacementText( + _ text: String, + settings: AppSettings + ) -> String { + textProcessor.cleanCommandGeneratedOutput( + text, + inputLanguage: settings.inputLanguage + ) + } + + func rewriteSelectedText( + raw: String, + intent: SelectionRewriteIntent, + settings: AppSettings, + targetApp: NSRunningApplication? + ) async { + cancelScreenContextCapture() + + guard let selectedText = await textInserter.selectedText(targetApp: targetApp), + !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + showErrorHint(L("pipeline.no_selected_text_to_replace")) + return + } + + let context = InputContext.capture( + targetApp: targetApp, + screenContext: "", + selectedTextOverride: selectedText, + outputMode: .command, + inputLanguage: settings.inputLanguage, + source: .menuBar + ) + var options = TextProcessingOptions(settings: settings) + options.llmModel = settings.llmModel + let memoryContext = VoicePipelinePolicy.memoryContext( + for: .command, + settings: settings, + currentContext: context + ) + + appState.phase = .processing + appState.statusMessage = L("pipeline.formatting") + let rewrittenText = await textProcessor.processSelectionEdit( + selectedText: selectedText, + intent: intent, + options: options, + spokenCommand: raw, + memoryContext: memoryContext, + inputContext: context + ) + guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + showNoSpeechDetected(reason: "selection rewrite returned empty text") + return + } + + appState.processedText = rewrittenText + appState.phase = .inserting + appState.statusMessage = L("pipeline.replacing") + + let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp) + appState.phase = .done + appState.statusMessage = L("status.done") + hideOverlayAfterDelay() + + if case .probablyFailed(let reason) = result { + Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)") + TextInserter.copyToClipboard(rewrittenText) + showInsertionFailedAlert(text: rewrittenText, reason: reason) + return + } + + let recordID = InputHistory.shared.addRecord( + rawText: raw, + processedText: rewrittenText, + wasProcessed: true, + context: context + ) + appState.lastInsertedText = rewrittenText + beginCorrectionCapture( + recordID: recordID, + insertedText: rewrittenText, + context: context + ) + } +} diff --git a/Sources/App/VoicePipeline+EditCommands.swift b/Sources/App/VoicePipeline+EditCommands.swift index 6babcba..b288aa9 100644 --- a/Sources/App/VoicePipeline+EditCommands.swift +++ b/Sources/App/VoicePipeline+EditCommands.swift @@ -8,6 +8,7 @@ extension VoicePipeline { settings: AppSettings, targetApp: NSRunningApplication? ) async -> Bool { + let expectedEspressoModelPath = settings.espressoModelPath guard let command = await resolvedSpokenEditCommand( raw: raw, settings: settings, @@ -24,7 +25,6 @@ extension VoicePipeline { settings: settings, targetApp: targetApp ) - return true case .replaceSelection(let replacementRaw): await replaceSelectedText( raw: raw, @@ -32,7 +32,6 @@ extension VoicePipeline { settings: settings, targetApp: targetApp ) - return true case .rewriteLast(let intent): await rewriteLastInsertion( raw: raw, @@ -40,7 +39,6 @@ extension VoicePipeline { settings: settings, targetApp: targetApp ) - return true case .rewriteSelection(let intent): await rewriteSelectedText( raw: raw, @@ -48,14 +46,29 @@ extension VoicePipeline { settings: settings, targetApp: targetApp ) - return true case .deleteSelection: await deleteSelectedText(targetApp: targetApp) - return true case .undoLastInsertion: await undoLastInsertion(targetApp: targetApp) - return true } + + guard !Task.isCancelled else { return true } + if let espressoOutcome = await consumeEspressoOutcome( + settings: settings, + expectedEspressoModelPath: expectedEspressoModelPath + ) { + if case .error = appState.phase, espressoOutcome == .fallback { + Log.info("[VoicePipeline] preserving edit-command error after ANE-LM fallback") + } else if espressoOutcome != .fallback { + showErrorHint(espressoOutcome.message) + } else { + appState.completionKind = .espressoFallback + appState.statusMessage = espressoOutcome.message + showOverlay() + hideOverlayAfterDelay() + } + } + return true } private func replaceLastInsertion( @@ -161,29 +174,6 @@ extension VoicePipeline { ) } - private func replacementInputContext( - settings: AppSettings, - targetApp: NSRunningApplication? - ) -> InputContext { - InputContext.capture( - targetApp: targetApp, - screenContext: "", - outputMode: .command, - inputLanguage: settings.inputLanguage, - source: .menuBar - ) - } - - private func finalizedReplacementText( - _ text: String, - settings: AppSettings - ) -> String { - textProcessor.cleanCommandGeneratedOutput( - text, - inputLanguage: settings.inputLanguage - ) - } - private func deleteSelectedText(targetApp: NSRunningApplication?) async { cancelScreenContextCapture() @@ -236,78 +226,4 @@ extension VoicePipeline { hideOverlayAfterDelay() } - private func rewriteSelectedText( - raw: String, - intent: SelectionRewriteIntent, - settings: AppSettings, - targetApp: NSRunningApplication? - ) async { - cancelScreenContextCapture() - - guard let selectedText = await textInserter.selectedText(targetApp: targetApp), - !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { - showErrorHint(L("pipeline.no_selected_text_to_replace")) - return - } - - let context = InputContext.capture( - targetApp: targetApp, - screenContext: "", - selectedTextOverride: selectedText, - outputMode: .command, - inputLanguage: settings.inputLanguage, - source: .menuBar - ) - var options = TextProcessingOptions(settings: settings) - options.llmModel = settings.llmModel - let memoryContext = VoicePipelinePolicy.memoryContext( - for: .command, - settings: settings, - currentContext: context - ) - - appState.phase = .processing - appState.statusMessage = L("pipeline.formatting") - let rewrittenText = await textProcessor.processSelectionEdit( - selectedText: selectedText, - intent: intent, - options: options, - spokenCommand: raw, - memoryContext: memoryContext, - inputContext: context - ) - guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { - showNoSpeechDetected(reason: "selection rewrite returned empty text") - return - } - - appState.processedText = rewrittenText - appState.phase = .inserting - appState.statusMessage = L("pipeline.replacing") - - let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp) - appState.phase = .done - appState.statusMessage = L("status.done") - hideOverlayAfterDelay() - - if case .probablyFailed(let reason) = result { - Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)") - TextInserter.copyToClipboard(rewrittenText) - showInsertionFailedAlert(text: rewrittenText, reason: reason) - return - } - - let recordID = InputHistory.shared.addRecord( - rawText: raw, - processedText: rewrittenText, - wasProcessed: true, - context: context - ) - appState.lastInsertedText = rewrittenText - beginCorrectionCapture( - recordID: recordID, - insertedText: rewrittenText, - context: context - ) - } } diff --git a/Sources/App/VoicePipeline+ModelLifecycle.swift b/Sources/App/VoicePipeline+ModelLifecycle.swift new file mode 100644 index 0000000..e5bed5f --- /dev/null +++ b/Sources/App/VoicePipeline+ModelLifecycle.swift @@ -0,0 +1,48 @@ +import Foundation + +@MainActor +extension VoicePipeline { + func unloadLLM() { + formattingPreloadGeneration += 1 + processingTask?.cancel() + processingTask = nil + replacementTask?.cancel() + replacementTask = nil + appState.clearPendingReplacement() + if appState.phase == .processing { + appState.phase = .idle + appState.statusMessage = L("status.ready") + } else if appState.statusMessage == L("pipeline.loading_llm") { + appState.statusMessage = L("status.ready") + } + appState.llmModelReady = false + + let precedingTask = formattingModelLifecycleTask + formattingModelLifecycleTask = Task { @MainActor [weak self] in + _ = await precedingTask?.value + guard let self else { return nil } + await self.textProcessor.unloadLLM() + return nil + } + } + + func loadLLM() { + enqueueFormattingModelPreload(showFailureInStatus: true) + } + + @discardableResult + func enqueueFormattingModelPreload( + showFailureInStatus: Bool + ) -> Task { + let precedingTask = formattingModelLifecycleTask + let task: Task = Task { @MainActor [weak self] in + _ = await precedingTask?.value + guard let self else { return nil } + return await self.preloadFormattingModelNow( + showFailureInStatus: showFailureInStatus + ) + } + formattingModelLifecycleTask = task + return task + } +} diff --git a/Sources/App/VoicePipeline+Models.swift b/Sources/App/VoicePipeline+Models.swift index dfb5a80..222a2d5 100644 --- a/Sources/App/VoicePipeline+Models.swift +++ b/Sources/App/VoicePipeline+Models.swift @@ -9,67 +9,128 @@ extension VoicePipeline { appState.statusMessage = L("pipeline.whisper_unloaded") } - func unloadLLM() { - processingTask?.cancel() - processingTask = nil - replacementTask?.cancel() - replacementTask = nil - appState.clearPendingReplacement() - if appState.phase == .processing { - appState.phase = .idle - appState.statusMessage = L("status.ready") - } - appState.llmModelReady = false - Task { await textProcessor.unloadLLM() } - } - func unloadLocalASR() { qwenSpeechEngine = nil } - func loadLLM() { - Task { - await preloadFormattingModel(showFailureInStatus: true) - } - } - - func preloadFormattingModel(showFailureInStatus: Bool) async { + @discardableResult + func preloadFormattingModelNow(showFailureInStatus: Bool) async -> EspressoGenerationOutcome? { + formattingPreloadGeneration += 1 + let preloadGeneration = formattingPreloadGeneration guard !appState.settings.useRemoteLLM else { appState.llmModelReady = true - return + return nil } - let model = appState.settings.llmModel.trimmingCharacters(in: .whitespacesAndNewlines) - guard !model.isEmpty else { return } + let settings = appState.settings + let backend = settings.localLLMBackend + let model = settings.llmModel.trimmingCharacters(in: .whitespacesAndNewlines) + let espressoPath = settings.espressoModelPath.trimmingCharacters(in: .whitespacesAndNewlines) + let selectedModel = backend == .espresso ? espressoPath : model + guard !selectedModel.isEmpty else { return nil } let catalog = ModelCatalog.shared - catalog.refreshStatus() - let modelStatus = catalog.llmModels.first(where: { $0.id == model })?.status - guard modelStatus == .downloaded || modelStatus == .ready else { + let modelIsAvailable: Bool + if backend == .espresso { + modelIsAvailable = FileManager.default.fileExists( + atPath: NSString(string: espressoPath).expandingTildeInPath + ) + } else { + catalog.refreshStatus() + let status = catalog.llmModels.first(where: { $0.id == model })?.status + modelIsAvailable = status == .downloaded || status == .ready + } + guard modelIsAvailable else { let message = L("model.download_required") - catalog.updateLLMStatus(model, status: .error(message)) + if backend == .mlx { + catalog.updateLLMStatus(model, status: .error(message)) + } appState.statusMessage = showFailureInStatus ? message : L("status.ready") - return + return nil } appState.statusMessage = L("pipeline.loading_llm") - catalog.updateLLMStatus(model, status: .loading, detail: L("model.loading")) + if backend == .mlx { + catalog.updateLLMStatus(model, status: .loading, detail: L("model.loading")) + } - let loaded = await textProcessor.warmUpLLM(model: model) - let ready = await textProcessor.isLLMReady - appState.llmModelReady = loaded && ready + let warmup = await textProcessor.warmUpLLM( + model: model, + backend: backend, + espressoModelPath: espressoPath, + fallbackToMLXOnEspressoFailure: settings.fallbackToMLXOnEspressoFailure + ) + guard !Task.isCancelled, + preloadGeneration == formattingPreloadGeneration, + formattingSelectionMatches(backend: backend, model: model, espressoPath: espressoPath) else { + return nil + } + let ready = warmup.loaded ? await textProcessor.isLLMReady(for: backend) : false + guard !Task.isCancelled, + preloadGeneration == formattingPreloadGeneration, + formattingSelectionMatches(backend: backend, model: model, espressoPath: espressoPath) else { + return nil + } + appState.llmModelReady = warmup.loaded && ready if appState.llmModelReady { - catalog.updateLLMStatus(model, status: .ready) + EspressoFallbackPolicy.selectMLXIfNeeded( + after: warmup.espressoOutcome, + settings: settings, + expectedEspressoModelPath: espressoPath + ) + if backend == .mlx { + catalog.updateLLMStatus(model, status: .ready) + } Log.info("[VoicePipeline] LLM model loaded into memory, ready for instant inference") - appState.statusMessage = L("status.ready") + appState.statusMessage = warmup.espressoOutcome?.message ?? L("status.ready") + presentEspressoWarmupOutcomeIfNeeded(warmup.espressoOutcome) + return warmup.espressoOutcome } else { - catalog.updateLLMStatus(model, status: .error(L("pipeline.model_load_failed"))) + if backend == .mlx { + catalog.updateLLMStatus(model, status: .error(L("pipeline.model_load_failed"))) + } Log.info("[VoicePipeline] LLM warmup failed, will retry on demand") - appState.statusMessage = showFailureInStatus ? L("pipeline.model_load_failed") : L("status.ready") + let message = backend == .espresso + ? (warmup.errorMessage ?? L("error.espresso_runtime_failed")) + : L("pipeline.model_load_failed") + appState.statusMessage = warmup.espressoOutcome != nil || showFailureInStatus + ? message + : L("status.ready") + presentEspressoWarmupOutcomeIfNeeded(warmup.espressoOutcome) + return warmup.espressoOutcome + } + } + + private func formattingSelectionMatches( + backend: LocalLLMBackend, + model: String, + espressoPath: String + ) -> Bool { + let settings = appState.settings + guard !settings.useRemoteLLM, settings.localLLMBackend == backend else { return false } + switch backend { + case .mlx: + return settings.llmModel.trimmingCharacters(in: .whitespacesAndNewlines) == model + case .espresso: + return settings.espressoModelPath.trimmingCharacters(in: .whitespacesAndNewlines) == espressoPath } } + func consumeEspressoOutcome( + settings: AppSettings, + expectedEspressoModelPath: String + ) async -> EspressoGenerationOutcome? { + let outcome = await textProcessor.consumeEspressoOutcome() + guard !Task.isCancelled else { return nil } + EspressoFallbackPolicy.selectMLXIfNeeded( + after: outcome, + settings: settings, + expectedEspressoModelPath: expectedEspressoModelPath + ) + return outcome + } + func ensureEngineLoaded(requestPermission: Bool = true) async { switch appState.settings.speechEngine { case .whisper: diff --git a/Sources/App/VoicePipeline+Processing.swift b/Sources/App/VoicePipeline+Processing.swift index 1e76715..0eb68de 100644 --- a/Sources/App/VoicePipeline+Processing.swift +++ b/Sources/App/VoicePipeline+Processing.swift @@ -49,13 +49,13 @@ extension VoicePipeline { } } + let expectedEspressoModelPath = settings.espressoModelPath let output = await outputText( for: preparedRaw, settings: settings, inputMode: inputMode, targetApp: targetApp ) - guard !Task.isCancelled else { resetToIdle() return @@ -65,6 +65,7 @@ extension VoicePipeline { output, raw: preparedRaw, settings: settings, + expectedEspressoModelPath: expectedEspressoModelPath, inputMode: inputMode, targetApp: targetApp ) @@ -237,80 +238,4 @@ extension VoicePipeline { return VoicePipelineOutput(text: text, context: inputContext) } - func recordFormattingDuration(_ started: CFAbsoluteTime, label: String) { - let elapsed = CFAbsoluteTimeGetCurrent() - started - appState.lastFormattingDurationSeconds = elapsed - Log.info("[VoicePipeline] \(label) completed in \(String(format: "%.2f", elapsed))s") - } - - private func insertFinalText( - _ output: VoicePipelineOutput, - raw: String, - settings: AppSettings, - inputMode: VoiceInputMode, - targetApp: NSRunningApplication? - ) async { - let finalText = output.text - guard !finalText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { - Log.info("[VoicePipeline] skipping empty final text") - showErrorHint(L("error.operation_failed")) - return - } - - appState.processedText = finalText - appState.phase = .inserting - appState.statusMessage = L("pipeline.inserting") - - Log.sensitive("[VoicePipeline] inserting \(finalText.count) chars") - let started = CFAbsoluteTimeGetCurrent() - let result = await textInserter.insert(text: finalText, targetApp: targetApp) - let elapsed = CFAbsoluteTimeGetCurrent() - started - Log.info("[VoicePipeline] insert stage finished in \(String(format: "%.2f", elapsed))s") - - appState.phase = .done - appState.statusMessage = L("status.done") - hideOverlayAfterDelay() - - if case .probablyFailed(let reason) = result { - Log.info("[VoicePipeline] insertion probably failed: \(reason)") - TextInserter.copyToClipboard(finalText) - showInsertionFailedAlert(text: finalText, reason: reason) - return - } - - let wasProcessed = inputMode.isTranslation - || settings.outputMode == .processed - || settings.outputMode == .command - let recordID = InputHistory.shared.addRecord( - rawText: raw, - processedText: finalText, - wasProcessed: wasProcessed, - context: output.context, - formatKind: output.formatKind - ) - appState.lastInsertedText = finalText - if !inputMode.isTranslation { - beginCorrectionCapture( - recordID: recordID, - insertedText: finalText, - context: output.context - ) - } - } -} - -struct VoicePipelineOutput { - let text: String - let context: InputContext - let formatKind: TextFormatKind? - - init(text: String, context: InputContext, formatKind: TextFormatKind? = nil) { - self.text = text - self.context = context - self.formatKind = formatKind - } -} - -private enum VoicePipelineStop: Error { - case noSpeech } diff --git a/Sources/App/VoicePipeline+Replacement.swift b/Sources/App/VoicePipeline+Replacement.swift index 418a9cf..3079089 100644 --- a/Sources/App/VoicePipeline+Replacement.swift +++ b/Sources/App/VoicePipeline+Replacement.swift @@ -241,11 +241,16 @@ extension VoicePipeline { guard !Task.isCancelled else { return } guard var replacement = appState.pendingReplacement, replacement.id == replacementID else { return } + let espressoOutcome = await consumeEspressoOutcome( + settings: appState.settings, + expectedEspressoModelPath: processingOptions.espressoModelPath + ) + guard !Task.isCancelled else { return } guard !formattedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { Log.info("[VoicePipeline] deferred Smart Format produced no LLM output") replacement.state = .failed - replacement.message = L("pipeline.formatting_failed") + replacement.message = espressoOutcome?.message ?? L("pipeline.formatting_failed") replacement.context = inputContext appState.pendingReplacement = replacement return @@ -253,7 +258,7 @@ extension VoicePipeline { replacement.formattedText = formattedText replacement.state = .ready - replacement.message = L("pipeline.formatted_ready") + replacement.message = espressoOutcome?.message ?? L("pipeline.formatted_ready") replacement.context = inputContext appState.pendingReplacement = replacement } @@ -277,36 +282,4 @@ extension VoicePipeline { ) } - private func immediateInsertText( - from raw: String, - inputLanguage: InputLanguage, - dictionarySnapshot: PersonalDictionarySnapshot - ) -> String { - let cleaned = textProcessor.prepareForFormatting( - text: raw, - inputLanguage: inputLanguage, - dictionarySnapshot: dictionarySnapshot - ) - let fallback = textProcessor.basicClean( - text: raw, - inputLanguage: inputLanguage, - dictionarySnapshot: dictionarySnapshot - ) - if !cleaned.isEmpty { return cleaned } - if !fallback.isEmpty { return fallback } - return "" - } - - private func replacementCopyMessage(for reason: DeferredReplacementCopyReason) -> String { - switch reason { - case .expired: - return L("pipeline.replacement_copied_expired") - case .missingTarget: - return L("pipeline.replacement_copied_missing_target") - case .appChanged: - return L("pipeline.replacement_copied_app_changed") - case .notReady: - return L("pipeline.replacement_not_ready") - } - } } diff --git a/Sources/App/VoicePipeline+ReplacementHelpers.swift b/Sources/App/VoicePipeline+ReplacementHelpers.swift new file mode 100644 index 0000000..dccac86 --- /dev/null +++ b/Sources/App/VoicePipeline+ReplacementHelpers.swift @@ -0,0 +1,37 @@ +import Foundation + +@MainActor +extension VoicePipeline { + func immediateInsertText( + from raw: String, + inputLanguage: InputLanguage, + dictionarySnapshot: PersonalDictionarySnapshot + ) -> String { + let cleaned = textProcessor.prepareForFormatting( + text: raw, + inputLanguage: inputLanguage, + dictionarySnapshot: dictionarySnapshot + ) + let fallback = textProcessor.basicClean( + text: raw, + inputLanguage: inputLanguage, + dictionarySnapshot: dictionarySnapshot + ) + if !cleaned.isEmpty { return cleaned } + if !fallback.isEmpty { return fallback } + return "" + } + + func replacementCopyMessage(for reason: DeferredReplacementCopyReason) -> String { + switch reason { + case .expired: + return L("pipeline.replacement_copied_expired") + case .missingTarget: + return L("pipeline.replacement_copied_missing_target") + case .appChanged: + return L("pipeline.replacement_copied_app_changed") + case .notReady: + return L("pipeline.replacement_not_ready") + } + } +} diff --git a/Sources/App/VoicePipeline+ScreenContext.swift b/Sources/App/VoicePipeline+ScreenContext.swift index 55f9cbe..276b2c8 100644 --- a/Sources/App/VoicePipeline+ScreenContext.swift +++ b/Sources/App/VoicePipeline+ScreenContext.swift @@ -15,7 +15,8 @@ extension VoicePipeline { screenOCRStartedAt = CFAbsoluteTimeGetCurrent() let mode = ScreenContextMode.effectiveCaptureMode( preference: appState.settings.screenContextMode, - useRemoteLLM: appState.settings.useRemoteLLM, + useRemoteLLM: appState.settings.useRemoteLLM + || appState.settings.localLLMBackend == .espresso, modelID: appState.settings.llmModel ) screenOCRTask = Task.detached(priority: .utility) { diff --git a/Sources/App/VoicePipeline+Status.swift b/Sources/App/VoicePipeline+Status.swift index ad23b7c..fb7a37d 100644 --- a/Sources/App/VoicePipeline+Status.swift +++ b/Sources/App/VoicePipeline+Status.swift @@ -38,6 +38,12 @@ extension VoicePipeline { appState.statusMessage = L("status.ready") } + func recordFormattingDuration(_ started: CFAbsoluteTime, label: String) { + let elapsed = CFAbsoluteTimeGetCurrent() - started + appState.lastFormattingDurationSeconds = elapsed + Log.info("[VoicePipeline] \(label) completed in \(String(format: "%.2f", elapsed))s") + } + func hideOverlayAfterDelay() { hideOverlayTask?.cancel() hideOverlayTask = Task { @MainActor in @@ -69,6 +75,19 @@ extension VoicePipeline { hideOverlayAfterDelay() } + func presentEspressoWarmupOutcomeIfNeeded(_ outcome: EspressoGenerationOutcome?) { + guard let outcome else { return } + if outcome != .fallback { + showErrorHint(outcome.message) + return + } + appState.phase = .done + appState.completionKind = .espressoFallback + appState.statusMessage = outcome.message + showOverlay() + hideOverlayAfterDelay() + } + func showInsertionFailedAlert(text: String, reason: String) { let alert = NSAlert() alert.messageText = L("pipeline.insert_failed_title") diff --git a/Sources/App/VoicePipeline.swift b/Sources/App/VoicePipeline.swift index 96310fa..1d0f811 100644 --- a/Sources/App/VoicePipeline.swift +++ b/Sources/App/VoicePipeline.swift @@ -8,7 +8,7 @@ final class VoicePipeline { let audioCapture = AudioCaptureManager() let textInserter = TextInserter() let correctionCapture = CorrectionCaptureService() - let textProcessor = TextProcessor() + let textProcessor: TextProcessor let overlay = OverlayPanel() var whisperEngine: WhisperEngine? var appleSpeechEngine: AppleSpeechEngine? @@ -19,7 +19,9 @@ final class VoicePipeline { var processingTask: Task? var replacementTask: Task? var hideOverlayTask: Task? + var formattingModelLifecycleTask: Task? var recordingTargetApp: NSRunningApplication? + var formattingPreloadGeneration = 0 var currentEngine: (any SpeechEngine)? { switch appState.settings.speechEngine { @@ -31,8 +33,9 @@ final class VoicePipeline { } } - init(appState: AppState) { + init(appState: AppState, textProcessor: TextProcessor = TextProcessor()) { self.appState = appState + self.textProcessor = textProcessor } func warmUp() async { @@ -40,6 +43,12 @@ final class VoicePipeline { let catalog = ModelCatalog.shared catalog.refreshStatus(recheckingErrors: true) let llmStatus = catalog.llmModels.first(where: { $0.id == settings.llmModel })?.status + let formattingModelID = settings.localLLMBackend == .espresso + ? settings.espressoModelPath + : settings.llmModel + let formattingModelAvailable = settings.localLLMBackend == .espresso + ? FileManager.default.fileExists(atPath: NSString(string: settings.espressoModelPath).expandingTildeInPath) + : (llmStatus == .downloaded || llmStatus == .ready) let shouldLoadSpeech = StartupModelPreloadPolicy.shouldPreloadSpeechModel( enabled: settings.preloadSpeechModelOnLaunch, speechEngine: settings.speechEngine, @@ -48,8 +57,8 @@ final class VoicePipeline { let shouldLoadFormatting = StartupModelPreloadPolicy.shouldPreloadFormattingModel( enabled: settings.preloadFormattingModelOnLaunch, useRemoteLLM: settings.useRemoteLLM, - modelID: settings.llmModel, - modelDownloaded: llmStatus == .downloaded || llmStatus == .ready + modelID: formattingModelID, + modelDownloaded: formattingModelAvailable ) if shouldLoadSpeech { @@ -57,7 +66,12 @@ final class VoicePipeline { } if shouldLoadFormatting { - await preloadFormattingModel(showFailureInStatus: false) + let espressoOutcome = await enqueueFormattingModelPreload( + showFailureInStatus: false + ).value + if espressoOutcome != nil { + return + } } markReadyIfPossible() @@ -181,14 +195,16 @@ final class VoicePipeline { processingTask = Task { @MainActor [weak self] in guard let self else { return } - await self.processRecording( - audioURL: audioURL, - audioActivity: audioActivity, - language: language, - settings: settings, - inputMode: inputMode, - targetApp: resolvedTargetApp - ) + await TextProcessor.withEspressoOutcomeTracking { + await self.processRecording( + audioURL: audioURL, + audioActivity: audioActivity, + language: language, + settings: settings, + inputMode: inputMode, + targetApp: resolvedTargetApp + ) + } } } diff --git a/Sources/App/VoicePipelineOutput.swift b/Sources/App/VoicePipelineOutput.swift new file mode 100644 index 0000000..3730a9b --- /dev/null +++ b/Sources/App/VoicePipelineOutput.swift @@ -0,0 +1,82 @@ +import AppKit +import Foundation + +struct VoicePipelineOutput { + let text: String + let context: InputContext + let formatKind: TextFormatKind? + + init(text: String, context: InputContext, formatKind: TextFormatKind? = nil) { + self.text = text + self.context = context + self.formatKind = formatKind + } +} + +enum VoicePipelineStop: Error { + case noSpeech +} + +@MainActor +extension VoicePipeline { + func insertFinalText( + _ output: VoicePipelineOutput, + raw: String, + settings: AppSettings, + expectedEspressoModelPath: String, + inputMode: VoiceInputMode, + targetApp: NSRunningApplication? + ) async { + let finalText = output.text + let espressoOutcome = await consumeEspressoOutcome( + settings: settings, + expectedEspressoModelPath: expectedEspressoModelPath + ) + guard !finalText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + Log.info("[VoicePipeline] skipping empty final text") + showErrorHint(espressoOutcome?.message ?? L("error.operation_failed")) + return + } + + appState.processedText = finalText + appState.phase = .inserting + appState.statusMessage = L("pipeline.inserting") + + Log.sensitive("[VoicePipeline] inserting \(finalText.count) chars") + let started = CFAbsoluteTimeGetCurrent() + let result = await textInserter.insert(text: finalText, targetApp: targetApp) + let elapsed = CFAbsoluteTimeGetCurrent() - started + Log.info("[VoicePipeline] insert stage finished in \(String(format: "%.2f", elapsed))s") + + appState.phase = .done + appState.completionKind = espressoOutcome == .fallback ? .espressoFallback : .standard + appState.statusMessage = espressoOutcome?.message ?? L("status.done") + hideOverlayAfterDelay() + + if case .probablyFailed(let reason) = result { + Log.info("[VoicePipeline] insertion probably failed: \(reason)") + TextInserter.copyToClipboard(finalText) + showInsertionFailedAlert(text: finalText, reason: reason) + return + } + + let wasProcessed = inputMode.isTranslation + || settings.outputMode == .processed + || settings.outputMode == .command + let recordID = InputHistory.shared.addRecord( + rawText: raw, + processedText: finalText, + wasProcessed: wasProcessed, + context: output.context, + formatKind: output.formatKind + ) + appState.lastInsertedText = finalText + if !inputMode.isTranslation { + beginCorrectionCapture( + recordID: recordID, + insertedText: finalText, + context: output.context + ) + } + } +} diff --git a/Sources/Config/AppSettingTypes.swift b/Sources/Config/AppSettingTypes.swift new file mode 100644 index 0000000..42dbde1 --- /dev/null +++ b/Sources/Config/AppSettingTypes.swift @@ -0,0 +1,242 @@ +import Foundation + +enum UILanguage: String, Codable, CaseIterable { + case chinese = "zh" + case english = "en" + + var displayName: String { + switch self { + case .chinese: return "中文" + case .english: return "English" + } + } +} + +enum OutputMode: String, Codable, CaseIterable { + case direct = "direct" + case processed = "processed" + case command = "command" + + var label: String { + switch self { + case .direct: return L("mode.verbatim") + case .processed: return L("mode.smart_format") + case .command: return L("mode.voice_command") + } + } +} + +enum SpeechEngineType: String, Codable, CaseIterable { + case whisper = "whisper" + case apple = "apple" + case volc = "volc" + case qwen3 = "qwen3" + case mimo = "mimo" + + static var selectableCases: [SpeechEngineType] { + [.qwen3, .whisper, .apple, .volc] + } + + var label: String { + switch self { + case .whisper: return "WhisperKit" + case .apple: return L("engine.apple_speech") + case .volc: return L("engine.volc_asr") + case .qwen3: return L("engine.qwen3_asr") + case .mimo: return L("engine.mimo_asr") + } + } +} + +enum LocalLLMBackend: String, Codable, CaseIterable { + case mlx + case espresso +} + +enum LanguageStyle: String, Codable, CaseIterable { + case casual = "casual" + case professional = "professional" + case custom = "custom" + + var label: String { + switch self { + case .casual: return L("style.casual") + case .professional: return L("style.professional") + case .custom: return L("style.custom") + } + } + + var defaultPrompt: String { + switch self { + case .casual: return L("style.prompt.casual") + case .professional: return L("style.prompt.professional") + case .custom: return L("style.prompt.custom") + } + } + + var icon: String { + switch self { + case .casual: return "bubble.left" + case .professional: return "list.number" + case .custom: return "slider.horizontal.3" + } + } + + var usesCustomPrompt: Bool { self == .custom } + + static func migrated(from savedValue: String) -> LanguageStyle { + if let style = LanguageStyle(rawValue: savedValue) { + return style + } + + let normalized = savedValue.lowercased() + if normalized.contains("casual") || savedValue.contains("口语") { + return .casual + } + if normalized.contains("custom") || savedValue.contains("自定义") { + return .custom + } + if normalized.contains("professional") + || normalized.contains("formal") + || normalized.contains("concise") + || savedValue.contains("专业") + || savedValue.contains("正式") + || savedValue.contains("简洁") { + return .professional + } + return .professional + } + + static func looksLikePresetPrompt(_ prompt: String) -> Bool { + let normalized = prompt.trimmingCharacters(in: .whitespacesAndNewlines) + let prompts = [ + L("style.prompt.casual"), + L("style.prompt.professional"), + L("style.prompt.concise"), + L("style.prompt.formal"), + ].map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + + return prompts.contains(normalized) + } +} + +enum HotkeyType: String, Codable, CaseIterable { + case ctrl = "Ctrl" + case shift = "Shift" + case option = "Option" + case fn = "Fn" +} + +enum ActivationMode: String, Codable, CaseIterable { + case longPress = "longPress" + case doubleTap = "doubleTap" + case toggle = "toggle" + + var label: String { + switch self { + case .longPress: return L("mode.hold_record") + case .doubleTap: return L("mode.double_tap") + case .toggle: return L("mode.tap_toggle") + } + } +} + +enum HistoryRetention: String, Codable, CaseIterable { + case forever = "forever" + case threeDays = "threeDays" + case sevenDays = "sevenDays" + case oneMonth = "oneMonth" + + var label: String { + switch self { + case .forever: return L("retention.forever") + case .threeDays: return L("retention.three_days") + case .sevenDays: return L("retention.seven_days") + case .oneMonth: return L("retention.one_month") + } + } + + var timeInterval: TimeInterval? { + switch self { + case .forever: return nil + case .threeDays: return 3 * 24 * 3600 + case .sevenDays: return 7 * 24 * 3600 + case .oneMonth: return 30 * 24 * 3600 + } + } +} + +enum MenuBarIcon: String, Codable, CaseIterable { + case mic = "mic" + case waveform = "waveform" + case bubble = "bubble" + + var symbolName: String { + switch self { + case .mic: return "mic.fill" + case .waveform: return "waveform" + case .bubble: return "bubble.left.fill" + } + } + + var label: String { + switch self { + case .mic: return L("icon.mic") + case .waveform: return L("icon.waveform") + case .bubble: return L("icon.bubble") + } + } +} + +enum AppIconAppearance: String, Codable, CaseIterable { + case system = "system" + case dark = "dark" + case light = "light" + + func resourceName(systemIsDark: Bool) -> String { + switch self { + case .system: return systemIsDark ? "AppIconDark" : "AppIconLight" + case .dark: return "AppIconDark" + case .light: return "AppIconLight" + } + } + + var label: String { + switch self { + case .system: return L("app_icon.system") + case .dark: return L("app_icon.dark") + case .light: return L("app_icon.light") + } + } +} + +enum InputLanguage: String, Codable, CaseIterable { + case auto = "Auto" + case chinese = "中文" + case english = "English" + case japanese = "日本語" + case korean = "한국어" + case cantonese = "粤语" + + var whisperCode: String? { + switch self { + case .auto: return nil + case .chinese: return "zh" + case .english: return "en" + case .japanese: return "ja" + case .korean: return "ko" + case .cantonese: return "yue" + } + } + + var localeIdentifier: String { + switch self { + case .auto: return Locale.current.identifier + case .chinese: return "zh-CN" + case .english: return "en-US" + case .japanese: return "ja-JP" + case .korean: return "ko-KR" + case .cantonese: return "zh-HK" + } + } +} diff --git a/Sources/Config/AppSettings.swift b/Sources/Config/AppSettings.swift index 691d759..4e218b9 100644 --- a/Sources/Config/AppSettings.swift +++ b/Sources/Config/AppSettings.swift @@ -2,242 +2,6 @@ import Foundation import Combine import Security -enum UILanguage: String, Codable, CaseIterable { - case chinese = "zh" - case english = "en" - - var displayName: String { - switch self { - case .chinese: return "中文" - case .english: return "English" - } - } -} - -enum OutputMode: String, Codable, CaseIterable { - case direct = "direct" - case processed = "processed" - case command = "command" - - var label: String { - switch self { - case .direct: return L("mode.verbatim") - case .processed: return L("mode.smart_format") - case .command: return L("mode.voice_command") - } - } -} - -enum SpeechEngineType: String, Codable, CaseIterable { - case whisper = "whisper" - case apple = "apple" - case volc = "volc" - case qwen3 = "qwen3" - case mimo = "mimo" - - static var selectableCases: [SpeechEngineType] { - [.qwen3, .whisper, .apple, .volc] - } - - var label: String { - switch self { - case .whisper: return "WhisperKit" - case .apple: return L("engine.apple_speech") - case .volc: return L("engine.volc_asr") - case .qwen3: return L("engine.qwen3_asr") - case .mimo: return L("engine.mimo_asr") - } - } -} - -enum LanguageStyle: String, Codable, CaseIterable { - case casual = "casual" - case professional = "professional" - case custom = "custom" - - var label: String { - switch self { - case .casual: return L("style.casual") - case .professional: return L("style.professional") - case .custom: return L("style.custom") - } - } - - var defaultPrompt: String { - switch self { - case .casual: return L("style.prompt.casual") - case .professional: return L("style.prompt.professional") - case .custom: return L("style.prompt.custom") - } - } - - var icon: String { - switch self { - case .casual: return "bubble.left" - case .professional: return "list.number" - case .custom: return "slider.horizontal.3" - } - } - - var usesCustomPrompt: Bool { self == .custom } - - static func migrated(from savedValue: String) -> LanguageStyle { - if let style = LanguageStyle(rawValue: savedValue) { - return style - } - - let normalized = savedValue.lowercased() - if normalized.contains("casual") || savedValue.contains("口语") { - return .casual - } - if normalized.contains("custom") || savedValue.contains("自定义") { - return .custom - } - if normalized.contains("professional") - || normalized.contains("formal") - || normalized.contains("concise") - || savedValue.contains("专业") - || savedValue.contains("正式") - || savedValue.contains("简洁") { - return .professional - } - return .professional - } - - static func looksLikePresetPrompt(_ prompt: String) -> Bool { - let normalized = prompt.trimmingCharacters(in: .whitespacesAndNewlines) - let prompts = [ - L("style.prompt.casual"), - L("style.prompt.professional"), - L("style.prompt.concise"), - L("style.prompt.formal"), - ].map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } - - return prompts.contains(normalized) - } -} - -enum HotkeyType: String, Codable, CaseIterable { - case ctrl = "Ctrl" - case shift = "Shift" - case option = "Option" - case fn = "Fn" -} - -enum ActivationMode: String, Codable, CaseIterable { - case longPress = "longPress" - case doubleTap = "doubleTap" - case toggle = "toggle" - - var label: String { - switch self { - case .longPress: return L("mode.hold_record") - case .doubleTap: return L("mode.double_tap") - case .toggle: return L("mode.tap_toggle") - } - } -} - -enum HistoryRetention: String, Codable, CaseIterable { - case forever = "forever" - case threeDays = "threeDays" - case sevenDays = "sevenDays" - case oneMonth = "oneMonth" - - var label: String { - switch self { - case .forever: return L("retention.forever") - case .threeDays: return L("retention.three_days") - case .sevenDays: return L("retention.seven_days") - case .oneMonth: return L("retention.one_month") - } - } - - var timeInterval: TimeInterval? { - switch self { - case .forever: return nil - case .threeDays: return 3 * 24 * 3600 - case .sevenDays: return 7 * 24 * 3600 - case .oneMonth: return 30 * 24 * 3600 - } - } -} - -enum MenuBarIcon: String, Codable, CaseIterable { - case mic = "mic" - case waveform = "waveform" - case bubble = "bubble" - - var symbolName: String { - switch self { - case .mic: return "mic.fill" - case .waveform: return "waveform" - case .bubble: return "bubble.left.fill" - } - } - - var label: String { - switch self { - case .mic: return L("icon.mic") - case .waveform: return L("icon.waveform") - case .bubble: return L("icon.bubble") - } - } -} - -enum AppIconAppearance: String, Codable, CaseIterable { - case system = "system" - case dark = "dark" - case light = "light" - - func resourceName(systemIsDark: Bool) -> String { - switch self { - case .system: return systemIsDark ? "AppIconDark" : "AppIconLight" - case .dark: return "AppIconDark" - case .light: return "AppIconLight" - } - } - - var label: String { - switch self { - case .system: return L("app_icon.system") - case .dark: return L("app_icon.dark") - case .light: return L("app_icon.light") - } - } -} - -enum InputLanguage: String, Codable, CaseIterable { - case auto = "Auto" - case chinese = "中文" - case english = "English" - case japanese = "日本語" - case korean = "한국어" - case cantonese = "粤语" - - var whisperCode: String? { - switch self { - case .auto: return nil - case .chinese: return "zh" - case .english: return "en" - case .japanese: return "ja" - case .korean: return "ko" - case .cantonese: return "yue" - } - } - - var localeIdentifier: String { - switch self { - case .auto: return Locale.current.identifier - case .chinese: return "zh-CN" - case .english: return "en-US" - case .japanese: return "ja-JP" - case .korean: return "ko-KR" - case .cantonese: return "zh-HK" - } - } -} - final class AppSettings: ObservableObject { static let shared = AppSettings() static let defaultLLMModelID = "mlx-community/Qwen3.5-2B-4bit" @@ -284,6 +48,9 @@ final class AppSettings: ObservableObject { @Published var useCustomSystemPrompt: Bool @Published var customSystemPrompt: String @Published var useRemoteLLM: Bool + @Published var localLLMBackend: LocalLLMBackend + @Published var espressoModelPath: String + @Published var fallbackToMLXOnEspressoFailure: Bool @Published var remoteProvider: RemoteProvider @Published var remoteAPIKey: String @Published var remoteBaseURL: String @@ -314,7 +81,8 @@ final class AppSettings: ObservableObject { case useScreenContext, screenContextMode, enableInstantInsert, hasCompletedOnboarding, uiLanguage, historyRetention case enableMemory, memoryWindowMinutes, enableCorrectionLearning, industryLexicon case useCustomSystemPrompt, customSystemPrompt - case useRemoteLLM, remoteProvider, remoteAPIKey, remoteBaseURL, remoteModel + case useRemoteLLM, localLLMBackend, espressoModelPath, fallbackToMLXOnEspressoFailure + case remoteProvider, remoteAPIKey, remoteBaseURL, remoteModel case menuBarIcon, appIconAppearance case volcAppKey, volcAccessKey, volcResourceId case qwenASRModel, qwenASRModelPath @@ -392,6 +160,13 @@ final class AppSettings: ObservableObject { useCustomSystemPrompt = ud.bool(forKey: Key.useCustomSystemPrompt.rawValue) customSystemPrompt = ud.string(forKey: Key.customSystemPrompt.rawValue) ?? "" useRemoteLLM = ud.bool(forKey: Key.useRemoteLLM.rawValue) + localLLMBackend = LocalLLMBackend( + rawValue: ud.string(forKey: Key.localLLMBackend.rawValue) ?? "" + ) ?? .mlx + espressoModelPath = ud.string(forKey: Key.espressoModelPath.rawValue) ?? "" + fallbackToMLXOnEspressoFailure = ud.object( + forKey: Key.fallbackToMLXOnEspressoFailure.rawValue + ) as? Bool ?? true remoteProvider = RemoteProvider(rawValue: ud.string(forKey: Key.remoteProvider.rawValue) ?? "") ?? .custom remoteAPIKey = ud.string(forKey: Key.remoteAPIKey.rawValue) ?? "" remoteBaseURL = ud.string(forKey: Key.remoteBaseURL.rawValue) ?? "" @@ -459,6 +234,15 @@ final class AppSettings: ObservableObject { $useCustomSystemPrompt.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.useCustomSystemPrompt.rawValue) }.store(in: &cancellables) $customSystemPrompt.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.customSystemPrompt.rawValue) }.store(in: &cancellables) $useRemoteLLM.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.useRemoteLLM.rawValue) }.store(in: &cancellables) + $localLLMBackend.dropFirst().sink { + [defaults] in defaults.set($0.rawValue, forKey: Key.localLLMBackend.rawValue) + }.store(in: &cancellables) + $espressoModelPath.dropFirst().sink { + [defaults] in defaults.set($0, forKey: Key.espressoModelPath.rawValue) + }.store(in: &cancellables) + $fallbackToMLXOnEspressoFailure.dropFirst().sink { + [defaults] in defaults.set($0, forKey: Key.fallbackToMLXOnEspressoFailure.rawValue) + }.store(in: &cancellables) $remoteProvider.dropFirst().sink { [defaults] in defaults.set($0.rawValue, forKey: Key.remoteProvider.rawValue) }.store(in: &cancellables) $remoteAPIKey.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.remoteAPIKey.rawValue) }.store(in: &cancellables) $remoteBaseURL.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.remoteBaseURL.rawValue) }.store(in: &cancellables) diff --git a/Sources/Integration/InputSessionCoordinator+Output.swift b/Sources/Integration/InputSessionCoordinator+Output.swift index a902da1..bb18354 100644 --- a/Sources/Integration/InputSessionCoordinator+Output.swift +++ b/Sources/Integration/InputSessionCoordinator+Output.swift @@ -3,6 +3,12 @@ import Foundation @MainActor extension InputSessionCoordinator { func outputText(for raw: String, active: ActiveSession) async throws -> String { + try await TextProcessor.withEspressoOutcomeTracking { + try await trackedOutputText(for: raw, active: active) + } + } + + private func trackedOutputText(for raw: String, active: ActiveSession) async throws -> String { let options = TextProcessingOptions(settings: settings, inputLanguage: active.inputLanguage) let dictionarySnapshot = PersonalDictionary.shared.snapshot(settings: settings) let enableMemory = settings.enableMemory @@ -63,8 +69,20 @@ extension InputSessionCoordinator { ) } + let espressoOutcome = await textProcessor.consumeEspressoOutcome() + if !Task.isCancelled, EspressoFallbackPolicy.selectMLXIfNeeded( + after: espressoOutcome, + settings: settings, + expectedEspressoModelPath: options.espressoModelPath + ) { + Log.info("[InputSessionCoordinator] ANE-LM failed; selected MLX as the active backend") + } + guard !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { Log.info("[InputSessionCoordinator] refusing to complete session with empty output") + if let espressoOutcome, espressoOutcome != .fallback { + throw IntegrationError.operationFailedWithMessage(espressoOutcome.message) + } throw IntegrationError.operationFailed } diff --git a/Sources/Integration/InputSessionCoordinator.swift b/Sources/Integration/InputSessionCoordinator.swift index 981da0c..cced5ae 100644 --- a/Sources/Integration/InputSessionCoordinator.swift +++ b/Sources/Integration/InputSessionCoordinator.swift @@ -229,7 +229,7 @@ final class InputSessionCoordinator { } let contextMode = ScreenContextMode.effectiveCaptureMode( preference: settings.screenContextMode, - useRemoteLLM: settings.useRemoteLLM, + useRemoteLLM: settings.useRemoteLLM || settings.localLLMBackend == .espresso, modelID: settings.llmModel ) return Task.detached(priority: .utility) { diff --git a/Sources/Integration/IntegrationError.swift b/Sources/Integration/IntegrationError.swift index 3045463..33d4d2a 100644 --- a/Sources/Integration/IntegrationError.swift +++ b/Sources/Integration/IntegrationError.swift @@ -16,6 +16,7 @@ enum IntegrationError: Error, Equatable { case invalidSessionState case noSpeechDetected case operationFailed + case operationFailedWithMessage(String) var payload: Payload { switch self { @@ -39,6 +40,8 @@ enum IntegrationError: Error, Equatable { return Payload(error: "no_speech_detected", message: "No speech was detected in the recording.") case .operationFailed: return Payload(error: "operation_failed", message: "Utter could not complete the input session.") + case .operationFailedWithMessage(let message): + return Payload(error: "operation_failed", message: message) } } } diff --git a/Sources/Integration/IntegrationHTTPDispatcher.swift b/Sources/Integration/IntegrationHTTPDispatcher.swift index cd73da3..04ca529 100644 --- a/Sources/Integration/IntegrationHTTPDispatcher.swift +++ b/Sources/Integration/IntegrationHTTPDispatcher.swift @@ -114,7 +114,7 @@ struct IntegrationHTTPDispatcher { return 404 case .permissionDenied, .modelNotReady, .noSpeechDetected: return 400 - case .operationFailed: + case .operationFailed, .operationFailedWithMessage: return 500 } } diff --git a/Sources/LLM/ANELMTokenizerValidation.swift b/Sources/LLM/ANELMTokenizerValidation.swift new file mode 100644 index 0000000..55de748 --- /dev/null +++ b/Sources/LLM/ANELMTokenizerValidation.swift @@ -0,0 +1,130 @@ +import Foundation + +enum ANELMTokenizerValidation { + static func samplerVocabularySize( + tokenizer: [String: Any], + tokenizerConfig: [String: Any], + modelVocabularySize: Int + ) -> Int? { + guard supportedComponents(in: tokenizer), + let model = tokenizer["model"] as? [String: Any], + model["type"] as? String == "BPE", + let vocabulary = model["vocab"] as? [String: Any], + !vocabulary.isEmpty, + let merges = model["merges"] as? [Any], + !merges.isEmpty, + tokenizerClass(in: tokenizerConfig) == "Qwen2Tokenizer" else { + return nil + } + + let vocabularyEntries = vocabulary.compactMapValues { $0 as? Int } + let addedTokens = tokenizer["added_tokens"] as? [[String: Any]] ?? [] + let addedEntries = addedTokens.reduce(into: [String: Int]()) { result, token in + if let content = token["content"] as? String, let id = token["id"] as? Int { + result[content] = id + } + } + guard vocabularyEntries.count == vocabulary.count, + addedEntries.count == addedTokens.count else { + return nil + } + + let allEntries = vocabularyEntries.merging(addedEntries) { _, _ in -1 } + let ids = Array(allEntries.values) + guard allEntries.count == vocabularyEntries.count + addedEntries.count, + Set(ids).count == ids.count, + ids.allSatisfy({ $0 >= 0 && $0 < modelVocabularySize }), + let maximumID = ids.max(), + Set(ids) == Set(0...maximumID), + byteLevelAlphabet.allSatisfy({ vocabularyEntries[$0] != nil }), + vocabularyEntries.keys.allSatisfy(byteLevelTokenIsValid), + merges.allSatisfy({ mergeIsValid($0, vocabulary: vocabularyEntries) }), + requiredChatTokensAreMapped(tokenizerConfig, addedTokens: addedTokens), + configuredTokensAreMapped(tokenizerConfig, vocabulary: allEntries) else { + return nil + } + return maximumID + 1 + } + + private static func supportedComponents(in tokenizer: [String: Any]) -> Bool { + guard let normalizer = tokenizer["normalizer"] as? [String: Any], + normalizer["type"] as? String == "NFC", + let preTokenizer = tokenizer["pre_tokenizer"] as? [String: Any], + preTokenizer["type"] as? String == "Sequence", + let parts = preTokenizer["pretokenizers"] as? [[String: Any]], + parts.count == 2, + parts[0]["type"] as? String == "Split", + parts[1]["type"] as? String == "ByteLevel", + let pattern = parts[0]["pattern"] as? [String: Any], + pattern.values.contains(where: { ($0 as? String)?.isEmpty == false }), + let postProcessor = tokenizer["post_processor"] as? [String: Any], + postProcessor["type"] as? String == "ByteLevel", + let decoder = tokenizer["decoder"] as? [String: Any], + decoder["type"] as? String == "ByteLevel" else { + return false + } + return true + } + + private static func tokenizerClass(in config: [String: Any]) -> String? { + (config["tokenizer_class"] as? String)?.replacingOccurrences(of: "Fast", with: "") + } + + private static func configuredTokensAreMapped( + _ config: [String: Any], + vocabulary: [String: Int] + ) -> Bool { + ["unk_token", "bos_token", "eos_token", "pad_token"].allSatisfy { key in + guard let value = config[key], !(value is NSNull) else { return true } + let content = value as? String ?? (value as? [String: Any])?["content"] as? String + return content.flatMap { vocabulary[$0] } != nil + } + } + + private static func requiredChatTokensAreMapped( + _ config: [String: Any], + addedTokens: [[String: Any]] + ) -> Bool { + let specialTokens = Set(addedTokens.compactMap { token -> String? in + guard token["special"] as? Bool == true else { return nil } + return token["content"] as? String + }) + return specialTokens.contains("<|im_start|>") + && specialTokens.contains("<|im_end|>") + && configuredTokenContent(config["eos_token"]) == "<|im_end|>" + } + + private static func configuredTokenContent(_ value: Any?) -> String? { + value as? String ?? (value as? [String: Any])?["content"] as? String + } + + private static func mergeIsValid(_ value: Any, vocabulary: [String: Int]) -> Bool { + let pair: [String]? + if let array = value as? [String], array.count == 2 { + pair = array + } else if let string = value as? String { + let parts = string.split(separator: " ", omittingEmptySubsequences: false).map(String.init) + pair = parts.count == 2 ? parts : nil + } else { + pair = nil + } + guard let pair else { return false } + return vocabulary[pair[0] + pair[1]] != nil + } + + private static func byteLevelTokenIsValid(_ token: String) -> Bool { + token.unicodeScalars.allSatisfy { byteLevelScalars.contains($0) } + } + + private static let byteLevelAlphabet: [String] = { + var bytes = Array(33...126) + Array(161...172) + Array(174...255) + var codePoints = bytes + for byte in 0...255 where !bytes.contains(byte) { + bytes.append(byte) + codePoints.append(256 + codePoints.count - 188) + } + return codePoints.compactMap(UnicodeScalar.init).map(String.init) + }() + + private static let byteLevelScalars = Set(byteLevelAlphabet.compactMap { $0.unicodeScalars.first }) +} diff --git a/Sources/LLM/EspressoLLMEngine.swift b/Sources/LLM/EspressoLLMEngine.swift new file mode 100644 index 0000000..a63008e --- /dev/null +++ b/Sources/LLM/EspressoLLMEngine.swift @@ -0,0 +1,263 @@ +import ANELMRuntime +import Foundation +import Tokenizers + +private final class ANELMGenerationContext { + var tokens: [Int32] = [] +} + +private func aneLMTokenCallback(_ token: Int32, _ context: UnsafeMutableRawPointer?) -> Int32 { + let isCancelled = withUnsafeCurrentTask { $0?.isCancelled ?? false } + guard !isCancelled, let context else { return 0 } + if token >= 0 { + Unmanaged.fromOpaque(context).takeUnretainedValue().tokens.append(token) + } + return 1 +} + +actor EspressoLLMEngine { + private final class LoadedModel { + let path: String + let runtime: OpaquePointer + let tokenizer: any Tokenizers.Tokenizer + let samplerVocabularySize: Int + + init( + path: String, + runtime: OpaquePointer, + tokenizer: any Tokenizers.Tokenizer, + samplerVocabularySize: Int + ) { + self.path = path + self.runtime = runtime + self.tokenizer = tokenizer + self.samplerVocabularySize = samplerVocabularySize + } + + deinit { + ane_lm_destroy(runtime) + } + } + + private var model: LoadedModel? + private var lastFailureMessage: String? + + func loadModel(path: String) async throws { + let expandedPath = NSString(string: path).expandingTildeInPath + let url = URL(fileURLWithPath: expandedPath, isDirectory: true).standardizedFileURL + lastFailureMessage = nil + guard model?.path != url.path else { return } + + Log.info("[ANELMEngine] loading Qwen3 model: \(url.lastPathComponent)") + do { + let validated = try await Self.makeValidatedTokenizer(at: url) + model = nil + var nativeError: UnsafeMutablePointer? + let runtime = url.path.withCString { ane_lm_create($0, &nativeError) } + guard let runtime else { + throw ANELMNativeError(Self.consumeNativeError(&nativeError)) + } + guard !Task.isCancelled else { + ane_lm_destroy(runtime) + throw CancellationError() + } + model = LoadedModel( + path: url.path, + runtime: runtime, + tokenizer: validated.tokenizer, + samplerVocabularySize: validated.samplerVocabularySize + ) + Log.info("[ANELMEngine] Qwen3 model ready for ANE inference") + } catch is CancellationError { + throw CancellationError() + } catch { + throw recordFailure(error) + } + } + + func generate( + prompt: String, + systemPrompt: String, + maxTokens: Int, + temperature: Double + ) throws -> String { + guard let model else { throw EspressoLLMError.modelNotLoaded } + lastFailureMessage = nil + + let input = Self.formatPrompt(user: prompt, system: systemPrompt, modelName: "Qwen3") + let promptTokens = model.tokenizer + .encode(text: input, addSpecialTokens: false) + .map { Int32(clamping: $0) } + guard !promptTokens.isEmpty else { throw EspressoLLMError.runtimeFailure } + + let context = ANELMGenerationContext() + context.tokens.reserveCapacity(max(0, maxTokens)) + let contextPointer = Unmanaged.passUnretained(context).toOpaque() + let eosToken = Int32(clamping: model.tokenizer.eosTokenId ?? -1) + let stopToken = Int32(clamping: model.tokenizer.convertTokenToId("<|im_end|>") ?? -1) + var nativeError: UnsafeMutablePointer? + let started = CFAbsoluteTimeGetCurrent() + + let status = promptTokens.withUnsafeBufferPointer { tokens in + ane_lm_generate( + model.runtime, + tokens.baseAddress, + tokens.count, + Int32(clamping: max(1, maxTokens)), + Float(temperature), + 1.2, + Int32(clamping: model.samplerVocabularySize), + eosToken, + stopToken, + aneLMTokenCallback, + contextPointer, + &nativeError + ) + } + + if status == ANE_LM_STATUS_CANCELLED { + throw CancellationError() + } + guard status == ANE_LM_STATUS_OK else { + let error = ANELMNativeError(Self.consumeNativeError(&nativeError)) + throw recordFailure(error) + } + try Task.checkCancellation() + + let output = model.tokenizer.decode( + tokens: context.tokens.map(Int.init), + skipSpecialTokens: true + ) + let elapsed = CFAbsoluteTimeGetCurrent() - started + let speed = elapsed > 0 ? Double(context.tokens.count) / elapsed : 0 + Log.info( + "[ANELMEngine] generated \(context.tokens.count) tokens on ANE in " + + "\(String(format: "%.1f", elapsed))s (\(String(format: "%.1f", speed)) tok/s)" + ) + return output + } + + var isLoaded: Bool { model != nil } + + func unload() { + model = nil + lastFailureMessage = nil + } + + func consumeLastFailureMessage() -> String? { + defer { lastFailureMessage = nil } + return lastFailureMessage + } + + static func validateModelDirectory(at url: URL) async throws { + _ = try await makeValidatedTokenizer(at: url) + } + + private struct ValidatedTokenizer { + let tokenizer: any Tokenizers.Tokenizer + let samplerVocabularySize: Int + } + + private static func makeValidatedTokenizer(at url: URL) async throws -> ValidatedTokenizer { + guard ModelStorage.llmRepoIsComplete(at: url) else { + throw EspressoLLMError.invalidModelDirectory + } + let configURL = url.appendingPathComponent("config.json") + guard let data = try? Data(contentsOf: configURL), + let config = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + config["model_type"] as? String == "qwen3" else { + throw EspressoLLMError.unsupportedModel + } + let textConfig = config["text_config"] as? [String: Any] ?? config + guard let modelVocabularySize = textConfig["vocab_size"] as? Int, + modelVocabularySize > 0 else { + throw EspressoLLMError.invalidModelDirectory + } + let tokenizerURL = url.appendingPathComponent("tokenizer.json") + let tokenizerConfigURL = url.appendingPathComponent("tokenizer_config.json") + guard let tokenizerData = try? Data(contentsOf: tokenizerURL), + let tokenizerJSON = try? JSONSerialization.jsonObject(with: tokenizerData) as? [String: Any], + let tokenizerConfigData = try? Data(contentsOf: tokenizerConfigURL), + let tokenizerConfig = try? JSONSerialization.jsonObject( + with: tokenizerConfigData + ) as? [String: Any], + let samplerVocabularySize = ANELMTokenizerValidation.samplerVocabularySize( + tokenizer: tokenizerJSON, + tokenizerConfig: tokenizerConfig, + modelVocabularySize: modelVocabularySize + ) else { + throw EspressoLLMError.invalidModelDirectory + } + var nativeError: UnsafeMutablePointer? + let status = url.path.withCString { ane_lm_validate_model($0, &nativeError) } + guard status == ANE_LM_STATUS_OK else { + let detail = consumeNativeError(&nativeError) + Log.sensitive("[ANELMEngine] rejected model directory: \(detail)") + throw EspressoLLMError.invalidModelDirectory + } + do { + let tokenizer = try await AutoTokenizer.from(modelFolder: url) + return ValidatedTokenizer( + tokenizer: tokenizer, + samplerVocabularySize: samplerVocabularySize + ) + } catch { + Log.sensitive("[ANELMEngine] rejected tokenizer: \(error.localizedDescription)") + throw EspressoLLMError.invalidModelDirectory + } + } + + static func formatPrompt(user: String, system: String, modelName: String) -> String { + if modelName.lowercased().contains("qwen") { + return "<|im_start|>system\n\(system)<|im_end|>\n" + + "<|im_start|>user\n\(user)<|im_end|>\n" + + "<|im_start|>assistant\n" + } + return "System:\n\(system)\n\nUser:\n\(user)\n\nAssistant:\n" + } + + private static func consumeNativeError(_ pointer: inout UnsafeMutablePointer?) -> String { + guard let allocated = pointer else { return "ANE-LM failed without an error message" } + defer { + ane_lm_free_string(allocated) + pointer = nil + } + return String(cString: allocated) + } + + private func recordFailure(_ error: Error) -> EspressoLLMError { + let mapped = error as? EspressoLLMError ?? .runtimeFailure + Log.sensitive("[ANELMEngine] runtime detail: \(error.localizedDescription)") + Log.error("[ANELMEngine] \(mapped.localizedDescription)") + lastFailureMessage = mapped.localizedDescription + return mapped + } +} + +private struct ANELMNativeError: LocalizedError { + let message: String + + init(_ message: String) { + self.message = message + } + + var errorDescription: String? { message } +} + +enum EspressoLLMError: LocalizedError { + case modelNotLoaded + case aneBackendUnavailable + case invalidModelDirectory + case unsupportedModel + case runtimeFailure + + var errorDescription: String? { + switch self { + case .modelNotLoaded: return L("error.espresso_not_loaded") + case .aneBackendUnavailable: return L("error.espresso_ane_unavailable") + case .invalidModelDirectory: return L("error.espresso_invalid_model") + case .unsupportedModel: return L("error.espresso_unsupported_model") + case .runtimeFailure: return L("error.espresso_runtime_failed") + } + } +} diff --git a/Sources/Processing/EspressoGenerationOutcome.swift b/Sources/Processing/EspressoGenerationOutcome.swift new file mode 100644 index 0000000..a26bd18 --- /dev/null +++ b/Sources/Processing/EspressoGenerationOutcome.swift @@ -0,0 +1,54 @@ +import Foundation + +enum EspressoGenerationOutcome: Equatable, Sendable { + case fallback + case failed + case unavailable + + var message: String { + switch self { + case .fallback: + return L("status.espresso_fell_back_to_mlx") + case .failed: + return L("error.espresso_runtime_failed") + case .unavailable: + return L("error.espresso_mlx_fallback_unavailable") + } + } +} + +actor EspressoGenerationTracker { + private var outcome: EspressoGenerationOutcome? + + func record(_ newOutcome: EspressoGenerationOutcome) { + outcome = newOutcome + } + + func clear() { + outcome = nil + } + + func consume() -> EspressoGenerationOutcome? { + defer { outcome = nil } + return outcome + } +} + +enum EspressoFallbackPolicy { + @discardableResult + static func selectMLXIfNeeded( + after outcome: EspressoGenerationOutcome?, + settings: AppSettings, + expectedEspressoModelPath: String + ) -> Bool { + guard outcome == .fallback, + !settings.useRemoteLLM, + settings.fallbackToMLXOnEspressoFailure, + settings.localLLMBackend == .espresso, + settings.espressoModelPath == expectedEspressoModelPath else { + return false + } + settings.localLLMBackend = .mlx + return true + } +} diff --git a/Sources/Processing/TextProcessingOptions.swift b/Sources/Processing/TextProcessingOptions.swift index 6f1b6e2..1de5d3f 100644 --- a/Sources/Processing/TextProcessingOptions.swift +++ b/Sources/Processing/TextProcessingOptions.swift @@ -16,6 +16,9 @@ struct TextProcessingOptions { var customStylePrompt: String var llmModel: String var useRemoteLLM: Bool + var localLLMBackend: LocalLLMBackend + var espressoModelPath: String + var fallbackToMLXOnEspressoFailure: Bool var remoteBaseURL: String var remoteAPIKey: String var remoteModel: String @@ -38,6 +41,9 @@ struct TextProcessingOptions { self.customStylePrompt = settings.customStylePrompt self.llmModel = settings.llmModel self.useRemoteLLM = settings.useRemoteLLM + self.localLLMBackend = settings.localLLMBackend + self.espressoModelPath = settings.espressoModelPath + self.fallbackToMLXOnEspressoFailure = settings.fallbackToMLXOnEspressoFailure self.remoteBaseURL = settings.remoteBaseURL self.remoteAPIKey = settings.remoteAPIKey self.remoteModel = settings.remoteModel diff --git a/Sources/Processing/TextProcessor+Generation.swift b/Sources/Processing/TextProcessor+Generation.swift index a6d0054..0a3d4ae 100644 --- a/Sources/Processing/TextProcessor+Generation.swift +++ b/Sources/Processing/TextProcessor+Generation.swift @@ -10,6 +10,7 @@ extension TextProcessor { temperature: Double ) async throws -> String { if options.useRemoteLLM { + await Self.clearEspressoOutcome() return try await remoteLLMClient.generate( prompt: prompt, systemPrompt: systemPrompt, @@ -22,13 +23,107 @@ extension TextProcessor { ) } - await ensureModelLoaded(options.llmModel) - return try await llm.generate( - prompt: prompt, - systemPrompt: systemPrompt, - maxTokens: maxTokens, - temperature: temperature - ) + return try await withLocalModelAccess { + try Task.checkCancellation() + switch options.localLLMBackend { + case .mlx: + await Self.clearEspressoOutcome() + await ensureModelLoaded(options.llmModel) + return try await llm.generate( + prompt: prompt, + systemPrompt: systemPrompt, + maxTokens: maxTokens, + temperature: temperature + ) + case .espresso: + do { + let result = try await Self.runEspressoWithMLXFallback( + fallbackEnabled: options.fallbackToMLXOnEspressoFailure, + espresso: { + try await self.espressoLLM.loadModel(path: options.espressoModelPath) + return try await self.espressoLLM.generate( + prompt: prompt, + systemPrompt: systemPrompt, + maxTokens: maxTokens, + temperature: temperature + ) + }, + mlx: { + try await self.llm.loadModel(id: options.llmModel) + return try await self.llm.generate( + prompt: prompt, + systemPrompt: systemPrompt, + maxTokens: maxTokens, + temperature: temperature + ) + } + ) + if result.usedMLX { + _ = await espressoLLM.consumeLastFailureMessage() + await Self.recordEspressoOutcome(.fallback) + Log.info("[TextProcessor] ANE-LM failed; used the selected MLX model") + } else { + await Self.clearEspressoOutcome() + } + return result.value + } catch is CancellationError { + throw CancellationError() + } catch let error as EspressoMLXFallbackError { + _ = await espressoLLM.consumeLastFailureMessage() + await Self.recordEspressoOutcome(.unavailable) + Log.sensitive("[TextProcessor] ANE-LM and MLX fallback failed: \(error.details)") + Log.error("[TextProcessor] MLX fallback unavailable") + throw error + } catch { + if !options.fallbackToMLXOnEspressoFailure { + _ = await espressoLLM.consumeLastFailureMessage() + await Self.recordEspressoOutcome(.failed) + Log.error("[TextProcessor] ANE-LM failed; MLX fallback is disabled") + } + throw error + } + } + } + } + + static func runEspressoWithMLXFallback( + fallbackEnabled: Bool = true, + espresso: () async throws -> Value, + mlx: () async throws -> Value + ) async throws -> (value: Value, usedMLX: Bool) { + do { + let value = try await espresso() + try Task.checkCancellation() + return (value, false) + } catch { + try Task.checkCancellation() + guard fallbackEnabled else { throw error } + let espressoFailure = error.localizedDescription + do { + let value = try await mlx() + try Task.checkCancellation() + return (value, true) + } catch { + try Task.checkCancellation() + throw EspressoMLXFallbackError( + espressoFailure: espressoFailure, + mlxFailure: error.localizedDescription + ) + } + } + } + + struct EspressoMLXFallbackError: LocalizedError { + let espressoFailure: String + let mlxFailure: String + + var errorDescription: String? { + L("error.espresso_mlx_fallback_unavailable") + } + + var details: String { + "ANE-LM: \(espressoFailure); MLX: \(mlxFailure)" + } } func generateWithScreenImage( @@ -39,19 +134,25 @@ extension TextProcessor { maxTokens: Int, temperature: Double ) async throws -> String { - try await vlm.loadModel(id: model) - return try await vlm.generate( - prompt: prompt, - systemPrompt: systemPrompt, - image: image, - maxTokens: maxTokens, - temperature: temperature - ) + try await withLocalModelAccess { + try Task.checkCancellation() + await Self.clearEspressoOutcome() + try await vlm.loadModel(id: model) + return try await vlm.generate( + prompt: prompt, + systemPrompt: systemPrompt, + image: image, + maxTokens: maxTokens, + temperature: temperature + ) + } } func shouldUseScreenImage(options: TextProcessingOptions, image: CGImage?) -> Bool { guard image != nil else { return false } - guard options.screenContextMode == .multimodal, !options.useRemoteLLM else { return false } + guard options.screenContextMode == .multimodal, + !options.useRemoteLLM, + options.localLLMBackend == .mlx else { return false } return ScreenContextMode.supportsScreenImageContext(modelID: options.llmModel) } diff --git a/Sources/Processing/TextProcessor+Models.swift b/Sources/Processing/TextProcessor+Models.swift new file mode 100644 index 0000000..29820e2 --- /dev/null +++ b/Sources/Processing/TextProcessor+Models.swift @@ -0,0 +1,198 @@ +import Foundation +import MLX + +actor LocalModelAccessGate { + private struct Waiter { + let id: UUID + let continuation: CheckedContinuation + } + + private var isOccupied = false + private var waiters: [Waiter] = [] + + var waitingTaskCount: Int { waiters.count } + + func acquire() async throws { + try Task.checkCancellation() + guard isOccupied else { + isOccupied = true + return + } + + let id = UUID() + try await withTaskCancellationHandler(operation: { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + guard !Task.isCancelled else { + continuation.resume(throwing: CancellationError()) + return + } + waiters.append(Waiter(id: id, continuation: continuation)) + } + }, onCancel: { + Task { await self.cancelWaiter(id: id) } + }) + } + + func release() { + guard !waiters.isEmpty else { + isOccupied = false + return + } + waiters.removeFirst().continuation.resume() + } + + private func cancelWaiter(id: UUID) { + guard let index = waiters.firstIndex(where: { $0.id == id }) else { return } + waiters.remove(at: index).continuation.resume(throwing: CancellationError()) + } +} + +extension TextProcessor { + func withLocalModelAccess( + _ operation: () async throws -> Value + ) async throws -> Value { + if Self.hasLocalModelAccess { + return try await operation() + } + + try await localModelAccessGate.acquire() + do { + let value = try await Self.$hasLocalModelAccess.withValue(true) { + try Task.checkCancellation() + return try await operation() + } + await localModelAccessGate.release() + return value + } catch { + await localModelAccessGate.release() + throw error + } + } + + static func withEspressoOutcomeTracking( + _ operation: () async throws -> Value + ) async rethrows -> Value { + if espressoGenerationTracker != nil { + return try await operation() + } + return try await $espressoGenerationTracker.withValue(EspressoGenerationTracker()) { + try await operation() + } + } + + static func recordEspressoOutcome(_ outcome: EspressoGenerationOutcome) async { + await espressoGenerationTracker?.record(outcome) + } + + static func clearEspressoOutcome() async { + await espressoGenerationTracker?.clear() + } + + func consumeEspressoOutcome() async -> EspressoGenerationOutcome? { + await Self.espressoGenerationTracker?.consume() + } + + func isLLMReady(for backend: LocalLLMBackend) async -> Bool { + do { + return try await withLocalModelAccess { + switch backend { + case .mlx: + return await llm.isLoaded + case .espresso: + let espressoIsLoaded = await espressoLLM.isLoaded + let mlxIsLoaded = await llm.isLoaded + return espressoIsLoaded || mlxIsLoaded + } + } + } catch { + return false + } + } + + func unloadLLM() async { + do { + try await withLocalModelAccess { + let llmWasLoaded = await llm.isLoaded + let benchmarkWasLoaded = await benchmarkEngine.isLoaded + let vlmWasLoaded = await vlm.isLoaded + await llm.unload() + await benchmarkEngine.unload() + await espressoLLM.unload() + await vlm.unload() + if llmWasLoaded || benchmarkWasLoaded || vlmWasLoaded { + Memory.clearCache() + } + } + } catch { + Log.info("[TextProcessor] local model unload cancelled") + } + } + + func benchmarkLLM(modelID: String) async throws -> LLMEngine.BenchmarkResult { + try await withLocalModelAccess { + try Task.checkCancellation() + do { + let result = try await benchmarkEngine.benchmark(modelID: modelID) + let wasLoaded = await benchmarkEngine.isLoaded + await benchmarkEngine.unload() + if wasLoaded { Memory.clearCache() } + return result + } catch { + let wasLoaded = await benchmarkEngine.isLoaded + await benchmarkEngine.unload() + if wasLoaded { Memory.clearCache() } + throw error + } + } + } + + @discardableResult + func warmUpLLM( + model: String, + backend: LocalLLMBackend, + espressoModelPath: String, + fallbackToMLXOnEspressoFailure: Bool + ) async -> (loaded: Bool, errorMessage: String?, espressoOutcome: EspressoGenerationOutcome?) { + do { + return try await withLocalModelAccess { + do { + switch backend { + case .mlx: + try await llm.loadModel(id: model) + case .espresso: + let result = try await Self.runEspressoWithMLXFallback( + fallbackEnabled: fallbackToMLXOnEspressoFailure, + espresso: { try await self.espressoLLM.loadModel(path: espressoModelPath) }, + mlx: { try await self.llm.loadModel(id: model) } + ) + if result.usedMLX { + _ = await espressoLLM.consumeLastFailureMessage() + return (true, nil, .fallback) + } + } + return (true, nil, nil) + } catch is CancellationError { + throw CancellationError() + } catch let error as EspressoMLXFallbackError { + Log.sensitive("[TextProcessor] ANE-LM and MLX warmup failed: \(error.details)") + Log.error("[TextProcessor] MLX fallback unavailable during warmup") + _ = await espressoLLM.consumeLastFailureMessage() + return (false, EspressoGenerationOutcome.unavailable.message, .unavailable) + } catch { + Log.error("[TextProcessor] LLM warmup failed: \(error.localizedDescription)") + if backend == .espresso { + _ = await espressoLLM.consumeLastFailureMessage() + if !fallbackToMLXOnEspressoFailure { + return (false, EspressoGenerationOutcome.failed.message, .failed) + } + } + return (false, error.localizedDescription, nil) + } + } + } catch is CancellationError { + return (false, nil, nil) + } catch { + return (false, error.localizedDescription, nil) + } + } +} diff --git a/Sources/Processing/TextProcessor.swift b/Sources/Processing/TextProcessor.swift index 031fb0f..064e277 100644 --- a/Sources/Processing/TextProcessor.swift +++ b/Sources/Processing/TextProcessor.swift @@ -4,33 +4,16 @@ import Foundation final class TextProcessor { static let defaultAllowsPreparedFallback = false + @TaskLocal static var espressoGenerationTracker: EspressoGenerationTracker? + @TaskLocal static var hasLocalModelAccess = false + let llm = LLMEngine() + let benchmarkEngine = LLMEngine() + let espressoLLM = EspressoLLMEngine() let vlm = VLMEngine() + let localModelAccessGate = LocalModelAccessGate() let remoteLLMClient = RemoteLLMClient() private let dictionary = PersonalDictionary.shared - var isLLMReady: Bool { - get async { - if AppSettings.shared.useRemoteLLM { return true } - return await llm.isLoaded - } - } - - func unloadLLM() async { - await llm.unload() - await vlm.unload() - } - - @discardableResult - func warmUpLLM(model: String) async -> Bool { - if AppSettings.shared.useRemoteLLM { return true } - do { - try await llm.loadModel(id: model) - return true - } catch { - Log.error("[TextProcessor] LLM warmup failed: \(error.localizedDescription)") - return false - } - } func basicClean( text: String, @@ -127,33 +110,35 @@ final class TextProcessor { var result: String let llmStarted = CFAbsoluteTimeGetCurrent() if let screenImage, useScreenImage { - do { - result = try await generateWithScreenImage( - prompt: userPrompt, - systemPrompt: systemPrompt, - model: options.llmModel, - image: screenImage, - maxTokens: generationOptions.maxTokens, - temperature: generationOptions.temperature - ) - } catch { - Log.error("[TextProcessor] VLM failed, falling back to text LLM: \(error.localizedDescription)") - let textFallbackSystemPrompt = formattingSystemPrompt( - options: options, - screenContext: screenContext, - screenImageAvailable: false, - memoryContext: memoryContext, - inputContext: inputContext, - formatKind: formatKind, - dictionarySnapshot: dictionarySnapshot - ) - result = try await generateText( - prompt: userPrompt, - systemPrompt: textFallbackSystemPrompt, - options: options, - maxTokens: generationOptions.maxTokens, - temperature: generationOptions.temperature - ) + result = try await withLocalModelAccess { + do { + return try await generateWithScreenImage( + prompt: userPrompt, + systemPrompt: systemPrompt, + model: options.llmModel, + image: screenImage, + maxTokens: generationOptions.maxTokens, + temperature: generationOptions.temperature + ) + } catch { + Log.error("[TextProcessor] VLM failed, falling back to text LLM: \(error.localizedDescription)") + let textFallbackSystemPrompt = formattingSystemPrompt( + options: options, + screenContext: screenContext, + screenImageAvailable: false, + memoryContext: memoryContext, + inputContext: inputContext, + formatKind: formatKind, + dictionarySnapshot: dictionarySnapshot + ) + return try await generateText( + prompt: userPrompt, + systemPrompt: textFallbackSystemPrompt, + options: options, + maxTokens: generationOptions.maxTokens, + temperature: generationOptions.temperature + ) + } } } else { result = try await generateText( @@ -246,32 +231,34 @@ final class TextProcessor { do { var result: String if let screenImage, useScreenImage { - do { - result = try await generateWithScreenImage( - prompt: userPrompt, - systemPrompt: systemPrompt, - model: options.llmModel, - image: screenImage, - maxTokens: 4096, - temperature: 0.3 - ) - } catch { - Log.error("[TextProcessor] Command VLM failed, falling back to text LLM: \(error.localizedDescription)") - let textFallbackSystemPrompt = commandSystemPrompt( - options: options, - screenContext: screenContext, - screenImageAvailable: false, - memoryContext: memoryContext, - inputContext: inputContext, - dictionarySnapshot: dictionarySnapshot - ) - result = try await generateText( - prompt: userPrompt, - systemPrompt: textFallbackSystemPrompt, - options: options, - maxTokens: 4096, - temperature: 0.3 - ) + result = try await withLocalModelAccess { + do { + return try await generateWithScreenImage( + prompt: userPrompt, + systemPrompt: systemPrompt, + model: options.llmModel, + image: screenImage, + maxTokens: 4096, + temperature: 0.3 + ) + } catch { + Log.error("[TextProcessor] Command VLM failed, falling back to text LLM: \(error.localizedDescription)") + let textFallbackSystemPrompt = commandSystemPrompt( + options: options, + screenContext: screenContext, + screenImageAvailable: false, + memoryContext: memoryContext, + inputContext: inputContext, + dictionarySnapshot: dictionarySnapshot + ) + return try await generateText( + prompt: userPrompt, + systemPrompt: textFallbackSystemPrompt, + options: options, + maxTokens: 4096, + temperature: 0.3 + ) + } } } else { result = try await generateText( diff --git a/Sources/Resources/en.lproj/Localizable.strings b/Sources/Resources/en.lproj/Localizable.strings index 2774114..7b12e95 100644 --- a/Sources/Resources/en.lproj/Localizable.strings +++ b/Sources/Resources/en.lproj/Localizable.strings @@ -22,6 +22,7 @@ /* ── Status ── */ "status.ready" = "Ready"; "status.done" = "Done"; +"status.espresso_fell_back_to_mlx" = "ANE-LM failed on this Mac. This request finished with MLX."; "status.no_speech_detected" = "No Speech Detected"; /* ── Tabs ── */ @@ -193,7 +194,7 @@ "model.preload.speech" = "Preload dictation model on launch"; "model.preload.speech_help" = "Applies to already downloaded WhisperKit models. Use the Download button in Models first."; "model.preload.formatting" = "Preload formatting model on launch"; -"model.preload.formatting_help" = "Applies to local MLX models. Remote LLM providers are called when formatting starts."; +"model.preload.formatting_help" = "Applies to local MLX and ANE models. Remote LLM providers are called when formatting starts."; "model.speech_recognition" = "Speech Recognition"; "model.apple_managed_by_system" = "Apple Speech uses the language and recognition services managed by macOS."; "model.text_formatting" = "Text Formatting (LLM)"; @@ -202,6 +203,12 @@ "model.family.gemma" = "Google Gemma - Lightweight"; "model.family.llama" = "Meta Llama - General Purpose"; "model.family.remote" = "Remote"; +"model.espresso.description" = "Run a local Qwen3 safetensors model with the experimental ANE-LM runtime."; +"model.espresso.no_bundle" = "No Qwen3 model selected"; +"model.espresso.choose" = "Choose Qwen3 Model…"; +"model.espresso.auto_fallback" = "Automatically switch to MLX if ANE-LM fails"; +"model.espresso.auto_fallback_help" = "Finish the request with the selected installed MLX model and make MLX active. Turn this off to keep ANE selected and show its error."; +"model.espresso.private_api_warning" = "ANE-LM uses private Apple Neural Engine APIs. Compatibility depends on the Mac and macOS version and is not guaranteed. It is not eligible for Mac App Store distribution. Models use substantial memory; macOS may temporarily retain some released memory after switching away."; "model.custom_id_placeholder" = "Custom model ID (e.g. mlx-community/…)"; "model.active" = "Active"; "model.use" = "Use"; @@ -461,6 +468,8 @@ "error.load_failed" = "Model loading failed: %@"; "error.network_request_failed" = "Network request failed — please try again"; "error.operation_failed" = "Operation failed — please try again"; +"error.espresso_runtime_failed" = "ANE-LM could not run this model on the ANE. Switch to MLX or try a supported macOS and device."; +"error.espresso_mlx_fallback_unavailable" = "ANE-LM failed, and the selected MLX model is unavailable. Download an MLX model in Settings → Models."; "error.volc_not_configured" = "Doubao ASR not configured — enter API credentials in Settings → Models"; "error.volc_invalid_endpoint" = "Invalid ASR endpoint URL"; "error.volc_audio_conversion" = "Audio conversion to PCM 16kHz failed"; @@ -484,6 +493,10 @@ "model.asr_incomplete" = "Only part of the model was downloaded. Select Resume to finish."; "error.llm_not_loaded" = "The model is stored locally but is not loaded into memory. Run the action again and Utter will retry."; "error.llm_not_downloaded" = "The model files have not been downloaded. Open Settings → Models and confirm the data usage first."; +"error.espresso_not_loaded" = "The ANE model is not loaded. Select a valid local Qwen3 model in Settings → Models."; +"error.espresso_ane_unavailable" = "The Apple Neural Engine runtime is unavailable on this Mac."; +"error.espresso_invalid_model" = "This folder is not a complete local Qwen3 model. Select a folder containing config, tokenizer, and safetensors files."; +"error.espresso_unsupported_model" = "ANE-LM currently supports Qwen3 models only. Choose a Qwen3 model or use MLX."; "onboarding.download_notice" = "No download starts automatically. Review the size, then confirm if you want this local model."; "onboarding.download_size" = "Download (%@)"; diff --git a/Sources/Resources/zh-Hans.lproj/Localizable.strings b/Sources/Resources/zh-Hans.lproj/Localizable.strings index a3b4eef..fc37af8 100644 --- a/Sources/Resources/zh-Hans.lproj/Localizable.strings +++ b/Sources/Resources/zh-Hans.lproj/Localizable.strings @@ -22,6 +22,7 @@ /* ── Status ── */ "status.ready" = "就绪"; "status.done" = "完成"; +"status.espresso_fell_back_to_mlx" = "ANE-LM 在这台 Mac 上失败。本次已改用 MLX 完成。"; "status.no_speech_detected" = "未检测到语音"; /* ── Tabs ── */ @@ -193,7 +194,7 @@ "model.preload.speech" = "启动时预加载听写模型"; "model.preload.speech_help" = "仅对已下载的 WhisperKit 模型生效。请先在模型页点击下载按钮。"; "model.preload.formatting" = "启动时预加载修稿模型"; -"model.preload.formatting_help" = "仅对本地 MLX 模型生效。远程 LLM 会在整理开始时调用。"; +"model.preload.formatting_help" = "对本地 MLX 和 ANE 模型生效。远程 LLM 会在整理开始时调用。"; "model.speech_recognition" = "语音识别"; "model.apple_managed_by_system" = "Apple 语音使用由 macOS 管理的语言与识别服务。"; "model.text_formatting" = "文本整理 (LLM)"; @@ -202,6 +203,12 @@ "model.family.gemma" = "Google Gemma - 轻量"; "model.family.llama" = "Meta Llama - 通用"; "model.family.remote" = "远程"; +"model.espresso.description" = "使用实验性的 ANE-LM 运行本地 Qwen3 safetensors 模型。"; +"model.espresso.no_bundle" = "尚未选择 Qwen3 模型"; +"model.espresso.choose" = "选择 Qwen3 模型…"; +"model.espresso.auto_fallback" = "ANE-LM 失败时自动切换到 MLX"; +"model.espresso.auto_fallback_help" = "使用已安装且选中的 MLX 模型完成本次请求，并将 MLX 设为当前后端。关闭后会保留 ANE 并显示错误。"; +"model.espresso.private_api_warning" = "ANE-LM 使用 Apple 神经网络引擎私有 API，兼容性取决于 Mac 机型和 macOS 版本，无法保证，也无法通过 Mac App Store 审核。模型会占用较多内存；切换后 macOS 仍可能暂时保留一部分已释放内存。"; "model.custom_id_placeholder" = "自定义模型 ID（如 mlx-community/…）"; "model.active" = "当前"; "model.use" = "启用"; @@ -461,6 +468,8 @@ "error.load_failed" = "模型加载失败: %@"; "error.network_request_failed" = "网络请求失败，请稍后重试"; "error.operation_failed" = "操作失败，请重试"; +"error.espresso_runtime_failed" = "ANE-LM 无法在这台设备的 ANE 上运行此模型。请切换到 MLX，或改用受支持的 macOS 与设备。"; +"error.espresso_mlx_fallback_unavailable" = "ANE-LM 运行失败，所选 MLX 模型也不可用。请在“设置 → 模型”中下载一个 MLX 模型。"; "error.volc_not_configured" = "豆包语音识别未配置 — 请在 设置 → 模型 中填写 API 凭据"; "error.volc_invalid_endpoint" = "语音识别接口地址无效"; "error.volc_audio_conversion" = "音频转换为 PCM 16kHz 失败"; @@ -484,6 +493,10 @@ "model.asr_incomplete" = "模型只下载了一部分。点击“继续下载”即可接着完成"; "error.llm_not_loaded" = "模型文件已在本地，但当前尚未加载到内存。请重新执行；Utter 会再次尝试加载"; "error.llm_not_downloaded" = "模型文件尚未下载。请前往 设置 → 模型，确认流量后下载"; +"error.espresso_not_loaded" = "ANE 模型尚未加载。请在“设置 → 模型”中选择有效的本地 Qwen3 模型。"; +"error.espresso_ane_unavailable" = "这台 Mac 无法使用 Apple 神经网络引擎运行时。"; +"error.espresso_invalid_model" = "这个文件夹不是完整的本地 Qwen3 模型。请选择包含配置、分词器和 safetensors 权重的文件夹。"; +"error.espresso_unsupported_model" = "ANE-LM 目前只支持 Qwen3 模型。请选择 Qwen3 模型，或改用 MLX。"; "onboarding.download_notice" = "这里不会自动下载。请先确认体积，再决定是否下载这个本地模型。"; "onboarding.download_size" = "下载（%@）"; diff --git a/Sources/UI/ModelManagementFamilies.swift b/Sources/UI/ModelManagementFamilies.swift index 50226cd..5037220 100644 --- a/Sources/UI/ModelManagementFamilies.swift +++ b/Sources/UI/ModelManagementFamilies.swift @@ -4,6 +4,7 @@ enum FormattingModelType: String, CaseIterable { case qwen case gemma case llama + case espresso case remote case custom @@ -12,7 +13,7 @@ enum FormattingModelType: String, CaseIterable { case .qwen: .qwen case .gemma: .gemma case .llama: .llama - case .remote, .custom: nil + case .espresso, .remote, .custom: nil } } @@ -24,10 +25,25 @@ enum FormattingModelType: String, CaseIterable { "\(ModelCatalog.ModelFamily.qwen.rawValue) · \(L("common.recommended_short"))" case .gemma: ModelCatalog.ModelFamily.gemma.rawValue case .llama: ModelCatalog.ModelFamily.llama.rawValue + case .espresso: "ANE" case .remote: L("model.family.remote") case .custom: L("common.custom") } } + + static func resolvedLocalSelection( + pending: FormattingModelType?, + activeFamily: ModelCatalog.ModelFamily?? + ) -> FormattingModelType { + if let pending { return pending } + guard let activeFamily else { return .qwen } + switch activeFamily { + case .qwen: return .qwen + case .gemma: return .gemma + case .llama: return .llama + case nil: return .custom + } + } } extension ModelManagementView { @@ -45,9 +61,8 @@ extension ModelManagementView { private var familySelection: Binding { Binding( get: { - if settings.useRemoteLLM { - return .remote - } + if settings.useRemoteLLM { return .remote } + if settings.localLLMBackend == .espresso { return .espresso } switch selectedModelFamily { case .qwen: return .qwen case .gemma: return .gemma @@ -63,6 +78,8 @@ extension ModelManagementView { selectLocalFamily(.gemma) case .llama: selectLocalFamily(.llama) + case .espresso: + selectEspressoLLM() case .remote: selectRemoteLLM() case .custom: @@ -74,12 +91,33 @@ extension ModelManagementView { func selectLocalFamily(_ family: ModelCatalog.ModelFamily) { selectedModelFamily = family - if settings.useRemoteLLM { - settings.useRemoteLLM = false - if catalog.llmModels.first(where: { $0.id == settings.llmModel })?.family == family { - onLoadLLM?() + if settings.localLLMBackend != .mlx { + switch family { + case .qwen: pendingFormattingTypeAfterBackendChange = .qwen + case .gemma: pendingFormattingTypeAfterBackendChange = .gemma + case .llama: pendingFormattingTypeAfterBackendChange = .llama } } + let changedBackend = settings.useRemoteLLM || settings.localLLMBackend != .mlx + if changedBackend { + onUnloadLLM?() + settings.useRemoteLLM = false + settings.localLLMBackend = .mlx + } + if changedBackend, + catalog.llmModels.first(where: { $0.id == settings.llmModel })?.family == family { + onLoadLLM?() + } + } + + func selectEspressoLLM() { + guard settings.useRemoteLLM || settings.localLLMBackend != .espresso else { return } + onUnloadLLM?() + settings.useRemoteLLM = false + settings.localLLMBackend = .espresso + if !settings.espressoModelPath.isEmpty { + onLoadLLM?() + } } func selectRemoteLLM() { @@ -91,12 +129,18 @@ extension ModelManagementView { func selectCustomLLM() { selectedModelFamily = nil - if settings.useRemoteLLM { + if settings.localLLMBackend != .mlx { + pendingFormattingTypeAfterBackendChange = .custom + } + let changedBackend = settings.useRemoteLLM || settings.localLLMBackend != .mlx + if changedBackend { + onUnloadLLM?() settings.useRemoteLLM = false - if let activeModel = catalog.llmModels.first(where: { $0.id == settings.llmModel }), - activeModel.family == nil { - onLoadLLM?() - } + settings.localLLMBackend = .mlx + } + if changedBackend, + catalog.llmModels.first(where: { $0.id == settings.llmModel })?.family == nil { + onLoadLLM?() } } } diff --git a/Sources/UI/ModelManagementRows.swift b/Sources/UI/ModelManagementRows.swift index accb7e6..a12dea8 100644 --- a/Sources/UI/ModelManagementRows.swift +++ b/Sources/UI/ModelManagementRows.swift @@ -221,6 +221,7 @@ extension ModelManagementView { case .llm: onUnloadLLM?() settings.useRemoteLLM = false + settings.localLLMBackend = .mlx settings.llmModel = model.id selectedModelFamily = model.family onLoadLLM?() @@ -254,12 +255,13 @@ extension ModelManagementView { } func runBenchmark(_ modelID: String) async { - guard let idx = catalog.llmModels.firstIndex(where: { $0.id == modelID }) else { return } + guard let idx = catalog.llmModels.firstIndex(where: { $0.id == modelID }), + let onBenchmarkLLM else { return } catalog.llmModels[idx].isBenchmarking = true catalog.llmModels[idx].benchmarkTPS = nil do { - let result = try await benchmarkEngine.benchmark(modelID: modelID) + let result = try await onBenchmarkLLM(modelID) if let i = catalog.llmModels.firstIndex(where: { $0.id == modelID }) { catalog.llmModels[i].benchmarkTPS = result.tokensPerSecond catalog.llmModels[i].isBenchmarking = false @@ -272,30 +274,4 @@ extension ModelManagementView { } } - func secondaryText(for model: ModelCatalog.ModelEntry) -> String { - switch model.status { - case .unavailable(let message), .error(let message): - return message - default: - return model.hint - } - } - - func statusDot(_ status: ModelCatalog.ModelStatus) -> some View { - Group { - switch status { - case .notDownloaded: - Circle().fill(.secondary.opacity(0.3)) - case .downloading, .compiling, .loading: - ProgressView().controlSize(.mini) - case .downloaded, .ready: - Circle().fill(.green) - case .unavailable: - Circle().fill(.orange) - case .error: - Circle().fill(.red) - } - } - .frame(width: 8, height: 8) - } } diff --git a/Sources/UI/ModelManagementSections.swift b/Sources/UI/ModelManagementSections.swift index 1a5134c..a5212ea 100644 --- a/Sources/UI/ModelManagementSections.swift +++ b/Sources/UI/ModelManagementSections.swift @@ -155,6 +155,8 @@ extension ModelManagementView { if settings.useRemoteLLM { RemoteLLMConfigView() + } else if settings.localLLMBackend == .espresso { + espressoLLMSection } else { localLLMModelsSection } @@ -195,15 +197,65 @@ extension ModelManagementView { } } - func syncSelectedFamilyFromActiveModel() { - guard !settings.useRemoteLLM else { return } - if let activeModel = catalog.llmModels.first(where: { $0.id == settings.llmModel }) { - selectedModelFamily = activeModel.family - } else { - selectedModelFamily = .qwen + var espressoLLMSection: some View { + VStack(alignment: .leading, spacing: 10) { + Label("ANE-LM", systemImage: "neural.engine") + .font(.system(size: 12, weight: .semibold)) + + Text(L("model.espresso.description")) + .font(.system(size: 11)) + .foregroundStyle(.secondary) + + HStack(spacing: 8) { + Text(settings.espressoModelPath.isEmpty + ? L("model.espresso.no_bundle") + : settings.espressoModelPath) + .font(.system(size: 10, design: .monospaced)) + .foregroundStyle(settings.espressoModelPath.isEmpty ? .secondary : .primary) + .lineLimit(2) + .textSelection(.enabled) + Spacer() + Button(L("model.espresso.choose")) { + chooseANEModel() + } + .controlSize(.small) + } + + Toggle( + L("model.espresso.auto_fallback"), + isOn: $settings.fallbackToMLXOnEspressoFailure + ) + .help(L("model.espresso.auto_fallback_help")) + + Label(L("model.espresso.private_api_warning"), systemImage: "exclamationmark.triangle.fill") + .font(.system(size: 10)) + .foregroundStyle(.orange) } } + func syncSelectedFamilyFromActiveModel() { + guard !settings.useRemoteLLM, settings.localLLMBackend == .mlx else { return } + let activeFamily = catalog.llmModels + .first(where: { $0.id == settings.llmModel }) + .map(\.family) + selectedModelFamily = FormattingModelType.resolvedLocalSelection( + pending: nil, + activeFamily: activeFamily + ).family + } + + func syncSelectedFamilyAfterBackendChange() { + guard !settings.useRemoteLLM, settings.localLLMBackend == .mlx else { return } + let activeFamily = catalog.llmModels + .first(where: { $0.id == settings.llmModel }) + .map(\.family) + selectedModelFamily = FormattingModelType.resolvedLocalSelection( + pending: pendingFormattingTypeAfterBackendChange, + activeFamily: activeFamily + ).family + pendingFormattingTypeAfterBackendChange = nil + } + /// Split a family's models into recommended (top), standard, and legacy (folded) tiers. @ViewBuilder func groupedLLMModelList( diff --git a/Sources/UI/ModelManagementStatus.swift b/Sources/UI/ModelManagementStatus.swift new file mode 100644 index 0000000..e252081 --- /dev/null +++ b/Sources/UI/ModelManagementStatus.swift @@ -0,0 +1,30 @@ +import SwiftUI + +extension ModelManagementView { + func secondaryText(for model: ModelCatalog.ModelEntry) -> String { + switch model.status { + case .unavailable(let message), .error(let message): + return message + default: + return model.hint + } + } + + func statusDot(_ status: ModelCatalog.ModelStatus) -> some View { + Group { + switch status { + case .notDownloaded: + Circle().fill(.secondary.opacity(0.3)) + case .downloading, .compiling, .loading: + ProgressView().controlSize(.mini) + case .downloaded, .ready: + Circle().fill(.green) + case .unavailable: + Circle().fill(.orange) + case .error: + Circle().fill(.red) + } + } + .frame(width: 8, height: 8) + } +} diff --git a/Sources/UI/ModelManagementView.swift b/Sources/UI/ModelManagementView.swift index 091912e..77227f8 100644 --- a/Sources/UI/ModelManagementView.swift +++ b/Sources/UI/ModelManagementView.swift @@ -9,6 +9,7 @@ struct ModelManagementView: View { var onUnloadWhisper: (() -> Void)? var onUnloadLLM: (() -> Void)? var onLoadLLM: (() -> Void)? + var onBenchmarkLLM: ((String) async throws -> LLMEngine.BenchmarkResult)? var onUnloadLocalASR: (() -> Void)? @State var customLLMInput = "" @@ -17,7 +18,7 @@ struct ModelManagementView: View { @State var selectedModelFamily: ModelCatalog.ModelFamily? = .qwen @State var showLegacyModels = false @State var pendingModelAction: PendingModelAction? - let benchmarkEngine = LLMEngine() + @State var pendingFormattingTypeAfterBackendChange: FormattingModelType? var body: some View { Form { @@ -56,6 +57,9 @@ struct ModelManagementView: View { syncSelectedFamilyFromActiveModel() } .onChange(of: settings.llmModel) { _, _ in syncSelectedFamilyFromActiveModel() } + .onChange(of: settings.localLLMBackend) { _, _ in + syncSelectedFamilyAfterBackendChange() + } .onChange(of: settings.qwenASRModel) { _, _ in onUnloadLocalASR?() } .alert(item: $pendingModelAction, content: modelActionAlert) } @@ -163,6 +167,30 @@ extension ModelManagementView { } } + func chooseANEModel() { + let panel = NSOpenPanel() + panel.canChooseFiles = false + panel.canChooseDirectories = true + panel.allowsMultipleSelection = false + panel.canCreateDirectories = false + panel.message = L("model.espresso.choose") + guard panel.runModal() == .OK, let url = panel.url else { return } + + Task { @MainActor in + do { + try await EspressoLLMEngine.validateModelDirectory(at: url) + onUnloadLLM?() + settings.espressoModelPath = url.path + settings.localLLMBackend = .espresso + settings.useRemoteLLM = false + onLoadLLM?() + } catch { + importErrorMessage = error.localizedDescription + showImportError = true + } + } + } + private func isValidWhisperFolder(_ url: URL) -> Bool { ModelStorage.whisperModelIsComplete(at: url) } diff --git a/Sources/UI/OverlayPanelContent.swift b/Sources/UI/OverlayPanelContent.swift index 60f8a2e..6ec6896 100644 --- a/Sources/UI/OverlayPanelContent.swift +++ b/Sources/UI/OverlayPanelContent.swift @@ -17,6 +17,8 @@ struct OverlayLayout: Equatable { @MainActor init(appState: AppState) { let hasPreview = appState.phase == .recording && !appState.rawTranscription.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + let showsEspressoFallback = appState.phase == .done + && appState.completionKind == .espressoFallback isInteractive = appState.isRecording switch appState.phase { @@ -52,6 +54,14 @@ struct OverlayLayout: Equatable { topPadding = 8 bottomPadding = 8 stackSpacing = 6 + case .done where showsEspressoFallback: + width = 288 + height = 56 + outerCornerRadius = 18 + horizontalPadding = 12 + topPadding = 8 + bottomPadding = 8 + stackSpacing = 6 default: width = 192 height = 40 @@ -89,6 +99,11 @@ struct OverlayContentView: View { return false } + private var showsEspressoFallback: Bool { + appState.phase == .done + && appState.completionKind == .espressoFallback + } + var body: some View { VStack(spacing: layout.stackSpacing) { if layout.isInteractive { @@ -128,7 +143,7 @@ struct OverlayContentView: View { Text(appState.statusMessage) .font(.caption.weight(.medium)) .foregroundStyle(.white.opacity(isError ? 0.94 : 0.88)) - .lineLimit(isError ? 2 : 1) + .lineLimit(isError || showsEspressoFallback ? 2 : 1) .fixedSize(horizontal: false, vertical: true) Spacer(minLength: 4) diff --git a/Sources/UI/SettingsView.swift b/Sources/UI/SettingsView.swift index 9dd9d96..187108f 100644 --- a/Sources/UI/SettingsView.swift +++ b/Sources/UI/SettingsView.swift @@ -27,6 +27,7 @@ struct SettingsView: View { var onUnloadWhisper: (() -> Void)? var onUnloadLLM: (() -> Void)? var onLoadLLM: (() -> Void)? + var onBenchmarkLLM: ((String) async throws -> LLMEngine.BenchmarkResult)? var onUnloadLocalASR: (() -> Void)? var body: some View { @@ -39,6 +40,7 @@ struct SettingsView: View { onUnloadWhisper: onUnloadWhisper, onUnloadLLM: onUnloadLLM, onLoadLLM: onLoadLLM, + onBenchmarkLLM: onBenchmarkLLM, onUnloadLocalASR: onUnloadLocalASR ) .tabItem { Label(L("tab.models"), systemImage: "cpu") } @@ -50,7 +52,7 @@ struct SettingsView: View { .tabItem { Label(L("tab.about"), systemImage: "info.circle") } } .frame(width: SettingsWindowLayout.width, height: SettingsWindowLayout.height) - .background(Color(nsColor: .windowBackgroundColor)) + .background(Color(nsColor: .underPageBackgroundColor)) .id(settings.uiLanguage) } } diff --git a/Sources/UI/SettingsVoiceIllustration.swift b/Sources/UI/SettingsVoiceIllustration.swift index 051c45a..bd1afba 100644 --- a/Sources/UI/SettingsVoiceIllustration.swift +++ b/Sources/UI/SettingsVoiceIllustration.swift @@ -21,6 +21,6 @@ struct SettingsCardBackground: View { extension View { func settingsPageSurface() -> some View { frame(maxWidth: .infinity, maxHeight: .infinity) - .background(Color(nsColor: .windowBackgroundColor)) + .background(Color(nsColor: .underPageBackgroundColor)) } } diff --git a/Tests/OpenTypeTests/ANELMRuntimeTestFixtures.swift b/Tests/OpenTypeTests/ANELMRuntimeTestFixtures.swift new file mode 100644 index 0000000..4a2a3e1 --- /dev/null +++ b/Tests/OpenTypeTests/ANELMRuntimeTestFixtures.swift @@ -0,0 +1,160 @@ +import Foundation + +let syntheticANEModelVocabularySize = 262 + +func makeSyntheticQwen3Directory( + dtype: String = "BF16", + omitting omittedName: String? = nil +) throws -> URL { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("utter-ane-model-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + try writeSyntheticQwen3Config(to: directory) + try writeSyntheticTokenizer(to: directory) + try writeSyntheticTokenizerConfig(to: directory) + + let tensors: [(String, [Int])] = [ + ("model.embed_tokens.weight", [syntheticANEModelVocabularySize, 2]), + ("model.norm.weight", [2]), + ("model.layers.0.input_layernorm.weight", [2]), + ("model.layers.0.post_attention_layernorm.weight", [2]), + ("model.layers.0.self_attn.q_norm.weight", [2]), + ("model.layers.0.self_attn.k_norm.weight", [2]), + ("model.layers.0.self_attn.q_proj.weight", [2, 2]), + ("model.layers.0.self_attn.k_proj.weight", [2, 2]), + ("model.layers.0.self_attn.v_proj.weight", [2, 2]), + ("model.layers.0.self_attn.o_proj.weight", [2, 2]), + ("model.layers.0.mlp.gate_proj.weight", [3, 2]), + ("model.layers.0.mlp.up_proj.weight", [3, 2]), + ("model.layers.0.mlp.down_proj.weight", [2, 3]), + ].filter { $0.0 != omittedName } + + var offset = 0 + var header: [String: Any] = [:] + for (name, shape) in tensors { + let byteCount = shape.reduce(1, *) * 2 + header[name] = [ + "dtype": dtype, + "shape": shape, + "data_offsets": [offset, offset + byteCount], + ] + offset += byteCount + } + var headerData = try JSONSerialization.data(withJSONObject: header, options: [.sortedKeys]) + headerData.append(contentsOf: repeatElement(0x20, count: (8 - headerData.count % 8) % 8)) + var headerLength = UInt64(headerData.count).littleEndian + var file = withUnsafeBytes(of: &headerLength) { Data($0) } + file.append(headerData) + file.append(Data(repeating: 0, count: offset)) + try file.write(to: directory.appendingPathComponent("model.safetensors")) + return directory +} + +func writeSyntheticQwen3Config( + to directory: URL, + qHeads: Int = 1, + kvHeads: Int = 1 +) throws { + let config: [String: Any] = [ + "model_type": "qwen3", + "hidden_size": 2, + "intermediate_size": 3, + "num_hidden_layers": 1, + "num_attention_heads": qHeads, + "num_key_value_heads": kvHeads, + "head_dim": 2, + "vocab_size": syntheticANEModelVocabularySize, + "max_position_embeddings": 16, + "tie_word_embeddings": true, + ] + try JSONSerialization.data(withJSONObject: config, options: [.sortedKeys]) + .write(to: directory.appendingPathComponent("config.json")) +} + +func rewriteSyntheticTokenizer( + at directory: URL, + mutate: (inout [String: Any]) -> Void +) throws { + var tokenizer = syntheticTokenizer() + mutate(&tokenizer) + try writeJSON(tokenizer, to: directory.appendingPathComponent("tokenizer.json")) +} + +func rewriteSyntheticTokenizerConfig( + at directory: URL, + mutate: (inout [String: Any]) -> Void +) throws { + var config = syntheticTokenizerConfig() + mutate(&config) + try writeJSON(config, to: directory.appendingPathComponent("tokenizer_config.json")) +} + +private func writeSyntheticTokenizer(to directory: URL) throws { + try writeJSON(syntheticTokenizer(), to: directory.appendingPathComponent("tokenizer.json")) +} + +private func writeSyntheticTokenizerConfig(to directory: URL) throws { + try writeJSON( + syntheticTokenizerConfig(), + to: directory.appendingPathComponent("tokenizer_config.json") + ) +} + +private func syntheticTokenizer() -> [String: Any] { + var vocabulary = Dictionary( + uniqueKeysWithValues: byteLevelAlphabet().enumerated().map { ($0.element, $0.offset) } + ) + vocabulary["of"] = 256 + return [ + "version": "1.0", + "added_tokens": [ + ["id": 257, "content": "", "special": true], + ["id": 258, "content": "", "special": true], + ["id": 259, "content": "<|im_start|>", "special": true], + ["id": 260, "content": "<|im_end|>", "special": true], + ["id": 261, "content": "", "special": true], + ], + "model": [ + "type": "BPE", + "vocab": vocabulary, + "merges": [["o", "f"]], + "unk_token": "", + ], + "normalizer": ["type": "NFC"], + "pre_tokenizer": [ + "type": "Sequence", + "pretokenizers": [ + ["type": "Split", "pattern": ["String": " "], "invert": false], + ["type": "ByteLevel", "add_prefix_space": false, "use_regex": false], + ], + ], + "post_processor": ["type": "ByteLevel", "use_regex": false], + "decoder": ["type": "ByteLevel", "use_regex": false], + ] +} + +private func syntheticTokenizerConfig() -> [String: Any] { + [ + "tokenizer_class": "Qwen2Tokenizer", + "bos_token": "", + "eos_token": "<|im_end|>", + "unk_token": "", + "pad_token": "", + "model_max_length": 128, + ] +} + +private func byteLevelAlphabet() -> [String] { + var bytes = Array(33...126) + Array(161...172) + Array(174...255) + var codePoints = bytes + for byte in 0...255 where !bytes.contains(byte) { + bytes.append(byte) + codePoints.append(256 + codePoints.count - 188) + } + return codePoints.compactMap(UnicodeScalar.init).map(String.init) +} + +private func writeJSON(_ object: [String: Any], to url: URL) throws { + try JSONSerialization.data(withJSONObject: object, options: [.sortedKeys]) + .write(to: url) +} diff --git a/Tests/OpenTypeTests/ANELMRuntimeTests.swift b/Tests/OpenTypeTests/ANELMRuntimeTests.swift new file mode 100644 index 0000000..40dce3e --- /dev/null +++ b/Tests/OpenTypeTests/ANELMRuntimeTests.swift @@ -0,0 +1,228 @@ +import Foundation +import XCTest +@testable import OpenType + +final class ANELMRuntimeTests: XCTestCase { + func testModelValidationRejectsTruncatedMissingAndWrongDtypeWeights() async throws { + let valid = try makeSyntheticQwen3Directory() + defer { try? FileManager.default.removeItem(at: valid) } + try await EspressoLLMEngine.validateModelDirectory(at: valid) + + let truncated = try makeSyntheticQwen3Directory() + defer { try? FileManager.default.removeItem(at: truncated) } + try Data([1]).write(to: truncated.appendingPathComponent("model.safetensors")) + await XCTAssertThrowsErrorAsync(try await EspressoLLMEngine.validateModelDirectory(at: truncated)) + + let missing = try makeSyntheticQwen3Directory(omitting: "model.layers.0.self_attn.q_proj.weight") + defer { try? FileManager.default.removeItem(at: missing) } + await XCTAssertThrowsErrorAsync(try await EspressoLLMEngine.validateModelDirectory(at: missing)) + + let wrongDtype = try makeSyntheticQwen3Directory(dtype: "F16") + defer { try? FileManager.default.removeItem(at: wrongDtype) } + await XCTAssertThrowsErrorAsync(try await EspressoLLMEngine.validateModelDirectory(at: wrongDtype)) + } + + func testModelValidationRejectsUnsupportedConfigAndMissingTokenizer() async throws { + let directory = try makeSyntheticQwen3Directory() + defer { try? FileManager.default.removeItem(at: directory) } + + try Data(#"{"model_type":"llama"}"#.utf8) + .write(to: directory.appendingPathComponent("config.json")) + await XCTAssertThrowsErrorAsync(try await EspressoLLMEngine.validateModelDirectory(at: directory)) + + try writeSyntheticQwen3Config(to: directory) + try FileManager.default.removeItem(at: directory.appendingPathComponent("tokenizer.json")) + await XCTAssertThrowsErrorAsync(try await EspressoLLMEngine.validateModelDirectory(at: directory)) + + let invalidTokenizer = try makeSyntheticQwen3Directory() + defer { try? FileManager.default.removeItem(at: invalidTokenizer) } + try Data("{}".utf8).write(to: invalidTokenizer.appendingPathComponent("tokenizer.json")) + await XCTAssertThrowsErrorAsync( + try await EspressoLLMEngine.validateModelDirectory(at: invalidTokenizer) + ) + + let unsafeTokenizer = try makeSyntheticQwen3Directory() + defer { try? FileManager.default.removeItem(at: unsafeTokenizer) } + try rewriteSyntheticTokenizer(at: unsafeTokenizer) { + $0["normalizer"] = ["type": "Bogus"] + } + await XCTAssertThrowsErrorAsync( + try await EspressoLLMEngine.validateModelDirectory(at: unsafeTokenizer) + ) + + let outOfRangeTokenizer = try makeSyntheticQwen3Directory() + defer { try? FileManager.default.removeItem(at: outOfRangeTokenizer) } + try rewriteSyntheticTokenizer(at: outOfRangeTokenizer) { + var addedTokens = $0["added_tokens"] as? [[String: Any]] ?? [] + addedTokens[0]["id"] = syntheticANEModelVocabularySize + $0["added_tokens"] = addedTokens + } + await XCTAssertThrowsErrorAsync( + try await EspressoLLMEngine.validateModelDirectory(at: outOfRangeTokenizer) + ) + + let missingUnknownToken = try makeSyntheticQwen3Directory() + defer { try? FileManager.default.removeItem(at: missingUnknownToken) } + try rewriteSyntheticTokenizerConfig(at: missingUnknownToken) { + $0["unk_token"] = "" + } + await XCTAssertThrowsErrorAsync( + try await EspressoLLMEngine.validateModelDirectory(at: missingUnknownToken) + ) + + let missingChatToken = try makeSyntheticQwen3Directory() + defer { try? FileManager.default.removeItem(at: missingChatToken) } + try rewriteSyntheticTokenizer(at: missingChatToken) { + var addedTokens = $0["added_tokens"] as? [[String: Any]] ?? [] + addedTokens[2]["content"] = "" + $0["added_tokens"] = addedTokens + } + await XCTAssertThrowsErrorAsync( + try await EspressoLLMEngine.validateModelDirectory(at: missingChatToken) + ) + + let nonSpecialChatToken = try makeSyntheticQwen3Directory() + defer { try? FileManager.default.removeItem(at: nonSpecialChatToken) } + try rewriteSyntheticTokenizer(at: nonSpecialChatToken) { + var addedTokens = $0["added_tokens"] as? [[String: Any]] ?? [] + addedTokens[3]["special"] = false + $0["added_tokens"] = addedTokens + } + await XCTAssertThrowsErrorAsync( + try await EspressoLLMEngine.validateModelDirectory(at: nonSpecialChatToken) + ) + + let wrongEndToken = try makeSyntheticQwen3Directory() + defer { try? FileManager.default.removeItem(at: wrongEndToken) } + try rewriteSyntheticTokenizerConfig(at: wrongEndToken) { + $0["eos_token"] = "" + } + await XCTAssertThrowsErrorAsync( + try await EspressoLLMEngine.validateModelDirectory(at: wrongEndToken) + ) + + let invalidByteToken = try makeSyntheticQwen3Directory() + defer { try? FileManager.default.removeItem(at: invalidByteToken) } + try rewriteSyntheticTokenizer(at: invalidByteToken) { + var model = $0["model"] as? [String: Any] ?? [:] + var vocabulary = model["vocab"] as? [String: Any] ?? [:] + vocabulary["☃"] = vocabulary.removeValue(forKey: "!") ?? 0 + model["vocab"] = vocabulary + $0["model"] = model + } + await XCTAssertThrowsErrorAsync( + try await EspressoLLMEngine.validateModelDirectory(at: invalidByteToken) + ) + + let invalidHeads = try makeSyntheticQwen3Directory() + defer { try? FileManager.default.removeItem(at: invalidHeads) } + try writeSyntheticQwen3Config(to: invalidHeads, qHeads: 3, kvHeads: 2) + await XCTAssertThrowsErrorAsync(try await EspressoLLMEngine.validateModelDirectory(at: invalidHeads)) + } + + func testRealGenerationLifecycleWhenModelIsProvided() async throws { + guard let modelPath = ProcessInfo.processInfo.environment["UTTER_ANE_TEST_MODEL"], + !modelPath.isEmpty else { + throw XCTSkip("Set UTTER_ANE_TEST_MODEL to a complete local Qwen3 model directory") + } + + let iterations = Int( + ProcessInfo.processInfo.environment["UTTER_ANE_TEST_ITERATIONS"] ?? "20" + ) ?? 20 + XCTAssertGreaterThanOrEqual(iterations, 20) + let baselineResidentSize = try residentSizeKB() + let engine = EspressoLLMEngine() + var residentSamples: [Int] = [] + var unloadMinima: [Int] = [] + var lifecycleGrowth: [Int] = [] + let lifecycleCount = 3 + + for lifecycle in 0.. Int { + let process = Process() + let pipe = Pipe() + process.executableURL = URL(fileURLWithPath: "/bin/ps") + process.arguments = ["-o", "rss=", "-p", String(ProcessInfo.processInfo.processIdentifier)] + process.standardOutput = pipe + try process.run() + process.waitUntilExit() + let data = pipe.fileHandleForReading.readDataToEndOfFile() + let value = String(decoding: data, as: UTF8.self) + .trimmingCharacters(in: .whitespacesAndNewlines) + guard process.terminationStatus == 0, let size = Int(value) else { + throw NSError(domain: "ANELMRuntimeTests", code: 1) + } + return size + } +} + +private func XCTAssertThrowsErrorAsync( + _ expression: @autoclosure () async throws -> Void, + file: StaticString = #filePath, + line: UInt = #line +) async { + do { + try await expression() + XCTFail("Expected async expression to throw", file: file, line: line) + } catch { } +} diff --git a/Tests/OpenTypeTests/ConfigurationTests.swift b/Tests/OpenTypeTests/ConfigurationTests.swift index cbdd20b..508faea 100644 --- a/Tests/OpenTypeTests/ConfigurationTests.swift +++ b/Tests/OpenTypeTests/ConfigurationTests.swift @@ -66,12 +66,47 @@ final class ConfigurationTests: XCTestCase { @MainActor func testFormattingModelTypesPutRecommendedQwenFirstAndCustomLast() { XCTAssertEqual(FormattingModelType.allCases.map(\.rawValue), [ - "qwen", "gemma", "llama", "remote", "custom", + "qwen", "gemma", "llama", "espresso", "remote", "custom", ]) XCTAssertTrue(FormattingModelType.qwen.isRecommended) XCTAssertTrue(FormattingModelType.allCases.dropFirst().allSatisfy { !$0.isRecommended }) } + @MainActor + func testFormattingModelBackendChangePreservesExplicitTypeSelection() { + let activeGemma: ModelCatalog.ModelFamily?? = .some(.gemma) + XCTAssertEqual( + FormattingModelType.resolvedLocalSelection( + pending: .qwen, + activeFamily: activeGemma + ), + .qwen + ) + XCTAssertEqual( + FormattingModelType.resolvedLocalSelection( + pending: .custom, + activeFamily: activeGemma + ), + .custom + ) + + let activeCustom: ModelCatalog.ModelFamily?? = .some(nil) + XCTAssertEqual( + FormattingModelType.resolvedLocalSelection( + pending: nil, + activeFamily: activeCustom + ), + .custom + ) + XCTAssertEqual( + FormattingModelType.resolvedLocalSelection( + pending: nil, + activeFamily: nil + ), + .qwen + ) + } + func testQwenASRDefaultUsesNativeCompatibleModel() { XCTAssertEqual(QwenASRModel.defaultID, "mlx-community/Qwen3-ASR-1.7B-bf16") } @@ -279,6 +314,46 @@ final class ConfigurationTests: XCTestCase { XCTAssertFalse(settings.developerInterfaceEnabled) } + func testLocalLLMBackendDefaultsToMLXAndPersistsEspressoSelection() { + let (defaults, suiteName) = makeIsolatedDefaults() + defer { defaults.removePersistentDomain(forName: suiteName) } + + let settings = AppSettings(defaults: defaults) + XCTAssertEqual(settings.localLLMBackend, .mlx) + XCTAssertTrue(settings.espressoModelPath.isEmpty) + + settings.localLLMBackend = .espresso + settings.espressoModelPath = "/tmp/Qwen3-0.6B" + + let reloaded = AppSettings(defaults: defaults) + XCTAssertEqual(reloaded.localLLMBackend, .espresso) + XCTAssertEqual(reloaded.espressoModelPath, "/tmp/Qwen3-0.6B") + } + + func testEspressoPromptUsesQwenChatTemplate() { + let prompt = EspressoLLMEngine.formatPrompt( + user: "整理这句话", + system: "只输出结果", + modelName: "Qwen2.5-0.5B-Instruct" + ) + + XCTAssertEqual( + prompt, + "<|im_start|>system\n只输出结果<|im_end|>\n" + + "<|im_start|>user\n整理这句话<|im_end|>\n" + + "<|im_start|>assistant\n" + ) + } + + func testEspressoRuntimeFailureSuggestsMLXFallback() { + XCTAssertTrue( + Loc.string("error.espresso_runtime_failed", language: .english).contains("MLX") + ) + XCTAssertTrue( + Loc.string("error.espresso_runtime_failed", language: .chinese).contains("MLX") + ) + } + func testDeveloperHTTPTokenCanBeReset() { let (defaults, suiteName) = makeIsolatedDefaults() defer { defaults.removePersistentDomain(forName: suiteName) } diff --git a/Tests/OpenTypeTests/EspressoFallbackTests.swift b/Tests/OpenTypeTests/EspressoFallbackTests.swift new file mode 100644 index 0000000..f13dca4 --- /dev/null +++ b/Tests/OpenTypeTests/EspressoFallbackTests.swift @@ -0,0 +1,258 @@ +import XCTest +import MLX +@testable import OpenType + +final class EspressoFallbackTests: XCTestCase { + private actor AsyncGate { + private var isOpen = false + private var continuation: CheckedContinuation? + + func wait() async { + guard !isOpen else { return } + await withCheckedContinuation { continuation = $0 } + } + + func open() { + isOpen = true + continuation?.resume() + continuation = nil + } + } + + private enum StubError: LocalizedError { + case espresso + case mlx + + var errorDescription: String? { + switch self { + case .espresso: return "espresso failed" + case .mlx: return "mlx failed" + } + } + } + + func testEspressoSuccessDoesNotRunMLXFallback() async throws { + var ranMLX = false + let result = try await TextProcessor.runEspressoWithMLXFallback( + espresso: { "espresso output" }, + mlx: { + ranMLX = true + return "mlx output" + } + ) + + XCTAssertEqual(result.value, "espresso output") + XCTAssertFalse(result.usedMLX) + XCTAssertFalse(ranMLX) + } + + func testEspressoFailureUsesMLXFallback() async throws { + let result = try await TextProcessor.runEspressoWithMLXFallback( + espresso: { throw StubError.espresso }, + mlx: { "mlx output" } + ) + + XCTAssertEqual(result.value, "mlx output") + XCTAssertTrue(result.usedMLX) + } + + func testDisabledFallbackDoesNotRunMLX() async { + var ranMLX = false + + do { + _ = try await TextProcessor.runEspressoWithMLXFallback( + fallbackEnabled: false, + espresso: { throw StubError.espresso }, + mlx: { + ranMLX = true + return "mlx output" + } + ) as (value: String, usedMLX: Bool) + XCTFail("Expected the Espresso failure") + } catch { + XCTAssertEqual(error.localizedDescription, "espresso failed") + XCTAssertFalse(ranMLX) + } + } + + func testDisabledFallbackCancellationDoesNotStartMLX() async { + let espressoStarted = expectation(description: "Espresso started") + let gate = AsyncGate() + var ranMLX = false + let task = Task { + try await TextProcessor.runEspressoWithMLXFallback( + fallbackEnabled: false, + espresso: { + espressoStarted.fulfill() + await gate.wait() + return "espresso output" + }, + mlx: { + ranMLX = true + return "mlx output" + } + ) + } + + await fulfillment(of: [espressoStarted], timeout: 1) + task.cancel() + await gate.open() + + do { + _ = try await task.value + XCTFail("Expected cancellation") + } catch is CancellationError { + XCTAssertFalse(ranMLX) + } catch { + XCTFail("Unexpected error: \(error)") + } + } + + func testEspressoAndMLXFailuresPreserveBothDiagnostics() async { + do { + _ = try await TextProcessor.runEspressoWithMLXFallback( + espresso: { throw StubError.espresso }, + mlx: { throw StubError.mlx } + ) as (value: String, usedMLX: Bool) + XCTFail("Expected both local backends to fail") + } catch let error as TextProcessor.EspressoMLXFallbackError { + XCTAssertTrue(error.details.contains("espresso failed")) + XCTAssertTrue(error.details.contains("mlx failed")) + } catch { + XCTFail("Unexpected error: \(error)") + } + } + + func testFallbackOutcomeSwitchesPersistedBackendToMLX() { + let suiteName = "EspressoFallbackTests.\(UUID().uuidString)" + let defaults = UserDefaults(suiteName: suiteName)! + defer { defaults.removePersistentDomain(forName: suiteName) } + let settings = AppSettings(defaults: defaults) + settings.localLLMBackend = .espresso + + XCTAssertTrue(EspressoFallbackPolicy.selectMLXIfNeeded( + after: .fallback, + settings: settings, + expectedEspressoModelPath: settings.espressoModelPath + )) + XCTAssertEqual(settings.localLLMBackend, .mlx) + XCTAssertEqual(defaults.string(forKey: "localLLMBackend"), LocalLLMBackend.mlx.rawValue) + } + + func testFallbackOutcomePreservesNewerEspressoSelection() { + let suiteName = "EspressoFallbackTests.\(UUID().uuidString)" + let defaults = UserDefaults(suiteName: suiteName)! + defer { defaults.removePersistentDomain(forName: suiteName) } + let settings = AppSettings(defaults: defaults) + settings.localLLMBackend = .espresso + settings.espressoModelPath = "/models/Qwen3-new" + + XCTAssertFalse(EspressoFallbackPolicy.selectMLXIfNeeded( + after: .fallback, + settings: settings, + expectedEspressoModelPath: "/models/Qwen3-old" + )) + XCTAssertEqual(settings.localLLMBackend, .espresso) + } + + func testDisabledFallbackPreservesEspressoSelection() { + let suiteName = "EspressoFallbackTests.\(UUID().uuidString)" + let defaults = UserDefaults(suiteName: suiteName)! + defer { defaults.removePersistentDomain(forName: suiteName) } + let settings = AppSettings(defaults: defaults) + XCTAssertTrue(settings.fallbackToMLXOnEspressoFailure) + settings.localLLMBackend = .espresso + settings.fallbackToMLXOnEspressoFailure = false + let reloaded = AppSettings(defaults: defaults) + XCTAssertFalse(reloaded.fallbackToMLXOnEspressoFailure) + + XCTAssertFalse(EspressoFallbackPolicy.selectMLXIfNeeded( + after: .fallback, + settings: reloaded, + expectedEspressoModelPath: reloaded.espressoModelPath + )) + XCTAssertEqual(reloaded.localLLMBackend, .espresso) + } + + func testLocalizedFallbackMessagesMentionMLX() { + for language in [UILanguage.english, .chinese] { + XCTAssertTrue(Loc.string("status.espresso_fell_back_to_mlx", language: language).contains("MLX")) + XCTAssertTrue(Loc.string("error.espresso_mlx_fallback_unavailable", language: language).contains("MLX")) + XCTAssertFalse(Loc.string("model.espresso.auto_fallback", language: language).isEmpty) + } + } + + func testRealANEFailureFallsBackToInstalledMLX() async throws { + let environment = ProcessInfo.processInfo.environment + guard environment["OPENTYPE_ANE_MLX_FALLBACK_INTEGRATION"] == "1" else { + throw XCTSkip("Set OPENTYPE_ANE_MLX_FALLBACK_INTEGRATION=1 to run") + } + guard let modelPath = environment["OPENTYPE_ANE_FAILURE_MODEL"], + let mlxModel = environment["OPENTYPE_MLX_MODEL"] else { + throw XCTSkip("Set OPENTYPE_ANE_FAILURE_MODEL and OPENTYPE_MLX_MODEL") + } + + var options = TextProcessingOptions(settings: AppSettings.shared, inputLanguage: .english) + options.useRemoteLLM = false + options.localLLMBackend = .espresso + options.espressoModelPath = modelPath + options.llmModel = mlxModel + + let processor = TextProcessor() + let initialFootprint = currentMemoryFootprint() + let iterations = Int(environment["OPENTYPE_FALLBACK_ITERATIONS"] ?? "1") ?? 1 + var output = "" + var outcome: EspressoGenerationOutcome? + var baselineFootprint: UInt64? + try await TextProcessor.withEspressoOutcomeTracking { + for index in 0.. 1, let baselineFootprint { + let currentFootprint = currentMemoryFootprint() + let growth = currentFootprint > baselineFootprint + ? currentFootprint - baselineFootprint + : 0 + XCTAssertLessThan(growth, 384 * 1_024 * 1_024, "Repeated MLX requests retained \(growth) bytes") + } + await processor.unloadLLM() + try await Task.sleep(for: .milliseconds(500)) + XCTAssertLessThan(Memory.activeMemory, 1 * 1_024 * 1_024) + XCTAssertEqual(Memory.cacheMemory, 0) + let footprintAfterUnload = currentMemoryFootprint() + let retainedAfterUnload = footprintAfterUnload > initialFootprint + ? footprintAfterUnload - initialFootprint + : 0 + XCTAssertLessThan( + retainedAfterUnload, + 1_024 * 1_024 * 1_024, + "Local model unload retained \(retainedAfterUnload) bytes" + ) + } +} + +private func currentMemoryFootprint() -> UInt64 { + var info = task_vm_info_data_t() + var count = mach_msg_type_number_t(MemoryLayout.size / MemoryLayout.size) + let result = withUnsafeMutablePointer(to: &info) { + $0.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { + task_info(mach_task_self_, task_flavor_t(TASK_VM_INFO), $0, &count) + } + } + return result == KERN_SUCCESS ? info.phys_footprint : 0 +} diff --git a/Tests/OpenTypeTests/EspressoOutcomeTests.swift b/Tests/OpenTypeTests/EspressoOutcomeTests.swift new file mode 100644 index 0000000..8435678 --- /dev/null +++ b/Tests/OpenTypeTests/EspressoOutcomeTests.swift @@ -0,0 +1,73 @@ +import XCTest +@testable import OpenType + +final class EspressoOutcomeTests: XCTestCase { + private final class WeakTrackerBox { + weak var value: EspressoGenerationTracker? + } + + func testOutcomeTrackingUsesLatestOperationAndIsRequestScoped() async { + let processor = TextProcessor() + var outcomes: [EspressoGenerationOutcome?] = [] + + await TextProcessor.withEspressoOutcomeTracking { + await TextProcessor.recordEspressoOutcome(.fallback) + await TextProcessor.recordEspressoOutcome(.failed) + outcomes.append(await processor.consumeEspressoOutcome()) + } + await TextProcessor.withEspressoOutcomeTracking { + await TextProcessor.recordEspressoOutcome(.failed) + await TextProcessor.recordEspressoOutcome(.fallback) + outcomes.append(await processor.consumeEspressoOutcome()) + } + await TextProcessor.withEspressoOutcomeTracking { + outcomes.append(await processor.consumeEspressoOutcome()) + } + for staleOutcome in [EspressoGenerationOutcome.fallback, .failed] { + await TextProcessor.withEspressoOutcomeTracking { + await TextProcessor.recordEspressoOutcome(staleOutcome) + await TextProcessor.clearEspressoOutcome() + outcomes.append(await processor.consumeEspressoOutcome()) + } + } + + XCTAssertEqual(outcomes[0], .failed) + XCTAssertEqual(outcomes[1], .fallback) + XCTAssertNil(outcomes[2]) + XCTAssertNil(outcomes[3]) + XCTAssertNil(outcomes[4]) + } + + func testRemoteAttemptClearsEarlierEspressoOutcome() async { + let processor = TextProcessor() + var options = TextProcessingOptions(settings: AppSettings.shared) + options.useRemoteLLM = true + options.remoteAPIKey = "" + var outcome: EspressoGenerationOutcome? + + await TextProcessor.withEspressoOutcomeTracking { + await TextProcessor.recordEspressoOutcome(.fallback) + _ = try? await processor.generateText( + prompt: "test", + systemPrompt: "test", + options: options, + maxTokens: 1, + temperature: 0 + ) + outcome = await processor.consumeEspressoOutcome() + } + + XCTAssertNil(outcome) + } + + func testOutcomeTrackerIsReleasedAfterRequestCompletes() async { + let box = WeakTrackerBox() + + await TextProcessor.withEspressoOutcomeTracking { + box.value = TextProcessor.espressoGenerationTracker + XCTAssertNotNil(box.value) + } + + XCTAssertNil(box.value) + } +} diff --git a/Tests/OpenTypeTests/IntegrationHTTPTests.swift b/Tests/OpenTypeTests/IntegrationHTTPTests.swift index 641256d..bfbb822 100644 --- a/Tests/OpenTypeTests/IntegrationHTTPTests.swift +++ b/Tests/OpenTypeTests/IntegrationHTTPTests.swift @@ -106,6 +106,10 @@ final class IntegrationHTTPTests: XCTestCase { XCTAssertEqual(IntegrationHTTPDispatcher.statusCode(for: IntegrationError.modelNotReady), 400) XCTAssertEqual(IntegrationHTTPDispatcher.statusCode(for: IntegrationError.noSpeechDetected), 400) XCTAssertEqual(IntegrationHTTPDispatcher.statusCode(for: IntegrationError.operationFailed), 500) + XCTAssertEqual( + IntegrationHTTPDispatcher.statusCode(for: IntegrationError.operationFailedWithMessage("details")), + 500 + ) } @MainActor diff --git a/Tests/OpenTypeTests/IntegrationOutputTests.swift b/Tests/OpenTypeTests/IntegrationOutputTests.swift index ff638ec..fbf7733 100644 --- a/Tests/OpenTypeTests/IntegrationOutputTests.swift +++ b/Tests/OpenTypeTests/IntegrationOutputTests.swift @@ -5,6 +5,12 @@ import XCTest @MainActor final class IntegrationOutputTests: XCTestCase { + func testAppDelegateSharesTextProcessorWithIntegrationCoordinator() { + let delegate = AppDelegate() + + XCTAssertTrue(delegate.integrationSessionCoordinator.textProcessor === delegate.textProcessor) + } + func testServiceRejectsEmptyFinalText() async throws { let store = registry() defer { store.cleanup() } @@ -41,6 +47,38 @@ final class IntegrationOutputTests: XCTestCase { _ = try await coordinator.outputText(for: " ", active: active) } } + + func testCoordinatorPreservesCombinedLocalModelFailureGuidance() async { + let store = registry() + defer { store.cleanup() } + store.registry.approve(IntegrationClient.localHTTP(tokenID: "token")) + let coordinator = InputSessionCoordinator(service: makeService(registry: store.registry)) + let active = InputSessionCoordinator.ActiveSession( + sessionID: UUID(), + clientID: clientID, + engine: TestSpeechEngine(transcript: ""), + languageCode: nil, + mode: .direct, + inputLanguage: .english, + useScreenContext: false, + streamingEnabled: false, + screenContextTask: nil, + client: IntegrationClient.localHTTP(tokenID: "token") + ) + + do { + try await TextProcessor.withEspressoOutcomeTracking { + await TextProcessor.recordEspressoOutcome(.unavailable) + _ = try await coordinator.outputText(for: " ", active: active) + } + XCTFail("Expected local model failure") + } catch let error as IntegrationError { + XCTAssertEqual(error, .operationFailedWithMessage(EspressoGenerationOutcome.unavailable.message)) + XCTAssertEqual(error.payload.error, "operation_failed") + } catch { + XCTFail("Unexpected error: \(error)") + } + } } private extension IntegrationOutputTests { diff --git a/Tests/OpenTypeTests/LocalModelAccessTests.swift b/Tests/OpenTypeTests/LocalModelAccessTests.swift new file mode 100644 index 0000000..4dd2719 --- /dev/null +++ b/Tests/OpenTypeTests/LocalModelAccessTests.swift @@ -0,0 +1,110 @@ +import XCTest +@testable import OpenType + +final class LocalModelAccessTests: XCTestCase { + private actor EventRecorder { + private var events: [String] = [] + + func append(_ event: String) { + events.append(event) + } + + func snapshot() -> [String] { + events + } + } + + func testLocalModelAccessIsReentrantWithinOneTask() async throws { + let processor = TextProcessor() + let recorder = EventRecorder() + + try await processor.withLocalModelAccess { + try await processor.withLocalModelAccess { + await recorder.append("nested") + } + } + + let events = await recorder.snapshot() + XCTAssertEqual(events, ["nested"]) + } + + func testUnloadWaitsForActiveLocalModelOperation() async { + let processor = TextProcessor() + let recorder = EventRecorder() + let operationStarted = expectation(description: "Local model operation started") + let unloadStarted = expectation(description: "Unload started") + let (releaseStream, releaseContinuation) = AsyncStream.makeStream() + + let operation = Task { + try await processor.withLocalModelAccess { + await recorder.append("operation-start") + operationStarted.fulfill() + for await _ in releaseStream { break } + await recorder.append("operation-end") + } + } + await fulfillment(of: [operationStarted], timeout: 1) + + let unload = Task { + unloadStarted.fulfill() + await processor.unloadLLM() + await recorder.append("unload-end") + } + await fulfillment(of: [unloadStarted], timeout: 1) + try? await Task.sleep(for: .milliseconds(25)) + let eventsWhileOperationIsActive = await recorder.snapshot() + XCTAssertEqual(eventsWhileOperationIsActive, ["operation-start"]) + + releaseContinuation.yield() + releaseContinuation.finish() + try? await operation.value + await unload.value + + let finalEvents = await recorder.snapshot() + XCTAssertEqual(finalEvents, ["operation-start", "operation-end", "unload-end"]) + } + + func testCancelledLocalModelWaiterDoesNotRunAfterGateOpens() async { + let processor = TextProcessor() + let recorder = EventRecorder() + let holderStarted = expectation(description: "Gate holder started") + let (releaseStream, releaseContinuation) = AsyncStream.makeStream() + + let holder = Task { + try await processor.withLocalModelAccess { + holderStarted.fulfill() + for await _ in releaseStream { break } + } + } + await fulfillment(of: [holderStarted], timeout: 1) + + let waiter = Task { + try await processor.withLocalModelAccess { + await recorder.append("waiter-ran") + } + } + for _ in 0..<1_000 { + if await processor.localModelAccessGate.waitingTaskCount == 1 { break } + await Task.yield() + } + let queuedWaiters = await processor.localModelAccessGate.waitingTaskCount + XCTAssertEqual(queuedWaiters, 1) + waiter.cancel() + + do { + try await waiter.value + XCTFail("Expected cancellation") + } catch is CancellationError { + } catch { + XCTFail("Unexpected error: \(error)") + } + let remainingWaiters = await processor.localModelAccessGate.waitingTaskCount + XCTAssertEqual(remainingWaiters, 0) + + releaseContinuation.yield() + releaseContinuation.finish() + try? await holder.value + let events = await recorder.snapshot() + XCTAssertTrue(events.isEmpty) + } +} diff --git a/Tests/OpenTypeTests/OverlayLayoutTests.swift b/Tests/OpenTypeTests/OverlayLayoutTests.swift index f0f158c..9ab9541 100644 --- a/Tests/OpenTypeTests/OverlayLayoutTests.swift +++ b/Tests/OpenTypeTests/OverlayLayoutTests.swift @@ -48,6 +48,28 @@ final class OverlayLayoutTests: XCTestCase { } } + func testEspressoFallbackCompletionMakesRoomForTwoLineStatus() { + let appState = AppState() + appState.phase = .done + appState.statusMessage = L("status.espresso_fell_back_to_mlx") + appState.completionKind = .espressoFallback + + let layout = OverlayLayout(appState: appState) + + XCTAssertEqual(layout.width, 288) + XCTAssertEqual(layout.height, 56) + XCTAssertFalse(layout.isInteractive) + } + + func testFallbackLayoutDoesNotDependOnLocalizedMessageText() { + let appState = AppState() + appState.phase = .done + appState.statusMessage = "Changed copy" + appState.completionKind = .espressoFallback + + XCTAssertEqual(OverlayLayout(appState: appState).height, 56) + } + func testOverlayPlacementCentersAboveVisibleScreenBottom() { let visibleFrame = CGRect(x: 100, y: 80, width: 1_200, height: 760) let frame = OverlayPanelPlacement.frame( diff --git a/Tests/OpenTypeTests/TextProcessorFallbackTests.swift b/Tests/OpenTypeTests/TextProcessorFallbackTests.swift index e8673ab..6f0b536 100644 --- a/Tests/OpenTypeTests/TextProcessorFallbackTests.swift +++ b/Tests/OpenTypeTests/TextProcessorFallbackTests.swift @@ -191,4 +191,5 @@ final class TextProcessorFallbackTests: XCTestCase { XCTAssertFalse(formattingPrompt.contains("A screen image is attached")) XCTAssertFalse(commandPrompt.contains("A screen image is attached")) } + } diff --git a/docs/research/2026-08-31-apple-neural-engine-m5-compatibility.md b/docs/research/2026-08-31-apple-neural-engine-m5-compatibility.md new file mode 100644 index 0000000..878e8e5 --- /dev/null +++ b/docs/research/2026-08-31-apple-neural-engine-m5-compatibility.md @@ -0,0 +1,208 @@ +# M5 / macOS 27 上的 Apple Neural Engine 兼容性调查 + +日期：2026-08-31 +范围：Utter 当前 Espresso 后端、Apple 公开的 Core ML / Core AI 路径，以及可直接访问 ANE 的开源项目。这里的“M5”包含本次实际可用的 M5 Max；没有把仅有项目自述、但没有设备和系统版本证据的支持声明当作已验证事实。 + +> 证据标记：**[事实]** 可由 Apple 文档、上游仓库或本机复现实验直接核对；**[推断]** 是基于这些事实的工程判断；**[未确认]** 仍需针对 Utter 的模型和发布形态验证。 + +## 结论 + +**Apple 没有在 M5 或 macOS 27 上彻底封堵 ANE。** 当前证据反而能排除“硬件被关闭”或“普通进程一律无权访问”这两种解释： + +- **[事实] 公开路径仍在。** Apple 当前的 [Core ML 文档](https://developer.apple.com/documentation/coreml)仍明确写明会利用 CPU、GPU 和 Neural Engine；[`MLComputeUnits`](https://developer.apple.com/documentation/coreml/mlcomputeunits)仍公开提供 `.all` 和 `.cpuAndNeuralEngine`。macOS 27 新增的 [Core AI](https://developer.apple.com/documentation/coreai)也明确面向 CPU、GPU 和 Neural Engine。 +- **[事实] 同一台 M5 Max / macOS 27.0 机器上，私有路径也能工作。** 本次从未修改的 [`maderix/ANE`](https://github.com/maderix/ANE) 源码编译运行了十组 `_ANEInMemoryModel` 程序、RMSNorm、32K classifier、softmax 和 classifier backward，全部成功。随后 [`ANEForge`](https://github.com/sbryngelson/ANEForge) 的另一条私有 `e5rt` 编译/执行路径也在同机跑通卷积、CNN 和 transformer encoder block。 +- **[事实] Utter 当前 Espresso 仍失败。** 当前 M5 Max / macOS 27 主机对 Espresso 的 24 组 MIL 组合全部返回 Apple 私有编译器 Code 10，真实 GPT-2 `.esp` 也在 layer 0 attention 失败。详见本仓库的 [验证记录](../sdlc/changes/2026-08-29-espresso-ane/verification.md)。 +- **[事实] Code 10 不是“ANE 被封”的专用错误。** [`oMLX` issue #3124](https://github.com/jundot/omlx/issues/3124)记录了一台 M5 Max / macOS 27 主机上 rc2 可编译、rc3 因给 `_ANEInMemoryModel` 调用 `setModelURL:` 而遭 bundle hash 校验 Code 10；维护者随后提交了[修复](https://github.com/jundot/omlx/commit/7d77098ed83d7abcafeeaffaa4de1491820021d7)。 + +因此，**[推断] Espresso 的失败是其生成的 MIL、权重/bundle 组织或私有运行时调用方式与 M5 Max / macOS 27 校验规则之间的具体不兼容，而不是 Apple 对 ANE 的系统性封锁。** 这也意味着它有修复空间，但不能保证私有 API 下一个系统版本仍稳定。 + +目前存在能在 M5 上直接调用 ANE 的开源实现，但**没有一个未经修改即可替换 Utter 当前 Swift 后端**。本次因此从 ANE-LM 做了窄 fork，只保留 Qwen3 加载、生成、取消、重置和卸载接口，并在 Utter 中继续保留用户可控的 MLX fallback。 + +## 先区分两个容易混淆的“神经加速器” + +- **Apple Neural Engine（ANE）**：独立的 Neural Engine。Core ML、Core AI，以及本报告中的 Espresso、maderix/ANE、ANEForge 私有路径讨论的是它。 +- **M5 GPU core 内的 Neural Accelerator**：属于 GPU/Metal 路径。Apple 的 [M5 Tech Talk](https://developer.apple.com/videos/play/tech-talks/111432/)说明 MLX、llama.cpp、PyTorch 可以通过 Metal 利用这类 GPU 加速器；这不等于它们使用独立 ANE。 + +**[事实]** Apple 官方 [`ml-explore/mlx`](https://github.com/ml-explore/mlx) README 将当前支持设备写为 CPU 和 GPU，没有 ANE backend。Utter 的 MLX fallback 在 M5 上可用，但它不是“换一种方式调用 ANE”。 + +## 证据矩阵 + +| 路径 | API 性质 | 设备 / 系统证据 | 当前结论 | +|---|---|---|---| +| Core ML | Apple 公开 API | Apple 文档持续列出 Neural Engine；FluidAudio 在 M5 Pro / macOS 26.5 做了 100 条 TTS 实测 | 可用，但系统决定每个算子落到 CPU、GPU 还是 ANE | +| Core AI | Apple 公开 API，macOS 27+ | Apple 文档明确跨 CPU、GPU、ANE；macOS 27 release notes 仍在改进大模型 ANE 加载和内存归属 | 是 macOS 27 上自定义现代模型的长期路径 | +| Utter + Espresso | 私有 `_ANEClient` / `_ANEInMemoryModel` | M5 Max / macOS 27，24/24 编译组合及真实 GPT-2 均 Code 10 | 当前不兼容；不能据此推断整个私有 API 被封 | +| maderix/ANE | 私有 `_ANEInMemoryModel` | 本机 M5 Max / macOS 27，十组峰值程序及四个 classifier 测试全部通过 | 直接反证“私有 ANE 已被全面封锁” | +| ANEForge | 私有 `e5rt` | 上游有 M5/M5 Pro + macOS 26.5 数据；本机 M5 Max / macOS 27 CNN/transformer 均通过 | M5 可直接调用 ANE，但项目是 Python 前端和研究运行时 | +| oMLX | 私有 `_ANEInMemoryModel` | issue 报告 M5 Max / macOS 27：rc2 成功，rc3 Code 10；维护者定位到 `setModelURL:` | 证明 Code 10 可由 bundle/staging 细节触发，不是芯片封禁 | +| FluidAudio | 公开 Core ML | M5 Pro/macOS 26.5 与 M5/iPadOS 27 的实际运行和分阶段路由 | ANE 可用；个别 Apple GPU/BNNS/Core ML stage 仍会有 OS 特定 bug | +| ANE-LM | 私有 `_ANEInMemoryModel` | 本机原版真实 Qwen3-0.6B 在 fused QKV Code 10；[`IchenDEV/ANE-LM@033472e`](https://github.com/IchenDEV/ANE-LM/commit/033472ec12ea796fc7ea4f8cefd7ed456f69900b) 已完成 28 层、ANE LM head 和 Utter 产品路径实测 | 已作为精确 revision 集成；仍是实验性私有 API，只确认当前 M5 Max/macOS 27 + Qwen3-0.6B | +| ane-infer | 私有 API | README 声称 M1–M5；本次未做当前主机的真实模型复测 | 可作 Rust 混合运行时参考，不能当作已验证的 drop-in | + +## 公开 API 没有被撤掉 + +### Core ML：macOS 26 及更早兼容面的公开选择 + +**[事实]** Apple 在 [WWDC25 的机器学习框架总览](https://developer.apple.com/videos/play/wwdc2025/360/?time=675)明确说明，Core ML 会在运行时跨 CPU、GPU、Neural Engine 优化执行。`MLComputeUnits.cpuAndNeuralEngine` 允许 CPU 与 ANE、排除 GPU；`.all` 允许系统选择全部可用计算单元。 + +边界也很明确： + +- **[事实]** 公开枚举没有 `neuralEngineOnly`。应用能限制允许的设备集合，不能要求整个模型或每个 op 必须落到 ANE。 +- **[事实]** Apple 公开了 `MLComputePlan` 和 `MLNeuralEngineComputeDevice`，可以检查预期的逐 op 设备使用，但模型能否编译、如何分区仍由系统决定。 +- **[事实]** Apple 自己的 [`coremltools` issue #2687](https://github.com/apple/coremltools/issues/2687)包含 M5 上某些 shape 的 `mish/softplus` 实际路由到 ANE 的报告，同时也显示路由规则和数值边界会随芯片变化。这是“ANE 在工作”与“不是每个图都稳定”的同时证据。 + +### Core AI：macOS 27 的前向路径 + +**[事实]** [Core AI](https://developer.apple.com/documentation/coreai)从 macOS 27 起可用，Apple 将其定位为运行最新模型架构和推理技术的框架，并明确写明会使用 CPU、GPU 和 Neural Engine。Apple 同时提供 [`coreai-models`](https://github.com/apple/coreai-models) 和模型准备工具，不再要求开发者依赖私有 ANE ABI 才能运行现代模型。 + +**[事实]** [macOS 27 release notes](https://developer.apple.com/documentation/macos-release-notes/macos-27-release-notes)提到大模型在 Neural Engine 上的加载改进、ANE 内存归属到应用进程，以及后台 ANE 推理的新 entitlement。它描述的是受支持框架的资源管理和后台限制，不是前台应用被禁止访问 ANE。该 release note 仍是 beta 文档，且其中一条写作 iOS 27，因此不能把每项限制无差别外推到 macOS 前台运行。 + +### 公开路径的真实开源项目证据 + +**[事实]** [`FluidAudio` issue #667](https://github.com/FluidInference/FluidAudio/issues/667)和已合并的 [PR #671](https://github.com/FluidInference/FluidAudio/pull/671)在 M5 Pro / macOS 26.5 上把大部分 Kokoro 阶段固定为 `.cpuAndNeuralEngine`，100 条英文、中文语料均完成；只是把 ANE 不接受或 Apple framework 有 bug 的 tail stage 路由到 GPU。 + +**[事实]** 已合并的 [FluidAudio PR #849](https://github.com/FluidInference/FluidAudio/pull/849)又针对 M5 iPad Pro / OS 27 把有 MPSGraph 问题的 noise/tail 路由到 CPU，RNN 和 vocoder 继续使用 `.cpuAndNeuralEngine`。这说明 OS 27 上需要逐 stage 规避 bug，但不是 ANE 整体不可用。 + +## 私有 API：同机实测排除了全面封锁 + +### Utter 当前 Espresso 的失败边界 + +**[事实]** 当前分支已在 M5 Max / macOS 27 上验证： + +- Espresso 0.9.0 和当时 upstream main 的真实 GPT-2 generation 均在 layer 0 attention 返回 `verifyBundleAtPath: invalid model`、Code 10。 +- iOS 18、iOS 19、macOS 26、macOS 27 四个 MIL target，乘 LayerNorm/RMSNorm、spatial 64/128/256，共 24 组全部失败。 +- 同一请求切换到 MLX 后可以完成，因此失败在 Espresso 私有 ANE 编译/加载路径，而不是 Utter 的通用文本处理流水线。 + +上游 [`Espresso` README](https://github.com/christopherkarani/Espresso)只把 M1–M4 列为已测试，未列 M5。另一方面，[Espresso issue #18](https://github.com/christopherkarani/Espresso/issues/18)显示 M1 Ultra / macOS 26 的真实模型问题后来通过保持 `ios18` target 得到解决，说明“某个系统上的 Code 10/compile failure”需要按 MIL 和调用细节诊断，不能直接归因为 Apple 封锁。 + +### maderix/ANE：同一私有类、同一主机成功 + +本次在 Apple M5 Max、macOS 27.0（26A5421a）上检出 [`maderix/ANE` commit `d91c984`](https://github.com/maderix/ANE/commit/d91c9845c0784dec7753048954fc6d0e8411fe29)，未修改源码： + +- **[事实]** [`inmem_peak.m`](https://github.com/maderix/ANE/blob/main/inmem_peak.m) 的十组 programmatic MIL 均完成 compile、load 和 eval；两次观察到的峰值约为 12.6–12.8 TFLOPS。 +- **[事实]** [`training/test_classifier.m`](https://github.com/maderix/ANE/blob/main/training/test_classifier.m) 完成四次 ANE 编译：RMSNorm 最大误差 `0.002856`；32K classifier projection 最大误差 `0.000831`；32K softmax 最大误差约 `0.000001`；classifier backward 最大误差 `0.006952`，全部判定通过。 +- **[事实]** 这些程序与 Utter 的 Espresso 一样使用 `_ANEInMemoryModelDescriptor`、`_ANEInMemoryModel`、`compileWithQoS`、`loadWithQoS` 和 IOSurface，而不是 Core ML 的公开模型 API。 + +这组结果直接证明，同一台机器上的 `_ANEInMemoryModel` 编译、加载和执行能力仍开放给普通进程。差异落在具体图、权重、bundle 或调用参数，而不是 API 类完全消失或权限被统一拒绝。 + +### ANEForge：另一条私有编译/执行栈也成功 + +**[事实]** [`ANEForge` README](https://github.com/sbryngelson/ANEForge)自述已在 M5 Pro 与 M1 Max 验证；其仓库还保存了 [M5 Pro / macOS 26.5.1](https://github.com/sbryngelson/ANEForge/blob/main/bench/results/rooflines/roofline-apple-m5-pro-Mac17_8-c38210cfc8ea-a154b89e.json)及 [M5 / macOS 26.5.2](https://github.com/sbryngelson/ANEForge/blob/main/bench/results/rooflines/roofline-apple-m5-Mac17_3-019d3ac27339-426ca052.json) 的可机器读取实测数据。 + +本次又在同一台 M5 Max / macOS 27.0 主机上，以未修改的 `ANEForge` commit `a0be5b5` 运行： + +- 最小 1×1 conv 完成编译和执行，输出 shape `(1, 1, 4, 4)`、sum `16.0`。 +- 上游 [`examples/quickstart.py`](https://github.com/sbryngelson/ANEForge/blob/main/examples/quickstart.py) 的 CNN 在 fp16/int8 下相对误差为 `0.0023/0.0112`，均通过。 +- 同一 quickstart 的 RMSNorm + attention + FFN transformer encoder block 在 fp16/int8 下相对误差为 `0.0029/0.0087`，均通过。 + +ANEForge 的 [`ane_e5rt_dispatch.mm`](https://github.com/sbryngelson/ANEForge/blob/main/aneforge/_lib/ane_e5rt_dispatch.mm)使用私有 `e5rt_e5_compiler_*` 与 execution stream 符号。这是与 Espresso `_ANEClient` 路径不同的第二个直接 ANE 反例。 + +但它不是 Utter 的直接依赖候选：其图构建、lowering 和模型前端以 Python 为主，而且首次运行会本地编译 dylib。Utter 若复用，只适合把已验证的窄 C ABI / MIL 生成模式移植进构建期产物，不能在已签名应用内照搬运行时 Python 与动态编译流程。 + +### oMLX：Code 10 的具体回归样本 + +**[事实]** [`oMLX issue #3124`](https://github.com/jundot/omlx/issues/3124)提供了很有辨识度的矩阵： + +- M5 Max / macOS 27 上 rc2 的 ANE procedure path 成功，rc3 在所有 split size 上 Code 10。 +- M5 Pro / macOS 26.6.2 上同一 rc3 可以成功。 +- M4 Max 和 M1 Ultra / macOS 27 也能复现 rc3，故失败不跟 M5 芯片本身绑定。 +- 维护者定位到 rc3 给 `_ANEInMemoryModel` 调用 `setModelURL:`，触发 macOS 27 的 per-file bundle hash 校验；[commit `7d77098`](https://github.com/jundot/omlx/commit/7d77098ed83d7abcafeeaffaa4de1491820021d7)移除了该覆盖。 + +修复已在 macOS 26.5 验证，但 issue 中尚没有原 macOS 27 报告者的最终回测。因此“修复方向”有源码证据，“已在 macOS 27 完全收口”仍属**[未确认]**。 + +### ANE-LM：原版失败，但最小差分 PoC 跑通真实 Qwen + +本次又在同一台 M5 Max / macOS 27.0 主机上检出 [`ANE-LM` commit +`04b1af1`](https://github.com/johnmai-dev/ANE-LM/commit/04b1af12aa30b3f280f511d286398469764b4326)， +使用官方 Qwen3-0.6B safetensors 做了端到端对照： + +- **[事实] 未修改上游版本失败。** 原版可以编译并加载约 1.5 GB、28 层的模型权重，但在 layer 0 + 的 `first_proj` 立即返回 `verifyBundleAtPath: invalid model` / Code 10；关闭 ANE compile + cache、将提示缩到一个 token 后仍稳定复现。 +- **[事实] 失败缩到了 fused-QKV 图。** 同机运行 maderix/ANE 原版 + `test_fused_qkv` 时，“三卷积 + concat”同样 Code 10，而三个独立 Q、K、V 卷积全部编译成功。 + 这排除了 Qwen 权重、投影本身和 M5 权限，指向 macOS 27 私有编译器对该融合图的拒绝。 +- **[事实] 全拆 kernel 可以越过 Code 10，但会在约 126 个程序附近触发 Code 54 + `no ANE resources`。** 因此把每个投影永久拆成独立 model 不是可用方案。 +- **[事实] 临时差分 PoC 最终跑通。** PoC 将 MIL 更新为本机可接受的 + `program(1.3) / ios18` 形式，并在生成 MIL 前按输出通道拼接 Q/K/V 权重，让一个单输出 + convolution 自然产生连续的 Q、K、V 三段；Gate/Up 同样预拼成一个 convolution，激活和逐元素乘法 + 留在 CPU，Down 保持独立 ANE projection。这样每层保持四个 ANE program，共 112 个 layer + kernel；151,936 词表的 LM head 再分成 10 个 ANE chunk。 +- **[事实] 完整中文生成成功。** 27 个提示 token 约为 `21.621 token/s`，22 个生成 token + 约为 `22.597 token/s`，模型初始化约 `6.16 s`。实际输出为： + “苹果神经引擎（Apple Neural Engine）支持本地语言模型的运行，但需要通过特定的方式实现。” +- **[事实] 短内存循环未出现大幅逐轮增长。** 五轮同进程聊天在第一轮后的 RSS 为 + `1,309,760 KB`，随后为 `1,309,872`、`1,309,968`、`1,310,048`、 + `1,310,144 KB`，四轮累计约 `384 KB`。这只排除了明显的每轮大块泄漏，不能替代 + 20–100 轮、取消、卸载和空闲回落验证。 + +随后已将 PoC 收敛进 [`IchenDEV/ANE-LM`](https://github.com/IchenDEV/ANE-LM) 的 SwiftPM C++ library： + +- **[事实]** Utter 固定到不可变 revision `033472ec12ea796fc7ea4f8cefd7ed456f69900b`，不跟随移动分支；Swift 负责 tokenizer/chat template，C ABI 只暴露验证、加载、生成、重置和卸载。 +- **[事实]** 模型保存前会解析 safetensors 头，校验全部 Qwen3 必需 tensor、shape 和 BF16 dtype；原生编译入口也拒绝空权重指针，避免坏目录绕过错误回退直接崩溃。 +- **[事实]** 部分 ANE kernel 初始化失败会统一 unload 私有 model、释放已创建的 IOSurface、数组和 Objective-C 对象，并删除临时目录。 +- **[事实]** Utter 产品路径完成了三次完整加载、生成和卸载，在三轮中共执行 20 次短生成，并在第三轮另行验证了一次取消；每轮生成期间的 RSS 增量分别为 `80 KB`、`32 KB` 和 `0 KB`。fork 将大块临时权重与 embedding 改为可精确 `munmap` 的映射分配后，每轮 unload 相对当轮峰值都释放超过 `512 MiB`。 +- **[事实]** 三次 unload 后采样到的 RSS 低点分别为 `636,512 KB`、`150,928 KB` 和 `662,592 KB`，最终低点仅比第一次高 `26,080 KB`，低于测试强制的 `128 MiB` 跨生命周期上限。该结果没有出现随请求或切换次数单调累积的大块泄漏，但卸载后的 RSS 也没有稳定回到加载前 `54,368 KB` 基线。 +- **[推断]** `vmmap` 和独立 1 GiB malloc 探针说明系统分配器的回收时机会造成一部分波动；现有测试尚未完全隔离 Apple framework 或私有 ANE runtime 是否也保留了内存，因此不能把全部残留确定归因于 allocator。Utter 在 ANE 选项旁只披露“切换后已释放内存可能不会立即从系统监视器中消失”，没有承诺立即回到基线。 + +这把“能不能修”从临时 PoC 升级为当前项目可选择的实验后端，但没有改变私有 API 的发布和跨版本风险。 + +## 其他开源项目能否直接替代 + +### 能证明 M5 可直连，但不能原样嵌入 Utter + +- [`maderix/ANE`](https://github.com/maderix/ANE)：本机已验证 private ANE 可用；它是研究样例和训练 PoC，不是通用 Swift 模型运行时。 +- [`ANEForge`](https://github.com/sbryngelson/ANEForge)：本机已验证 M5 Max / macOS 27；模型覆盖很广，但当前产品形态是 Python frontend + Objective-C++ bridge。 +- [`oMLX`](https://github.com/jundot/omlx)：已有 M5 Max / macOS 27 rc2 成功证据，并给出 Code 10 的修复线索；它不是 Swift Package，也不是 Utter `.esp` bundle 的替代格式。 +- [`ANE-LM`](https://github.com/johnmai-dev/ANE-LM)：原生 C++、MIT。原版 fused-QKV 在当前主机失败；本次的 [`IchenDEV fork`](https://github.com/IchenDEV/ANE-LM) 已整理出 Swift/C ABI 并完成 Qwen3-0.6B 生命周期验证。Qwen3.5 的上游代码没有接入本次 Swift product，因此不能宣称当前 Utter 支持。 +- [`ane-infer`](https://github.com/thebasedcapital/ane-infer)：Rust + Objective-C + Metal 的 Qwen3.5 专用混合引擎，README 声称覆盖 M1–M5；同样缺少本次当前主机的真实模型复测，而且不是通用 Swift 库。 + +[`hollance/neural-engine`](https://github.com/hollance/neural-engine)主要是 ANE 架构与设备资料，不是运行时，不能当作替代实现。 + +### App Store 与长期稳定性边界没有改变 + +上述直接 ANE 项目都调用未公开 framework 或符号。即使当前 M5 能运行，它们仍没有 ABI 稳定承诺，也不适合 App Store 发布。Espresso 的 README 同样明确说明私有 ANE API 会触发 App Store 拒绝。公开 Core ML / Core AI 才是可发布、可获得系统兼容承诺的路径。 + +## Utter 的最小可行技术路径 + +### 已实施的实验路径：ANE-LM + 用户可控 fallback + +Utter 现在允许用户选择普通本地 Qwen3 Hugging Face 目录，由修改后的 ANE-LM 在当前 M5 主机执行；默认允许失败后切换 MLX，用户可关闭。这样不会把私有 API 的一次系统回归变成语音输入不可用，同时保留错误可见性。 + +### 本次私有修复的边界 + +实现只承诺 Qwen3 safetensors 和本次实测的 Qwen3-0.6B。它没有证明 Qwen3.5、其他尺寸、M1–M4、普通 M5/M5 Pro、macOS 26 或后续 macOS 版本可用；这些组合仍需逐项做真实生成、取消、重复内存和卸载验证。即使验证通过，这条私有路径仍只适合实验、自签名或侧载版本。 + +### 受支持的长期路径 + +- **macOS 27+：优先 Core AI PoC。** 先转换一个 Utter 实际使用的小型 Qwen 模型，验证 state/KV cache、逐 token decode、内存、首 token 延迟和设备 placement。Apple 已把现代模型架构指向 Core AI，它比新写一套私有 ANE runtime 更可持续。 +- **若必须覆盖 macOS 26 的 ANE：再做 Core ML PoC。** 需要把 decoder prefill/decode、KV cache 和 tokenizer/采样拆开；用 `MLComputePlan` 验证真实 placement，而不是只因设置 `.cpuAndNeuralEngine` 就宣称全模型在 ANE。 +- **MLX 保持稳定底座。** 它走 CPU/GPU，不是 ANE，但当前模型资产、Swift 集成和产品稳定性已经得到验证，适合作为所有系统版本的 fallback。 + +最合理的双轨是：**短期把当前 ANE-LM fork 限定为可选择的实验后端；长期在 macOS 27 上用 Core AI 建立公开、可发布的 ANE 路径。** 不应把“Qwen3-0.6B 在一台 M5 Max 能运行”包装成跨设备、跨系统兼容承诺。 + +## 来源清单 + +### Apple + +- [Core ML documentation](https://developer.apple.com/documentation/coreml) +- [MLComputeUnits](https://developer.apple.com/documentation/coreml/mlcomputeunits) +- [WWDC25: Discover machine learning & AI frameworks on Apple platforms](https://developer.apple.com/videos/play/wwdc2025/360/?time=675) +- [Core AI documentation](https://developer.apple.com/documentation/coreai) +- [macOS 27 release notes](https://developer.apple.com/documentation/macos-release-notes/macos-27-release-notes) +- [WWDC26: Meet Core AI](https://developer.apple.com/videos/play/wwdc2026/324/) +- [Apple coreai-models](https://github.com/apple/coreai-models) + +### 开源项目官方仓库 + +- [christopherkarani/Espresso](https://github.com/christopherkarani/Espresso)；[macOS 26 issue #18](https://github.com/christopherkarani/Espresso/issues/18) +- [maderix/ANE](https://github.com/maderix/ANE)；[`inmem_peak.m`](https://github.com/maderix/ANE/blob/main/inmem_peak.m)；[`test_classifier.m`](https://github.com/maderix/ANE/blob/main/training/test_classifier.m) +- [sbryngelson/ANEForge](https://github.com/sbryngelson/ANEForge)；[`e5rt` bridge](https://github.com/sbryngelson/ANEForge/blob/main/aneforge/_lib/ane_e5rt_dispatch.mm)；[`quickstart.py`](https://github.com/sbryngelson/ANEForge/blob/main/examples/quickstart.py) +- [jundot/oMLX issue #3124](https://github.com/jundot/omlx/issues/3124)；[修复 commit](https://github.com/jundot/omlx/commit/7d77098ed83d7abcafeeaffaa4de1491820021d7) +- [FluidInference/FluidAudio issue #667](https://github.com/FluidInference/FluidAudio/issues/667)；[PR #671](https://github.com/FluidInference/FluidAudio/pull/671)；[PR #849](https://github.com/FluidInference/FluidAudio/pull/849) +- [apple/coremltools issue #2687](https://github.com/apple/coremltools/issues/2687) +- [ml-explore/mlx](https://github.com/ml-explore/mlx) +- [johnmai-dev/ANE-LM](https://github.com/johnmai-dev/ANE-LM) +- [IchenDEV/ANE-LM fork](https://github.com/IchenDEV/ANE-LM)；[Utter 固定 revision `033472e`](https://github.com/IchenDEV/ANE-LM/commit/033472ec12ea796fc7ea4f8cefd7ed456f69900b) +- [thebasedcapital/ane-infer](https://github.com/thebasedcapital/ane-infer) diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/intent.md b/docs/sdlc/changes/2026-08-29-espresso-ane/intent.md new file mode 100644 index 0000000..c8f7091 --- /dev/null +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/intent.md @@ -0,0 +1,56 @@ +# Intent: Add a selectable Espresso ANE inference backend + +## Problem + +Utter's local LLM processing is limited to MLX, so users cannot choose an +Apple Neural Engine runtime for local post-processing. + +## Outcome + +The model settings let users choose Espresso, select an `.esp` model bundle, +and route local warmup and text generation through Espresso's ANE runtime. +Users choose whether an Espresso failure should finish with the selected, +installed MLX model or stop with an Espresso error. Automatic fallback remains +the default and identifies which backend produced the result. + +## Scope + +This change covers the local LLM backend setting, Espresso bundle selection, +runtime dispatch, persistence, localization, and dependency integration. It does +not create or download Espresso bundles, alter speech recognition, or claim App +Store compatibility for Espresso's private ANE API. + +## Constraints + +- Utter remains local-first and sends no new data to a remote service. +- The existing MLX backend remains the default. +- Espresso uses private Apple APIs and may fail across macOS or SoC revisions. +- Model preparation remains an upstream Espresso workflow. + +## Acceptance criteria + +- MLX and Espresso are selectable local LLM backends and the choice persists. +- A valid `.esp` directory can be selected while malformed bundles are rejected. +- Espresso selection routes warmup and generation through the Espresso engine. +- Users can disable automatic MLX fallback while keeping Espresso selected. +- When fallback is enabled, an Espresso runtime failure falls back to an + installed selected MLX model, persists MLX as the active backend, and + surfaces the fallback to the user. +- When fallback is disabled, an Espresso runtime failure does not run MLX and + surfaces an actionable Espresso error. +- If the selected MLX model is unavailable, the Espresso failure remains + visible with guidance to install an MLX model. +- Repeated fallback requests do not retain request-scoped state, and explicitly + unloading local models waits for active local inference, releases every + production model container, and clears MLX's reusable memory cache. +- Existing local MLX and remote LLM behavior remains covered by passing tests. +- The available M5/macOS 27 host rejects Espresso's private ANE program without + preventing local formatting when a compatible MLX model is installed. + +## Open questions + +Upstream documents M1 through M4 as tested, but Utter has not independently +verified that matrix. M5 is absent from that matrix, and direct Espresso +inference on the available M5 Max/macOS 27 host is blocked by the ANE compiler. +Because the backend uses private APIs, the failing variable may be the SoC, the +OS, or their combination; current evidence does not prove an M5-only failure. diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md b/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md new file mode 100644 index 0000000..2d226eb --- /dev/null +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/plan.md @@ -0,0 +1,46 @@ +# Plan: Add a selectable Espresso ANE inference backend + +## Work items + +- [x] Add and resolve the Espresso Swift package products. +- [x] Persist the local backend and Espresso bundle path with MLX defaults. +- [x] Add bundle selection, validation, localization, and private-API warning. +- [x] Route local warmup, readiness, unload, and generation to Espresso. +- [x] Add focused settings and prompt-format tests. +- [x] Exercise a prepared GPT-2 bundle on the available ANE host. +- [x] Merge current `origin/main` and preserve both dependency sets. +- [x] Resolve independent-review findings for error visibility, stale preload state, and path privacy. +- [x] Diagnose the Xcode 26.6 Release linker failure and pin the minimal Espresso metadata fix. +- [x] Fall back from failed Espresso warmup and generation to an installed MLX model. +- [x] Persist MLX after successful fallback and surface a localized completion notice. +- [x] Cover fallback ordering, success, and dual-failure behavior with focused tests. +- [x] Scope fallback outcomes to one request and preserve actionable dual-failure guidance. +- [x] Reproduce retained MLX memory after unload and clear the MLX cache on explicit unload. +- [x] Share one processor across voice and integration workflows and include benchmarking in its lifecycle. +- [x] Serialize complete local-model transactions and remove cancelled waiters from the lifecycle queue. +- [x] Let users disable automatic MLX fallback while preserving it as the default. +- [x] Keep Espresso selected and surface its runtime error when fallback is disabled. +- [x] Clarify in-product compatibility copy without claiming an M5-only failure. +- [x] Resolve independent-review findings for latest-operation outcomes, cancellation, and file size. + +## Verification plan + +- [x] `bash scripts/ci-basic-checks.sh` +- [x] `swift test` +- [x] Targeted `ConfigurationTests` +- [x] Release-style app build +- [x] Real GPT-2 `.esp` inspection and generation attempt +- [x] GitHub Xcode 26.6 release-style app build after the dependency pin +- [x] Real M5 ANE compile matrix across deployment targets and normalization variants +- [x] Targeted Espresso fallback tests +- [x] Current full repository gates and release-style app build +- [x] Repeated real fallback requests and explicit-unload memory regression +- [x] Independent cancellation, unload-order, shared-ownership, and file-size review +- [x] Focused disabled-fallback and persistence tests +- [x] Real settings-window light and dark verification for the fallback control +- [x] Current repository gates and release-style app build after the follow-up + +## Human gates + +A maintainer must decide whether to merge while M5 Max/macOS 27 inference is +blocked upstream. Publishing or releasing remains a separate approval. diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/spec.md b/docs/sdlc/changes/2026-08-29-espresso-ane/spec.md new file mode 100644 index 0000000..4f5491e --- /dev/null +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/spec.md @@ -0,0 +1,91 @@ +# Spec: Add a selectable Espresso ANE inference backend + +## Context + +`AppSettings` owns persisted model choices, `ModelManagementView` owns model +selection, and `TextProcessor` owns LLM dispatch. Existing local generation uses +`LLMEngine` and MLX; remote generation remains independent. Espresso packages +models and tokenizers as `.esp` directories consumed by `ESPRuntime` and +`RealModelInference`. + +## Design + +Add a persisted `LocalLLMBackend` choice with MLX as the default, a persisted +Espresso bundle path, and a persisted automatic-MLX-fallback toggle that +defaults on. The models UI validates a selected directory with +`ESPRuntimeBundle.open`, displays the private-API warning and fallback toggle, +and requests warmup. + +`TextProcessor` owns one `EspressoLLMEngine` actor alongside the MLX engine and +dispatches warmup, readiness, unloading, and generation according to the +captured processing options. The actor retains Espresso's move-only inference +engine, verifies that the bundle resolves to the private ANE backend, applies a +Qwen chat template when appropriate, and returns generated text through the +existing processing pipeline. Remote LLM behavior is unchanged. + +For local Espresso warmup and generation, `TextProcessor` first attempts the +selected `.esp` bundle. Captured processing options decide whether a failed +attempt may try the already-selected MLX model through the existing +`LLMEngine`; `LLMEngine` continues to require a complete local model and never +downloads during fallback. A successful MLX fallback records a request-scoped +semantic outcome. The main app consumes that outcome, changes the persisted +backend to MLX only if Espresso and automatic fallback are still selected, and +shows the notice in the completion state. With fallback disabled, MLX is not +called, Espresso remains selected, and the existing error surfaces identify the +runtime failure. Integration response schemas remain unchanged. + +The voice pipeline, integration sessions, and model benchmark UI share one +application-owned `TextProcessor`. A cancellable FIFO gate serializes complete +local-model transactions, including multimodal-to-text fallback, benchmarking, +warmup, and explicit unload. Benchmark containers are dropped after each run; +explicit unload waits for active work, drops every local container, and then +clears MLX's reusable allocation cache. + +Pin Espresso to the reviewed `v0.9.0` source plus a three-line Swift 6.2 +compatibility patch. The patch gives three internal compiled-kernel holder +types package visibility so Xcode 26.6 can resolve metadata references emitted +across the `RealModelInference`, `ESPRuntime`, and app modules. It changes no +runtime logic or public API and avoids taking unrelated upstream `main` changes. + +## Safety and failure modes + +The model and prompt remain on-device. Espresso depends on private ANE APIs, so +OS or hardware changes can reject generated ANE programs even when bundle +metadata is valid. With automatic fallback enabled, such failures first use the +selected, installed MLX model; if MLX is not available or also fails, the +combined failure propagates through the existing model-load or generation error +path. With fallback disabled, no MLX work starts and the Espresso error follows +that path directly. Selecting a missing or malformed bundle does not replace +the current setting. + +The UI warning explicitly states the private-API and App Store limitation. No +fallback is silent: successful fallback produces a localized status message +and changes the persisted backend to MLX so later requests do not repeatedly +compile a rejected private ANE program. A concurrent user change away from +Espresso is preserved. Outcome tracking is scoped to one processing task, so +cancellation, unload, language changes, and unrelated requests cannot consume +stale fallback state. Explicit model unload drops all local model containers and +clears MLX's reusable memory cache. + +The patched dependency is pinned by commit rather than a moving branch. Rollback +returns the package URL and version requirement to upstream `v0.9.0` once an +equivalent fix is released there, or removes Espresso with the rest of this +experimental backend. + +## Test strategy + +Persistence, prompt formatting, enabled and disabled Espresso-to-MLX fallback +ordering, failure behavior, cancellable local-model serialization, shared +processor wiring, and unload memory behavior have focused tests. The complete +Swift suite and repository checks cover existing paths and package integration. +A release-style app build checks dependency and Metal/resource packaging. The +fallback toggle is checked in the real settings window in both appearances. +Real inference is exercised with a prepared GPT-2 `.esp` bundle and recorded +even if the host's private ANE compiler rejects it. + +## Rollout and rollback + +Ship behind an explicit non-default setting. Stop rollout if Espresso cannot +compile a real bundle on the supported release environment. Users can return to +MLX immediately; repository rollback removes the Espresso dependency, engine, +settings fields, and settings UI without migrating stored user data. diff --git a/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md b/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md new file mode 100644 index 0000000..32ed13f --- /dev/null +++ b/docs/sdlc/changes/2026-08-29-espresso-ane/verification.md @@ -0,0 +1,67 @@ +# Verification: Add a selectable Espresso ANE inference backend + +## Evidence + +| Check | Result | Evidence | +|---|---|---| +| `bash scripts/ci-basic-checks.sh` | Pass | Current SDLC, harness, plist, localization, resource, identifier, secret-file, and symlink checks passed | +| `DEVELOPER_DIR=/Applications/Xcode-beta.app/Contents/Developer swift test` | Pass | 589 XCTest tests passed, 9 skipped, plus 1 Swift Testing test passed with user-selectable fallback behavior | +| Focused fallback, outcome, and settings tests | Pass | 51 tests passed with 1 hardware integration skip; disabling fallback preserves the Espresso error, never invokes MLX, and persists across `AppSettings` reloads; the last generation operation determines the request outcome, non-Espresso generation clears stale outcomes, and cancellation remains neutral | +| Fallback settings real-window matrix | Pass | The 760 by 680 Release settings view rendered Chinese and English in forced Aqua and Dark Aqua; the six model segments, fallback switch, bundle action, and one-line Chinese or two-line English private-API warning remain visible without clipping | +| Final no-QA-hook Release build | Pass | After removing the temporary screenshot entry and its captures, the app and CLI rebuilt, were ad-hoc signed with hardened runtime, and passed release artifact verification | +| Focused fallback and localization tests | Pass | Espresso success avoids MLX; Espresso failure uses MLX; cancellation skips fallback; dual failure preserves both diagnostics; request trackers deallocate and do not cross requests; successful fallback persists MLX; localized status and typed 288 by 56 two-line overlay layout pass | +| `DEVELOPER_DIR=/Applications/Xcode-beta.app/Contents/Developer bash scripts/build-app.sh --app-only --sign=-` | Pass | Final source without the temporary visual-QA trigger built the Release app and CLI, assembled, ad-hoc signed, and passed artifact verification | +| GitHub run `33296463655`, Xcode 26.6 Release build | Fail; diagnosed | Swift 6.2 emitted cross-module references to three internal `RealModelInferenceEngine.Compiled*` metadata symbols, then failed final arm64 linking | +| Pinned Espresso commit `f3603c7` symbol probe | Pass | The three metadata symbols are emitted as external after changing only the holder types from internal to package visibility | +| GitHub run `33297701825`, Xcode 26.6 | Pass | Contract & Tests, Release-style App Build, and SDLC Gate all passed; the app build completed in 9m12s | +| GitHub run `33304348927`, Xcode 26.6 unit tests | Fail; diagnosed | The empty lifecycle-gate test called `Memory.clearCache()` before any MLX container existed, which made the test runner initialize MLX without an app-bundled default metallib; cache clearing is now conditional on a loaded MLX LLM, benchmark, or VLM container | +| Independent high-risk reviews | Completed; final code review clean | Successive reviews found stale state, cancellation and selection races, unordered unload/reload, independent integration and benchmark containers, multimodal fallback interleaving, retained cancelled waiters, stale first-operation outcomes, and cancellation misreporting. The final implementation uses latest-operation request-scoped outcomes, neutral cancellation, selection identity checks, one shared processor, and a cancellable reentrant lifecycle gate | +| Espresso 0.9.0 GPT-2 generation | Fail | M5 Max/macOS 27 ANE compiler returned code 10, `verifyBundleAtPath: invalid model`, while compiling layer 0 attention | +| Espresso main `eafb33d` GPT-2 generation | Fail | Latest upstream source produced the same ANE code 10 on the same host | +| M5 ANE compile matrix | Fail as expected | All 24 combinations failed: iOS 18, iOS 19, macOS 26, and macOS 27 MIL targets; LayerNorm and RMSNorm; spatial sizes 64, 128, and 256 | +| Real Espresso-to-MLX fallback | Pass | On the M5 Max/macOS 27 host, a real GPT-2 `.esp` bundle produced ANE code 10, then the installed `mlx-community/Qwen3.5-2B-4bit` model generated non-empty output in the same test and produced the fallback notice | +| Repeated fallback memory loop | Pass after fix | Twenty post-fallback MLX requests grew physical footprint by about 64 KiB. Before `Memory.clearCache()`, explicit unload retained about 1.74 GiB over baseline; after the fix MLX reported about 3 KiB active and zero cache memory, and the guarded footprint check passed. `vmmap` identified most remaining delta as empty malloc regions and first-loaded framework pages rather than active model tensors | +| Production model ownership and unload ordering | Pass | AppDelegate injects one processor into voice and integration workflows; benchmarking runs through the same processor and releases its container; focused tests prove unload waits for active local work, nested multimodal transactions are reentrant, and cancelled waiters never run | +| Latest-main settings merge | Pass | The native segmented model picker now orders Qwen, Gemma, Llama, ANE, Remote, and Custom; focused tests preserve explicit selection across ANE-to-MLX backend changes; Chinese and English light/dark 760-point window renders fit; and the complete suite and Release build pass | +| Real-window fallback notice | Pass | The actual Release app displayed the 288 by 56 non-modal completion overlay without truncation in Chinese light and dark appearances and English dark appearance; the temporary environment-triggered QA entry was removed before the final build | + +## Acceptance criteria + +- Backend selection and persistence — pass; focused settings test and full suite. +- Bundle selection and malformed-bundle rejection — pass at metadata validation level through `ESPRuntimeBundle.open`. +- Espresso warmup and generation dispatch — pass by code path and build coverage. +- User-controlled MLX recovery — pass; automatic fallback defaults on and still + passes the real M5 integration test, while the disabled setting neither starts + MLX nor changes the selected Espresso backend. +- Persisted backend correction — pass with isolated `UserDefaults`; MLX replaces + Espresso only when Espresso remains the selected backend. +- User feedback — pass in both localizations and real Release windows without + treating the successful recovery as an error. +- Request and memory lifecycle — pass; outcome trackers deallocate at task end, + the last local operation determines the request result, cancellation does not + manufacture an Espresso failure, repeated requests remain flat, every + production model engine is owned by the shared processor, and explicit unload + clears MLX active/cache memory after active local work completes. +- Existing MLX and remote behavior — pass; complete suite has no failures. +- Direct Espresso generation on a supported host — blocked on the available M5 + Max/macOS 27 host; both the pinned release and upstream main fail in Apple's + private ANE compiler. The new fallback prevents that failure from blocking + local formatting when the selected MLX model is installed. + +## Residual risk + +Espresso still relies on a private ANE interface whose generated programs are +rejected on the available M5 Max/macOS 27 environment. Upstream documents M1 +through M4 as tested, but Utter has not independently verified those devices and +private-API behavior may also change with macOS. The fallback requires an +already-installed selected MLX model and deliberately does not start a download. +This change makes Utter resilient on M5; it does not establish direct Espresso +or private-ANE compatibility on M5. A public Core ML backend remains separate +future work. + +## Decision + +User-controlled recovery, persistence, regression checks, real M5 fallback, +localized window behavior, dependency resolution, and the Release link are verified. +Direct Espresso inference remains blocked on the tested host. Do not describe +the private ANE backend itself as runtime-compatible with M5 Max/macOS 27. diff --git a/docs/sdlc/changes/2026-08-30-model-type-selection/intent.md b/docs/sdlc/changes/2026-08-30-model-type-selection/intent.md index a3f43a7..fd42d69 100644 --- a/docs/sdlc/changes/2026-08-30-model-type-selection/intent.md +++ b/docs/sdlc/changes/2026-08-30-model-type-selection/intent.md @@ -30,7 +30,7 @@ persisted models, inference behavior, and remote-provider configuration. ## Acceptance criteria -- Formatting order is Qwen recommended, Gemma, Llama, Remote, Custom. +- Formatting order is Qwen recommended, Gemma, Llama, ANE, Remote, Custom. - Custom controls and family-less models appear only under Custom. - Active family-less models synchronize to Custom. - Speech order is Qwen recommended, Whisper, Apple, Doubao. diff --git a/docs/sdlc/changes/2026-08-30-model-type-selection/spec.md b/docs/sdlc/changes/2026-08-30-model-type-selection/spec.md index e28d93d..d6c7c31 100644 --- a/docs/sdlc/changes/2026-08-30-model-type-selection/spec.md +++ b/docs/sdlc/changes/2026-08-30-model-type-selection/spec.md @@ -9,11 +9,11 @@ custom list and add/import controls after every family. Speech choices use ## Design -Define one ordered formatting presentation enum with five cases: Qwen, Gemma, -Llama, Remote, and Custom. The Qwen case exposes a recommendation flag and a -picker title containing the localized recommended marker. Local-family cases -map to `ModelCatalog.ModelFamily`; Custom maps to the existing `nil` family; -Remote maps to `settings.useRemoteLLM`. +Define one ordered formatting presentation enum with six cases: Qwen, Gemma, +Llama, ANE, Remote, and Custom. The Qwen case exposes a recommendation flag and +a picker title containing the localized recommended marker. Local-family cases +map to `ModelCatalog.ModelFamily`; ANE maps to the Espresso backend; Custom maps +to the existing `nil` family; Remote maps to `settings.useRemoteLLM`. The segmented picker remains one row in the grouped form. It uses the full available section width, native compact height, and no nested background. @@ -21,8 +21,11 @@ Custom content uses the same 8/12-point internal rhythm as current model rows. `syncSelectedFamilyFromActiveModel()` assigns the active entry's family even when it is `nil`, making family-less custom/imported models select Custom. -Browsing types does not change `settings.llmModel`. Leaving Remote preserves -the existing conditional load callback for the currently active local type. +An explicit Qwen, Gemma, Llama, or Custom selection survives the asynchronous +backend change from ANE to MLX instead of being overwritten by the old active +model family. Browsing types does not change `settings.llmModel`. Leaving +Remote preserves the existing conditional load callback for the currently +active local type. Speech presentation order becomes explicit and independent from persisted enum declaration order: Qwen, Whisper, Apple, Doubao. Qwen's title includes the same @@ -30,7 +33,7 @@ localized recommendation marker. ## Safety and failure modes -- A fifth formatting segment can clip in English. Real-window verification +- A sixth formatting segment can clip in English. Real-window verification checks both languages; labels remain short and use the native control. - `nil` must mean Custom only in this UI selection layer; catalog model-family semantics remain unchanged. diff --git a/docs/sdlc/changes/2026-08-30-model-type-selection/verification.md b/docs/sdlc/changes/2026-08-30-model-type-selection/verification.md index ec5583c..c8a96c9 100644 --- a/docs/sdlc/changes/2026-08-30-model-type-selection/verification.md +++ b/docs/sdlc/changes/2026-08-30-model-type-selection/verification.md @@ -2,21 +2,20 @@ ## Evidence -- The focused regression loop initially failed on the old presentation order: - formatting exposed no Custom case or recommended Qwen state, and speech put - Qwen after Whisper, Apple, and Doubao. After implementation, - `swift test --filter ConfigurationTests` passed 34 tests. -- `swift test` passed 565 tests with 8 intentionally skipped integration tests - and no failures. +- Focused regression tests verify the six presentation types, recommended Qwen + state, active Custom synchronization, and preservation of explicit Qwen or + Custom selection when the backend changes from ANE to MLX. +- `swift test` passed 586 tests with 9 intentionally skipped integration tests, + plus 1 Swift Testing test, with no failures. - `bash scripts/ci-basic-checks.sh` passed, including SDLC validation, localization plist linting and key parity, deterministic vocabulary checks, conflict-marker checks, and secret-bearing file checks. -- `bash scripts/build-and-run.sh --verify` built, bundled, signed, launched, and - detected the final app after the temporary QA launch hook had been removed. -- Real-window inspection covered Chinese and English in light and dark - appearances. Both selectors stayed within the fixed 760-point window. The - initial full English `Recommended` label exposed horizontal overflow; the - final localized type marker is `Rec.` in English and `推荐` in Chinese. +- `bash scripts/build-app.sh --app-only --sign=-` built, bundled, ad-hoc signed, + and verified the final app after the temporary QA launch hook had been removed. +- Real-window rendering covered Chinese and English in light and dark + appearances. The six Qwen, Gemma, Llama, ANE, Remote, and Custom segments all + stayed within the fixed 760-point window. The localized recommendation marker + remains `Rec.` in English and `推荐` in Chinese. - In the live Chinese light window, selecting Custom removed all Qwen entries and displayed only the custom model ID field, Add button, and local import. - `python3 scripts/sdlc.py validate --worktree` and `git diff --check` passed. @@ -35,7 +34,7 @@ ## Residual risk The segmented-control recommendation is deliberately abbreviated to `Rec.` in -English to fit five equal-width native segments. The full `Recommended` text +English to fit six equal-width native segments. The full `Recommended` text continues to appear on individual recommended model rows. The repository's path policy classifies the `AppSettings.swift` order change as high risk, so an independent verifier, final visual approval, PR approval, and any protected diff --git a/docs/sdlc/changes/2026-08-31-ane-lm-runtime/intent.md b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/intent.md new file mode 100644 index 0000000..827c9e6 --- /dev/null +++ b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/intent.md @@ -0,0 +1,63 @@ +# Intent: Replace Espresso with a packaged ANE-LM runtime + +**Status:** approved +**Approved-by:** IchenDEV (user) +**Approved-date:** 2026-08-31 +**Upstream:** User request and PR #86 + +## Problem + +The selectable ANE backend currently depends on Espresso, whose bundled ANE +program is rejected by the private compiler on the available M5 Max/macOS 27 +host. The original ANE-LM project also fails there because its fused projection +programs are rejected, even though smaller single-output ANE convolutions work. + +## Outcome + +Utter uses a reviewed ANE-LM fork whose Qwen3 projection layout compiles on the +available host. Users keep the existing ANE selection and optional MLX fallback, +while selecting an ordinary local Qwen3 Hugging Face model directory instead of +an Espresso bundle. + +## Scope + +This change packages the smallest ANE-LM inference runtime as a Swift package, +integrates load, generation, cancellation, reset, and unload into the existing +local-model lifecycle, updates model validation and product copy, and verifies +real Qwen3-0.6B inference. It does not add model download management, change the +MLX or remote backends, support training, or claim support for untested model +families and Apple devices. + +## Constraints + +- All prompts, weights, and generated tokens stay on-device. +- The runtime uses private Apple ANE APIs and is not App Store compatible. +- MLX remains the default local backend and the user controls fallback. +- Existing persisted backend and model-path keys remain readable. +- The fork is pinned to an immutable commit; no moving branch is accepted. +- Runtime ownership must be explicit so unload does not retain model weights or + compiled programs. + +## Acceptance criteria + +- The app resolves an exact `IchenDEV/ANE-LM` revision and removes Espresso. +- Selection accepts a complete Qwen3 directory and rejects missing or unsupported + configuration and weight files without replacing the saved selection. +- The available M5 Max/macOS 27 host generates non-empty Qwen3-0.6B output on ANE + without invoking MLX fallback. +- Enabled fallback completes with the selected installed MLX model after an ANE + error; disabled fallback keeps ANE selected and exposes the ANE error. +- Cancellation stops token generation, and explicit unload destroys the native + runtime before clearing reusable MLX allocations. +- A repeated real-generation sample has no sustained post-warmup growth above + 16 MiB across 20 short requests; any unavoidable retained allocation is + measured and documented. +- Existing automated tests and a release-style app build pass. +- User-visible copy identifies this as experimental private-API ANE support and + does not promise compatibility beyond tested evidence. + +## Open questions + +The proof of concept covers Qwen3-0.6B on one M5 Max/macOS 27 host. Qwen3.5, +other parameter sizes, older Apple Silicon, and future operating systems remain +outside this change until independently exercised. diff --git a/docs/sdlc/changes/2026-08-31-ane-lm-runtime/plan.md b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/plan.md new file mode 100644 index 0000000..c5b4e6c --- /dev/null +++ b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/plan.md @@ -0,0 +1,40 @@ +# Plan: Replace Espresso with a packaged ANE-LM runtime + +**Status:** approved +**Approved-by:** IchenDEV (user) +**Approved-date:** 2026-08-31 +**Upstream:** [spec.md](spec.md) + +## Work items + +- [x] Reduce the successful M5 proof of concept to the required Qwen3 program + layout and remove exploratory kernels. +- [x] Add a narrow C ABI and SwiftPM library product to `IchenDEV/ANE-LM`. +- [x] Pin the reviewed fork revision and remove Espresso products. +- [x] Replace bundle validation with supported Qwen3 directory validation. +- [x] Connect Swift tokenization to native generation through the existing ANE + actor and local-model lifecycle. +- [x] Preserve enabled and disabled MLX fallback behavior and settings migration. +- [x] Update localized private-API and compatibility copy. +- [x] Add focused behavioral tests where the changed seams are deterministic. +- [x] Exercise real M5 generation, cancellation, unload, and 20-request memory. +- [x] Resolve and close independent verifier findings. + +## Verification plan + +- [x] Focused package build and runtime tests in the ANE-LM fork. +- [x] Real Qwen3-0.6B generation on M5 Max/macOS 27. +- [x] Same-process 20-request resident-memory sample and post-unload sample. +- [x] Focused ANE selection and fallback tests. +- [x] `bash scripts/sdlc-checks.sh` +- [x] `bash scripts/ci-basic-checks.sh` +- [x] `swift test` +- [x] `bash scripts/build-app.sh` +- [x] Independent high-risk verification. + +## Human gates + +The user explicitly requested the ANE-LM modification and current-project +integration after reviewing the original-runtime failure and proof-of-concept +result. PR approval, acceptance of residual private-API risk, and any release or +publication remain separate maintainer decisions. diff --git a/docs/sdlc/changes/2026-08-31-ane-lm-runtime/spec.md b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/spec.md new file mode 100644 index 0000000..5e032f2 --- /dev/null +++ b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/spec.md @@ -0,0 +1,80 @@ +# Spec: Replace Espresso with a packaged ANE-LM runtime + +**Status:** approved +**Approved-by:** IchenDEV (user) +**Approved-date:** 2026-08-31 +**Upstream:** [intent.md](intent.md) + +## Context + +`TextProcessor` already owns one actor for the optional ANE backend and already +serializes local-model operations with MLX fallback. `ModelManagementView` +already owns local model selection. The current Espresso package crosses both +seams. ANE-LM supplies Qwen3 tensor loading, CPU operators, sampling, and private +ANE program compilation, but its CLI tokenizer and CMake-only packaging are not +needed by Utter. + +## Design + +Fork ANE-LM under `IchenDEV` and expose a single SwiftPM C++ library product. Its +public surface is a C ABI for runtime creation/destruction and token generation. +The handle owns one Qwen3 model, resets its KV cache before each independent +request, invokes a token callback, and treats a callback stop as cancellation. +Errors cross the ABI as owned UTF-8 strings with one matching free function. + +The M5 compatibility patch uses current MIL syntax and only single-output ANE +programs. Q, K, and V weights are packed row-wise before compilation and emitted +by one projection. Gate and up weights use the same layout; SiLU and elementwise +multiplication remain on CPU, and down projection remains one ANE program. This +keeps Qwen3 at four layer programs, avoiding both the compiler's fused/multiple- +output rejection and the host's finite ANE program resource limit. + +Swift keeps prompt construction and tokenization. `AutoTokenizer` loads from the +selected model folder, the existing chat prompt is encoded without added special +tokens, native generation returns token IDs, and Swift decodes the final IDs. +This avoids bringing ANE-LM's CLI, Jinja, or tokenizers-cpp dependency into the +app. Only `model_type == qwen3` with complete single-file or HF-indexed sharded +safetensors weights is admitted in this first version; unsupported directories +fail validation before native loading. + +The existing stored `.espresso` backend value, model-path key, engine property, +and fallback outcome identifiers remain internal compatibility names for this +change. Their behavior changes to ANE-LM while user-visible text says ANE or +ANE-LM. This preserves existing installations and avoids a second migration or +parallel model-management system. The external Espresso dependency and bundle +validation are removed completely. + +## Safety and failure modes + +Model loading can fail because files are incomplete, tensors do not match the +supported Qwen3 layout, private symbols change, compilation is rejected, or ANE +resources are exhausted. Validation handles structural failures; native errors +then follow the existing explicit fallback policy. No failure silently changes +the backend unless MLX actually completes the request. + +The native handle is actor-isolated. A new path or explicit unload destroys it; +cancellation stops the callback loop; each independent request resets model +state. The app must not call through a destroyed handle. Generated error strings +and temporary token buffers have one owner on each side of the C boundary. + +The API remains private and therefore unsuitable for Mac App Store distribution. +The selection UI retains this warning. Compatibility claims are limited to real +evidence recorded in verification. + +## Test strategy + +Focused tests cover directory validation, prompt token flow, fallback enabled +and disabled behavior, and engine lifecycle seams. The exact fork revision is +built through SwiftPM. A real Qwen3-0.6B model exercises direct generation on the +available M5 host, followed by 20 same-process short requests with resident-memory +samples and an unload sample. Repository checks, the full Swift suite, and a +release-style app build cover integration and packaging. An independent verifier +reviews ABI ownership, fallback semantics, dependency provenance, and evidence. + +## Rollout and rollback + +ANE remains explicit and non-default. Stop rollout on compiler rejection, +unbounded memory, corrupted output, or a lifecycle crash. Rollback restores the +pinned Espresso dependency and engine implementation from the preceding commit; +the persisted backend and path keys require no data migration. Publishing or +releasing the app remains a separate protected human action. diff --git a/docs/sdlc/changes/2026-08-31-ane-lm-runtime/verification.md b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/verification.md new file mode 100644 index 0000000..456c151 --- /dev/null +++ b/docs/sdlc/changes/2026-08-31-ane-lm-runtime/verification.md @@ -0,0 +1,62 @@ +# Verification: Replace Espresso with a packaged ANE-LM runtime + +**Status:** approved +**Approved-by:** IchenDEV (user) +**Approved-date:** 2026-08-31 +**Upstream:** [plan.md](plan.md) + +## Evidence + +| Check | Result | Evidence | +|---|---|---| +| Exact dependency | Pass | `Package.swift` and `Package.resolved` resolve `IchenDEV/ANE-LM` at immutable revision `033472ec12ea796fc7ea4f8cefd7ed456f69900b`; no Espresso package or product remains. | +| Fork package build | Pass | `swift build -c release` passed for the fork; its CMake CLI also rebuilt after parser and cleanup hardening. | +| Fork CLI smoke | Pass | Rebuilt CLI generated two tokens from the official Qwen3-0.6B model at 19.197 prompt tok/s and 18.904 generation tok/s after the mapped-allocation change. | +| Model validation | Pass | Focused tests accept a structurally complete synthetic Qwen3 model and reject truncated or missing weights, F16, unsupported config, missing or malformed tokenizer data, unsupported tokenizer components, out-of-range token IDs, missing or non-special ChatML boundary tokens, an EOS mismatch, and an invalid GQA head ratio before loading ANE. | +| Real M5 Qwen3 generation | Pass | The Utter Swift tokenizer → pinned C ABI → private ANE path generated non-empty output on M5 Max/macOS 27 without MLX fallback. | +| Three-lifecycle memory sample | Pass | 20 requests across three complete load/generate/unload lifecycles, plus cancellation in the final lifecycle, passed in 84.367s. Within-lifecycle RSS growth was 80 KB, 32 KB, and 0 KB, each below the 16 MiB bound. Unload released more than 512 MiB from every lifecycle peak; the three sampled post-unload minima were 636,512 KB, 150,928 KB, and 662,592 KB. The final minimum was 26,080 KB above the first and below the enforced 128 MiB cross-lifecycle bound. | +| Fallback behavior | Pass | Existing enabled, disabled, load-failure, generation-failure, and cancellation policy tests pass in the full suite; selection changes only after native structural validation succeeds. | +| `bash scripts/sdlc-checks.sh` | Pass | The strict shell gate checked all 12 change bundles after merging current `main`; the active intents remain pending human approval and later stages remain draft. | +| `bash scripts/ci-basic-checks.sh` | Pass | Package, shell SDLC gate, plist and localization, identifiers, vocabulary, resources, conflict, secret-bearing file, and symlink checks passed after conflict resolution. | +| `swift test` | Pass | 592 XCTest cases passed, 10 environment-gated cases skipped, and the Swift Testing model-upgrade test passed after the final runtime and UI changes. | +| `bash scripts/build-app.sh` | Pass | The final no-hook `Utter.app` and `Utter-0.0.43.dmg` built; app and mounted-DMG signatures passed and the DMG checksum verified. | +| Real-window UI | Pass | Actual 760 by 680 settings window exposed the ANE selector, Qwen3 directory chooser, enabled MLX fallback, and private-API/App Store warning with no clipping. | +| Independent verification | Pass | Final P1/P2-only review returned CLEAN after confirming ChatML boundary-token validation, the three tokenizer regressions, lifecycle evidence wording, and plan/verification consistency. | + +## Acceptance criteria + +- Exact fork pin and Espresso removal — Pass. +- Qwen3 directory validation — Pass, including required tensor, shape, BF16 + dtype, safe header, and tokenizer checks before saved selection changes. +- Real M5 ANE generation — Pass for official Qwen3-0.6B on the available M5 + Max/macOS 27 host. +- User-selectable MLX fallback — Pass in existing policy tests and real UI. +- Cancellation, unload, and bounded repeated-generation memory — Pass across + three complete lifecycles with one cancellation: 80/32/0 KB within-lifecycle + growth, at least 512 MiB recovery from each peak, and 26,080 KB final growth under the enforced + 128 MiB cross-lifecycle bound. +- Existing MLX and remote behavior — Pass in the full suite. +- Release-style packaging — Pass for the final no-hook app and mounted DMG. +- Private-API and compatibility disclosure — Pass in both localizations and the + real settings window. + +## Residual risk + +Only Qwen3-0.6B on one M5 Max/macOS 27 host has real product-path evidence. +Qwen3.5, other parameter sizes, other Apple Silicon devices, macOS 26, and +future operating-system versions remain outside the compatibility claim. The +runtime uses private APIs, has no ABI stability promise, and is not suitable for +Mac App Store distribution. Loading still compiles 122 private ANE programs and +can fail under resource pressure; the corrected path now cleans partial state +and follows the explicit fallback policy instead of retaining it. Post-unload +RSS was not deterministic: the three observed minima varied between 150,928 KB +and 662,592 KB even though request-time and cross-lifecycle growth stayed +bounded. `vmmap` and a standalone allocation probe indicate that allocator +reclaim timing contributes, but this test does not completely isolate framework +or private-runtime retention. The UI therefore only states that memory may not +immediately return to its pre-load level after switching away. + +## Decision + +Implementation evidence and the strict shell governance gate are ready for +review after merging current `main`; no release or human approval is recorded. diff --git a/docs/sdlc/changes/2026-08-31-settings-semantic-background/intent.md b/docs/sdlc/changes/2026-08-31-settings-semantic-background/intent.md new file mode 100644 index 0000000..cf53b93 --- /dev/null +++ b/docs/sdlc/changes/2026-08-31-settings-semantic-background/intent.md @@ -0,0 +1,40 @@ +# Intent: Match the macOS settings background hierarchy + +**Status:** approved +**Approved-by:** IchenDEV (user) +**Approved-date:** 2026-08-31 +**Upstream:** User report and PR #86 + +## Problem + +The settings shell forces `windowBackgroundColor` behind every tab. In dark +appearance that semantic color is substantially darker than the page surface +used by macOS Settings, so Utter reads as a near-black canvas beside the system +window. + +## Outcome + +The settings window uses the macOS semantic under-page background while keeping +the existing grouped sections, cards, controls, spacing, and fixed window size. + +## Scope + +Only the shared settings shell and page surface colors change. Tab content, +layout, behavior, model selection, other windows, and overlays are out of scope. + +## Constraints + +- Use an AppKit semantic color rather than a fixed RGB value. +- Keep the existing section/card contrast and all six tab layouts unchanged. +- Do not add appearance preferences or a custom visual-effect layer. + +## Acceptance criteria + +- The shared settings shell and page surfaces use `underPageBackgroundColor`. +- The real dark-appearance settings window has the lighter system-like page + hierarchy without clipping or reducing text/control contrast. +- Existing automated tests and the release-style build continue to pass. + +## Open questions + +Final visual acceptance remains a maintainer decision. diff --git a/docs/sdlc/changes/2026-08-31-settings-semantic-background/plan.md b/docs/sdlc/changes/2026-08-31-settings-semantic-background/plan.md new file mode 100644 index 0000000..8cb52ac --- /dev/null +++ b/docs/sdlc/changes/2026-08-31-settings-semantic-background/plan.md @@ -0,0 +1,27 @@ +# Plan: Match the macOS settings background hierarchy + +**Status:** approved +**Approved-by:** IchenDEV (user) +**Approved-date:** 2026-08-31 +**Upstream:** [spec.md](spec.md) + +## Work items + +- [x] Identify the shared forced background used by every settings tab. +- [x] Replace it with the macOS semantic under-page background. +- [x] Inspect Activity and the ANE model pane in the real dark window. +- [x] Record current evidence and residual risk. + +## Verification plan + +- [x] Real-window dark-appearance inspection +- [x] `swift test` +- [x] `bash scripts/sdlc-checks.sh` +- [x] `bash scripts/ci-basic-checks.sh` +- [x] Final `bash scripts/build-app.sh` +- [x] `git diff --check` + +## Human gates + +The user reported the mismatch and requested the system-like background. Final +visual acceptance and any release remain separate maintainer decisions. diff --git a/docs/sdlc/changes/2026-08-31-settings-semantic-background/spec.md b/docs/sdlc/changes/2026-08-31-settings-semantic-background/spec.md new file mode 100644 index 0000000..cf4f590 --- /dev/null +++ b/docs/sdlc/changes/2026-08-31-settings-semantic-background/spec.md @@ -0,0 +1,38 @@ +# Spec: Match the macOS settings background hierarchy + +**Status:** approved +**Approved-by:** IchenDEV (user) +**Approved-date:** 2026-08-31 +**Upstream:** [intent.md](intent.md) + +## Context + +`SettingsView` and `settingsPageSurface()` both paint +`NSColor.windowBackgroundColor`. The duplicated opaque paint makes the whole +settings window use the dark-window base color even where macOS settings pages +normally use the lighter semantic under-page surface. + +## Design + +Replace those two shared paints with `NSColor.underPageBackgroundColor`. This is +the smallest change that keeps AppKit responsible for light/dark adaptation and +changes every settings tab consistently. Existing grouped forms and custom +cards continue to own their own semantic control backgrounds. + +## Safety and failure modes + +The color remains system-provided and appearance-aware. The main risk is reduced +contrast between page and grouped sections, so the actual dark window is +inspected on the affected model page and the card-heavy Activity page. + +## Test strategy + +Inspect the real built settings window in dark appearance, including Activity +and the ANE model pane, then run repository checks, the full Swift suite, and a +release-style artifact build. No unit test is added for a two-line semantic +color choice because it would only duplicate the implementation. + +## Rollout and rollback + +Ship through normal review. Roll back the two semantic-color substitutions if +the accepted system appearance or contrast is worse; no user data is affected. diff --git a/docs/sdlc/changes/2026-08-31-settings-semantic-background/verification.md b/docs/sdlc/changes/2026-08-31-settings-semantic-background/verification.md new file mode 100644 index 0000000..5b22cd5 --- /dev/null +++ b/docs/sdlc/changes/2026-08-31-settings-semantic-background/verification.md @@ -0,0 +1,37 @@ +# Verification: Match the macOS settings background hierarchy + +**Status:** approved +**Approved-by:** IchenDEV (user) +**Approved-date:** 2026-08-31 +**Upstream:** [plan.md](plan.md) + +## Evidence + +| Check | Result | Evidence | +|---|---|---| +| Source inspection | Pass | Both shared settings paints now use AppKit `underPageBackgroundColor`; no fixed RGB or new material layer was added. | +| Real-window dark inspection | Pass | The built 760 by 680 window was opened through the actual AppKit/SwiftUI settings path; Activity and the ANE model pane use the lighter page surface while grouped sections, warning copy, switches, and labels remain legible. | +| `swift test` | Pass | 592 XCTest cases passed, 10 environment-gated cases skipped, and the Swift Testing model-upgrade test passed. | +| `bash scripts/sdlc-checks.sh` | Pass | The strict shell gate checked all 12 change bundles after merging current `main`; this intent remains pending human approval and later stages remain draft. | +| `bash scripts/ci-basic-checks.sh` | Pass | Package, shell SDLC gate, plist/localization parity, identifiers, vocabulary, resources, conflict markers, secrets, and symlink checks passed after conflict resolution. | +| Final `bash scripts/build-app.sh` | Pass | Final no-hook app and DMG rebuilt; app and mounted-DMG signatures and DMG checksum passed. | +| `git diff --check` | Pass | No whitespace errors after final evidence updates. | + +## Acceptance criteria + +- Shared semantic under-page background — pass in both shared source seams. +- Dark real-window page hierarchy and contrast — pass on Activity and ANE model + content without layout changes. +- Regression and release-style checks — full tests, basic CI, final artifact, + signature, checksum, and diff checks pass. + +## Residual risk + +Only the current dark appearance was visually compared with the reported issue. +The chosen AppKit semantic color adapts in light and accessibility appearances, +but final human visual acceptance still applies. + +## Decision + +Verified with automated checks, a final release-style build, and real-window +dark-appearance inspection. No release or human approval is recorded.