Replace Espresso with a hardened ANE-LM backend#86

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
IchenDEV merged 17 commits into from
Aug 31, 2026
Merged

Replace Espresso with a hardened ANE-LM backend #86

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
6128fb7
Add Espresso ANE inference backend
IchenDEV Aug 27, 2026
def962c
Merge remote-tracking branch 'origin/main' into lava-topaz
IchenDEV Aug 28, 2026
10bf158
Document Espresso ANE verification
IchenDEV Aug 29, 2026
11318c0
Handle Espresso runtime failures safely
IchenDEV Aug 30, 2026
ef4872e
Fix Espresso linkage on Swift 6.2
IchenDEV Aug 30, 2026
50427a4
Record hosted Espresso linkage verification
IchenDEV Aug 30, 2026
68eaa6b
fix: fall back to MLX when Espresso fails
IchenDEV Aug 30, 2026
d0e6629
fix: close local model lifecycle leaks
IchenDEV Aug 30, 2026
9307cf6
Merge remote-tracking branch 'origin/main' into lava-topaz
IchenDEV Aug 30, 2026
9b94621
fix: skip MLX cache clear before model load
IchenDEV Aug 30, 2026
3534e08
fix: make Espresso fallback user-selectable
IchenDEV Aug 30, 2026
d8307eb
feat: replace Espresso with hardened ANE-LM runtime
IchenDEV Aug 30, 2026
ee8b3be
Merge origin/main into lava-topaz
IchenDEV Aug 30, 2026
2f477b8
docs: record intent approval
IchenDEV Aug 31, 2026
41158c2
docs: record spec approval
IchenDEV Aug 31, 2026
61d53a6
docs: record plan approval
IchenDEV Aug 31, 2026
829d0a5
docs: record verification approval
IchenDEV Aug 31, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion Package.resolved
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 6 additions & 1 deletion Package.swift
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
// swift-tools-version: 6.0
// swift-tools-version: 6.2
import PackageDescription

let package = Package(
Expand All@@ -13,6 +13,10 @@ let package = Package(
],
dependencies: [
.package(url: "https://github.com/argmaxinc/argmax-oss-swift.git", from: "1.0.0"),
.package(
url: "https://github.com/IchenDEV/ANE-LM.git",
revision: "033472ec12ea796fc7ea4f8cefd7ed456f69900b"
),
.package(url: "https://github.com/Blaizzy/mlx-audio-swift.git", exact: "0.1.3"),
.package(url: "https://github.com/huggingface/swift-transformers", from: "1.3.3"),
.package(url: "https://github.com/ml-explore/mlx-swift-lm", exact: "3.31.4"),
Expand All@@ -22,6 +26,7 @@ let package = Package(
name: "OpenType",
dependencies: [
.product(name: "WhisperKit", package: "argmax-oss-swift"),
.product(name: "ANELMRuntime", package: "ANE-LM"),
.product(name: "MLXAudioCore", package: "mlx-audio-swift"),
.product(name: "MLXAudioSTT", package: "mlx-audio-swift"),
.product(name: "Hub", package: "swift-transformers"),
Expand Down
1 change: 1 addition & 0 deletions Sources/App/AppDelegate+Integrations.swift
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -117,6 +117,7 @@ extension AppDelegate {
func makeIntegrationSessionCoordinator(service: OpenTypeService) -> InputSessionCoordinator {
InputSessionCoordinator(
service: service,
textProcessor: textProcessor,
isUserWorkflowBusy: { [weak self] in
self?.appState.isBusy ?? false
}
Expand Down
7 changes: 7 additions & 0 deletions Sources/App/AppState.swift
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,11 @@ enum AppPhase: Equatable {
case error(String)
}

enum AppCompletionKind: Equatable {
case standard
case espressoFallback
}

@MainActor
final class AppState: ObservableObject {
@Published var phase: AppPhase = .idle
Expand All@@ -32,6 +37,7 @@ final class AppState: ObservableObject {
@Published var lastFormattingDurationSeconds: Double = 0
@Published var pendingReplacement: DeferredReplacement?
@Published var activeInputMode: VoiceInputMode = .dictation
@Published var completionKind: AppCompletionKind = .standard

let settings = AppSettings.shared

Expand All@@ -54,6 +60,7 @@ final class AppState: ObservableObject {
resetDownloadProgress()
pendingReplacement = nil
activeInputMode = .dictation
completionKind = .standard
}

func clearPendingReplacement() {
Expand Down
8 changes: 7 additions & 1 deletion Sources/App/OpenTypeApp.swift
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
private var settingsWindowDelegate: SettingsWindowDelegate?
private var onboardingWindow: NSWindow?
private let popoverOutsideClickMonitor = PopoverOutsideClickMonitor()
let textProcessor: TextProcessor
var cancellables = Set<AnyCancellable>()
var iconTimer: Timer?
private var previousApp: NSRunningApplication?
Expand All@@ -39,6 +40,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
let registry = IntegrationClientRegistry()
integrationClientRegistry = registry
integrationService = OpenTypeService(registry: registry)
textProcessor = TextProcessor()
super.init()
integrationSessionCoordinator = makeIntegrationSessionCoordinator(service: integrationService)
}
Expand DownExpand Up@@ -95,7 +97,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
}

private func setupPipeline() {
pipeline = VoicePipeline(appState: appState)
pipeline = VoicePipeline(appState: appState, textProcessor: textProcessor)
Task { await pipeline?.warmUp() }
}

Expand DownExpand Up@@ -202,6 +204,10 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
onUnloadWhisper: { [weak self] in self?.pipeline?.unloadWhisper() },
onUnloadLLM: { [weak self] in self?.pipeline?.unloadLLM() },
onLoadLLM: { [weak self] in self?.pipeline?.loadLLM() },
onBenchmarkLLM: { [weak self] modelID in
guard let self else { throw CancellationError() }
return try await self.textProcessor.benchmarkLLM(modelID: modelID)
},
onUnloadLocalASR: { [weak self] in self?.pipeline?.unloadLocalASR() }
)
.environmentObject(appState)
Expand Down
103 changes: 103 additions & 0 deletions Sources/App/VoicePipeline+EditCommandHelpers.swift
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
import AppKit
import Foundation

@MainActor
extension VoicePipeline {
func replacementInputContext(
settings: AppSettings,
targetApp: NSRunningApplication?
) -> InputContext {
InputContext.capture(
targetApp: targetApp,
screenContext: "",
outputMode: .command,
inputLanguage: settings.inputLanguage,
source: .menuBar
)
}

func finalizedReplacementText(
_ text: String,
settings: AppSettings
) -> String {
textProcessor.cleanCommandGeneratedOutput(
text,
inputLanguage: settings.inputLanguage
)
}

func rewriteSelectedText(
raw: String,
intent: SelectionRewriteIntent,
settings: AppSettings,
targetApp: NSRunningApplication?
) async {
cancelScreenContextCapture()

guard let selectedText = await textInserter.selectedText(targetApp: targetApp),
!selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
showErrorHint(L("pipeline.no_selected_text_to_replace"))
return
}

let context = InputContext.capture(
targetApp: targetApp,
screenContext: "",
selectedTextOverride: selectedText,
outputMode: .command,
inputLanguage: settings.inputLanguage,
source: .menuBar
)
var options = TextProcessingOptions(settings: settings)
options.llmModel = settings.llmModel
let memoryContext = VoicePipelinePolicy.memoryContext(
for: .command,
settings: settings,
currentContext: context
)

appState.phase = .processing
appState.statusMessage = L("pipeline.formatting")
let rewrittenText = await textProcessor.processSelectionEdit(
selectedText: selectedText,
intent: intent,
options: options,
spokenCommand: raw,
memoryContext: memoryContext,
inputContext: context
)
guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
showNoSpeechDetected(reason: "selection rewrite returned empty text")
return
}

appState.processedText = rewrittenText
appState.phase = .inserting
appState.statusMessage = L("pipeline.replacing")

let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp)
appState.phase = .done
appState.statusMessage = L("status.done")
hideOverlayAfterDelay()

if case .probablyFailed(let reason) = result {
Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)")
TextInserter.copyToClipboard(rewrittenText)
showInsertionFailedAlert(text: rewrittenText, reason: reason)
return
}

let recordID = InputHistory.shared.addRecord(
rawText: raw,
processedText: rewrittenText,
wasProcessed: true,
context: context
)
appState.lastInsertedText = rewrittenText
beginCorrectionCapture(
recordID: recordID,
insertedText: rewrittenText,
context: context
)
}
}
122 changes: 19 additions & 103 deletions Sources/App/VoicePipeline+EditCommands.swift
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -8,6 +8,7 @@ extension VoicePipeline {
settings: AppSettings,
targetApp: NSRunningApplication?
) async -> Bool {
let expectedEspressoModelPath = settings.espressoModelPath
guard let command = await resolvedSpokenEditCommand(
raw: raw,
settings: settings,
Expand All@@ -24,38 +25,50 @@ extension VoicePipeline {
settings: settings,
targetApp: targetApp
)
return true
case .replaceSelection(let replacementRaw):
await replaceSelectedText(
raw: raw,
replacementRaw: replacementRaw,
settings: settings,
targetApp: targetApp
)
return true
case .rewriteLast(let intent):
await rewriteLastInsertion(
raw: raw,
intent: intent,
settings: settings,
targetApp: targetApp
)
return true
case .rewriteSelection(let intent):
await rewriteSelectedText(
raw: raw,
intent: intent,
settings: settings,
targetApp: targetApp
)
return true
case .deleteSelection:
await deleteSelectedText(targetApp: targetApp)
return true
case .undoLastInsertion:
await undoLastInsertion(targetApp: targetApp)
return true
}

guard !Task.isCancelled else { return true }
if let espressoOutcome = await consumeEspressoOutcome(
settings: settings,
expectedEspressoModelPath: expectedEspressoModelPath
) {
if case .error = appState.phase, espressoOutcome == .fallback {
Log.info("[VoicePipeline] preserving edit-command error after ANE-LM fallback")
} else if espressoOutcome != .fallback {
showErrorHint(espressoOutcome.message)
} else {
appState.completionKind = .espressoFallback
appState.statusMessage = espressoOutcome.message
showOverlay()
hideOverlayAfterDelay()
}
}
return true
}

private func replaceLastInsertion(
Expand DownExpand Up@@ -161,29 +174,6 @@ extension VoicePipeline {
)
}

private func replacementInputContext(
settings: AppSettings,
targetApp: NSRunningApplication?
) -> InputContext {
InputContext.capture(
targetApp: targetApp,
screenContext: "",
outputMode: .command,
inputLanguage: settings.inputLanguage,
source: .menuBar
)
}

private func finalizedReplacementText(
_ text: String,
settings: AppSettings
) -> String {
textProcessor.cleanCommandGeneratedOutput(
text,
inputLanguage: settings.inputLanguage
)
}

private func deleteSelectedText(targetApp: NSRunningApplication?) async {
cancelScreenContextCapture()

Expand DownExpand Up@@ -236,78 +226,4 @@ extension VoicePipeline {
hideOverlayAfterDelay()
}

private func rewriteSelectedText(
raw: String,
intent: SelectionRewriteIntent,
settings: AppSettings,
targetApp: NSRunningApplication?
) async {
cancelScreenContextCapture()

guard let selectedText = await textInserter.selectedText(targetApp: targetApp),
!selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
showErrorHint(L("pipeline.no_selected_text_to_replace"))
return
}

let context = InputContext.capture(
targetApp: targetApp,
screenContext: "",
selectedTextOverride: selectedText,
outputMode: .command,
inputLanguage: settings.inputLanguage,
source: .menuBar
)
var options = TextProcessingOptions(settings: settings)
options.llmModel = settings.llmModel
let memoryContext = VoicePipelinePolicy.memoryContext(
for: .command,
settings: settings,
currentContext: context
)

appState.phase = .processing
appState.statusMessage = L("pipeline.formatting")
let rewrittenText = await textProcessor.processSelectionEdit(
selectedText: selectedText,
intent: intent,
options: options,
spokenCommand: raw,
memoryContext: memoryContext,
inputContext: context
)
guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
showNoSpeechDetected(reason: "selection rewrite returned empty text")
return
}

appState.processedText = rewrittenText
appState.phase = .inserting
appState.statusMessage = L("pipeline.replacing")

let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp)
appState.phase = .done
appState.statusMessage = L("status.done")
hideOverlayAfterDelay()

if case .probablyFailed(let reason) = result {
Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)")
TextInserter.copyToClipboard(rewrittenText)
showInsertionFailedAlert(text: rewrittenText, reason: reason)
return
}

let recordID = InputHistory.shared.addRecord(
rawText: raw,
processedText: rewrittenText,
wasProcessed: true,
context: context
)
appState.lastInsertedText = rewrittenText
beginCorrectionCapture(
recordID: recordID,
insertedText: rewrittenText,
context: context
)
}
}
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
Replace Espresso with a hardened ANE-LM backend#86

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

IchenDEV
merged 17 commits into

from

Aug 31, 2026
 Merged


Replace Espresso with a hardened ANE-LM backend
#86
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


17 commits


Select commit
Hold shift + click to select a range
6128fb7

Add Espresso ANE inference backend


IchenDEV Aug 27, 2026
def962c

Merge remote-tracking branch 'origin/main' into lava-topaz


IchenDEV Aug 28, 2026
10bf158

Document Espresso ANE verification


IchenDEV Aug 29, 2026
11318c0

Handle Espresso runtime failures safely


IchenDEV Aug 30, 2026
ef4872e

Fix Espresso linkage on Swift 6.2


IchenDEV Aug 30, 2026
50427a4

Record hosted Espresso linkage verification


IchenDEV Aug 30, 2026
68eaa6b

fix: fall back to MLX when Espresso fails


IchenDEV Aug 30, 2026
d0e6629

fix: close local model lifecycle leaks


IchenDEV Aug 30, 2026
9307cf6

Merge remote-tracking branch 'origin/main' into lava-topaz


IchenDEV Aug 30, 2026
9b94621

fix: skip MLX cache clear before model load


IchenDEV Aug 30, 2026
3534e08

fix: make Espresso fallback user-selectable


IchenDEV Aug 30, 2026
d8307eb

feat: replace Espresso with hardened ANE-LM runtime


IchenDEV Aug 30, 2026
ee8b3be

Merge origin/main into lava-topaz


IchenDEV Aug 30, 2026
2f477b8

docs: record intent approval


IchenDEV Aug 31, 2026
41158c2

docs: record spec approval


IchenDEV Aug 31, 2026
61d53a6

docs: record plan approval


IchenDEV Aug 31, 2026
829d0a5

docs: record verification approval


IchenDEV Aug 31, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


10 changes: 9 additions & 1 deletion
Package.resolved

View file


Open in desktop


Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.


7 changes: 6 additions & 1 deletion
Package.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
// swift-tools-version: 6.0
// swift-tools-version: 6.2
import PackageDescription

let package = Package(
Expand All@@ -13,6 +13,10 @@ let package = Package(
 ],
 dependencies: [
 .package(url: "https://github.com/argmaxinc/argmax-oss-swift.git", from: "1.0.0"),
 .package(
 url: "https://github.com/IchenDEV/ANE-LM.git",
 revision: "033472ec12ea796fc7ea4f8cefd7ed456f69900b"
 ),
 .package(url: "https://github.com/Blaizzy/mlx-audio-swift.git", exact: "0.1.3"),
 .package(url: "https://github.com/huggingface/swift-transformers", from: "1.3.3"),
 .package(url: "https://github.com/ml-explore/mlx-swift-lm", exact: "3.31.4"),
Expand All@@ -22,6 +26,7 @@ let package = Package(
 name: "OpenType",
 dependencies: [
 .product(name: "WhisperKit", package: "argmax-oss-swift"),
 .product(name: "ANELMRuntime", package: "ANE-LM"),
 .product(name: "MLXAudioCore", package: "mlx-audio-swift"),
 .product(name: "MLXAudioSTT", package: "mlx-audio-swift"),
 .product(name: "Hub", package: "swift-transformers"),
Expand Down

1 change: 1 addition & 0 deletions
Sources/App/AppDelegate+Integrations.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -117,6 +117,7 @@ extension AppDelegate {
 func makeIntegrationSessionCoordinator(service: OpenTypeService) -> InputSessionCoordinator {
 InputSessionCoordinator(
 service: service,
 textProcessor: textProcessor,
 isUserWorkflowBusy: { [weak self] in
 self?.appState.isBusy ?? false
 }
Expand Down

7 changes: 7 additions & 0 deletions
Sources/App/AppState.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,11 @@ enum AppPhase: Equatable {
 case error(String)
}

enum AppCompletionKind: Equatable {
 case standard
 case espressoFallback
}

@MainActor
final class AppState: ObservableObject {
 @Published var phase: AppPhase = .idle
Expand All@@ -32,6 +37,7 @@ final class AppState: ObservableObject {
 @Published var lastFormattingDurationSeconds: Double = 0
 @Published var pendingReplacement: DeferredReplacement?
 @Published var activeInputMode: VoiceInputMode = .dictation
 @Published var completionKind: AppCompletionKind = .standard

 let settings = AppSettings.shared

Expand All@@ -54,6 +60,7 @@ final class AppState: ObservableObject {
 resetDownloadProgress()
 pendingReplacement = nil
 activeInputMode = .dictation
 completionKind = .standard
 }

 func clearPendingReplacement() {
Expand Down

8 changes: 7 additions & 1 deletion
Sources/App/OpenTypeApp.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 private var settingsWindowDelegate: SettingsWindowDelegate?
 private var onboardingWindow: NSWindow?
 private let popoverOutsideClickMonitor = PopoverOutsideClickMonitor()
 let textProcessor: TextProcessor
 var cancellables = Set<AnyCancellable>()
 var iconTimer: Timer?
 private var previousApp: NSRunningApplication?
Expand All@@ -39,6 +40,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 let registry = IntegrationClientRegistry()
 integrationClientRegistry = registry
 integrationService = OpenTypeService(registry: registry)
 textProcessor = TextProcessor()
 super.init()
 integrationSessionCoordinator = makeIntegrationSessionCoordinator(service: integrationService)
 }
Expand DownExpand Up@@ -95,7 +97,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 }

 private func setupPipeline() {
 pipeline = VoicePipeline(appState: appState)
 pipeline = VoicePipeline(appState: appState, textProcessor: textProcessor)
 Task { await pipeline?.warmUp() }
 }

Expand DownExpand Up@@ -202,6 +204,10 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 onUnloadWhisper: { [weak self] in self?.pipeline?.unloadWhisper() },
 onUnloadLLM: { [weak self] in self?.pipeline?.unloadLLM() },
 onLoadLLM: { [weak self] in self?.pipeline?.loadLLM() },
 onBenchmarkLLM: { [weak self] modelID in
 guard let self else { throw CancellationError() }
 return try await self.textProcessor.benchmarkLLM(modelID: modelID)
 },
 onUnloadLocalASR: { [weak self] in self?.pipeline?.unloadLocalASR() }
 )
 .environmentObject(appState)
Expand Down

103 changes: 103 additions & 0 deletions
Sources/App/VoicePipeline+EditCommandHelpers.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
import AppKit
import Foundation

@MainActor
extension VoicePipeline {
 func replacementInputContext(
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) -> InputContext {
 InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 }

 func finalizedReplacementText(
 _ text: String,
 settings: AppSettings
 ) -> String {
 textProcessor.cleanCommandGeneratedOutput(
 text,
 inputLanguage: settings.inputLanguage
 )
 }

 func rewriteSelectedText(
 raw: String,
 intent: SelectionRewriteIntent,
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) async {
 cancelScreenContextCapture()

 guard let selectedText = await textInserter.selectedText(targetApp: targetApp),
 !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showErrorHint(L("pipeline.no_selected_text_to_replace"))
 return
 }

 let context = InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 selectedTextOverride: selectedText,
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 var options = TextProcessingOptions(settings: settings)
 options.llmModel = settings.llmModel
 let memoryContext = VoicePipelinePolicy.memoryContext(
 for: .command,
 settings: settings,
 currentContext: context
 )

 appState.phase = .processing
 appState.statusMessage = L("pipeline.formatting")
 let rewrittenText = await textProcessor.processSelectionEdit(
 selectedText: selectedText,
 intent: intent,
 options: options,
 spokenCommand: raw,
 memoryContext: memoryContext,
 inputContext: context
 )
 guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showNoSpeechDetected(reason: "selection rewrite returned empty text")
 return
 }

 appState.processedText = rewrittenText
 appState.phase = .inserting
 appState.statusMessage = L("pipeline.replacing")

 let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp)
 appState.phase = .done
 appState.statusMessage = L("status.done")
 hideOverlayAfterDelay()

 if case .probablyFailed(let reason) = result {
 Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)")
 TextInserter.copyToClipboard(rewrittenText)
 showInsertionFailedAlert(text: rewrittenText, reason: reason)
 return
 }

 let recordID = InputHistory.shared.addRecord(
 rawText: raw,
 processedText: rewrittenText,
 wasProcessed: true,
 context: context
 )
 appState.lastInsertedText = rewrittenText
 beginCorrectionCapture(
 recordID: recordID,
 insertedText: rewrittenText,
 context: context
 )
 }
}

122 changes: 19 additions & 103 deletions
Sources/App/VoicePipeline+EditCommands.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,6 +8,7 @@ extension VoicePipeline {
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) async -> Bool {
 let expectedEspressoModelPath = settings.espressoModelPath
 guard let command = await resolvedSpokenEditCommand(
 raw: raw,
 settings: settings,
Expand All@@ -24,38 +25,50 @@ extension VoicePipeline {
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .replaceSelection(let replacementRaw):
 await replaceSelectedText(
 raw: raw,
 replacementRaw: replacementRaw,
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .rewriteLast(let intent):
 await rewriteLastInsertion(
 raw: raw,
 intent: intent,
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .rewriteSelection(let intent):
 await rewriteSelectedText(
 raw: raw,
 intent: intent,
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .deleteSelection:
 await deleteSelectedText(targetApp: targetApp)
 return true
 case .undoLastInsertion:
 await undoLastInsertion(targetApp: targetApp)
 return true
 }

 guard !Task.isCancelled else { return true }
 if let espressoOutcome = await consumeEspressoOutcome(
 settings: settings,
 expectedEspressoModelPath: expectedEspressoModelPath
 ) {
 if case .error = appState.phase, espressoOutcome == .fallback {
 Log.info("[VoicePipeline] preserving edit-command error after ANE-LM fallback")
 } else if espressoOutcome != .fallback {
 showErrorHint(espressoOutcome.message)
 } else {
 appState.completionKind = .espressoFallback
 appState.statusMessage = espressoOutcome.message
 showOverlay()
 hideOverlayAfterDelay()
 }
 }
 return true
 }

 private func replaceLastInsertion(
Expand DownExpand Up@@ -161,29 +174,6 @@ extension VoicePipeline {
 )
 }

 private func replacementInputContext(
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) -> InputContext {
 InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 }

 private func finalizedReplacementText(
 _ text: String,
 settings: AppSettings
 ) -> String {
 textProcessor.cleanCommandGeneratedOutput(
 text,
 inputLanguage: settings.inputLanguage
 )
 }

 private func deleteSelectedText(targetApp: NSRunningApplication?) async {
 cancelScreenContextCapture()

Expand DownExpand Up@@ -236,78 +226,4 @@ extension VoicePipeline {
 hideOverlayAfterDelay()
 }

 private func rewriteSelectedText(
 raw: String,
 intent: SelectionRewriteIntent,
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) async {
 cancelScreenContextCapture()

 guard let selectedText = await textInserter.selectedText(targetApp: targetApp),
 !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showErrorHint(L("pipeline.no_selected_text_to_replace"))
 return
 }

 let context = InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 selectedTextOverride: selectedText,
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 var options = TextProcessingOptions(settings: settings)
 options.llmModel = settings.llmModel
 let memoryContext = VoicePipelinePolicy.memoryContext(
 for: .command,
 settings: settings,
 currentContext: context
 )

 appState.phase = .processing
 appState.statusMessage = L("pipeline.formatting")
 let rewrittenText = await textProcessor.processSelectionEdit(
 selectedText: selectedText,
 intent: intent,
 options: options,
 spokenCommand: raw,
 memoryContext: memoryContext,
 inputContext: context
 )
 guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showNoSpeechDetected(reason: "selection rewrite returned empty text")
 return
 }

 appState.processedText = rewrittenText
 appState.phase = .inserting
 appState.statusMessage = L("pipeline.replacing")

 let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp)
 appState.phase = .done
 appState.statusMessage = L("status.done")
 hideOverlayAfterDelay()

 if case .probablyFailed(let reason) = result {
 Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)")
 TextInserter.copyToClipboard(rewrittenText)
 showInsertionFailedAlert(text: rewrittenText, reason: reason)
 return
 }

 let recordID = InputHistory.shared.addRecord(
 rawText: raw,
 processedText: rewrittenText,
 wasProcessed: true,
 context: context
 )
 appState.lastInsertedText = rewrittenText
 beginCorrectionCapture(
 recordID: recordID,
 insertedText: rewrittenText,
 context: context
 )
 }
}
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Replace Espresso with a hardened ANE-LM backend#86

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

IchenDEV
merged 17 commits into

from

Aug 31, 2026
 Merged


Replace Espresso with a hardened ANE-LM backend
#86
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


17 commits


Select commit
Hold shift + click to select a range
6128fb7

Add Espresso ANE inference backend


IchenDEV Aug 27, 2026
def962c

Merge remote-tracking branch 'origin/main' into lava-topaz


IchenDEV Aug 28, 2026
10bf158

Document Espresso ANE verification


IchenDEV Aug 29, 2026
11318c0

Handle Espresso runtime failures safely


IchenDEV Aug 30, 2026
ef4872e

Fix Espresso linkage on Swift 6.2


IchenDEV Aug 30, 2026
50427a4

Record hosted Espresso linkage verification


IchenDEV Aug 30, 2026
68eaa6b

fix: fall back to MLX when Espresso fails


IchenDEV Aug 30, 2026
d0e6629

fix: close local model lifecycle leaks


IchenDEV Aug 30, 2026
9307cf6

Merge remote-tracking branch 'origin/main' into lava-topaz


IchenDEV Aug 30, 2026
9b94621

fix: skip MLX cache clear before model load


IchenDEV Aug 30, 2026
3534e08

fix: make Espresso fallback user-selectable


IchenDEV Aug 30, 2026
d8307eb

feat: replace Espresso with hardened ANE-LM runtime


IchenDEV Aug 30, 2026
ee8b3be

Merge origin/main into lava-topaz


IchenDEV Aug 30, 2026
2f477b8

docs: record intent approval


IchenDEV Aug 31, 2026
41158c2

docs: record spec approval


IchenDEV Aug 31, 2026
61d53a6

docs: record plan approval


IchenDEV Aug 31, 2026
829d0a5

docs: record verification approval


IchenDEV Aug 31, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


10 changes: 9 additions & 1 deletion
Package.resolved

View file


Open in desktop


Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.


7 changes: 6 additions & 1 deletion
Package.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
// swift-tools-version: 6.0
// swift-tools-version: 6.2
import PackageDescription

let package = Package(
Expand All@@ -13,6 +13,10 @@ let package = Package(
 ],
 dependencies: [
 .package(url: "https://github.com/argmaxinc/argmax-oss-swift.git", from: "1.0.0"),
 .package(
 url: "https://github.com/IchenDEV/ANE-LM.git",
 revision: "033472ec12ea796fc7ea4f8cefd7ed456f69900b"
 ),
 .package(url: "https://github.com/Blaizzy/mlx-audio-swift.git", exact: "0.1.3"),
 .package(url: "https://github.com/huggingface/swift-transformers", from: "1.3.3"),
 .package(url: "https://github.com/ml-explore/mlx-swift-lm", exact: "3.31.4"),
Expand All@@ -22,6 +26,7 @@ let package = Package(
 name: "OpenType",
 dependencies: [
 .product(name: "WhisperKit", package: "argmax-oss-swift"),
 .product(name: "ANELMRuntime", package: "ANE-LM"),
 .product(name: "MLXAudioCore", package: "mlx-audio-swift"),
 .product(name: "MLXAudioSTT", package: "mlx-audio-swift"),
 .product(name: "Hub", package: "swift-transformers"),
Expand Down

1 change: 1 addition & 0 deletions
Sources/App/AppDelegate+Integrations.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -117,6 +117,7 @@ extension AppDelegate {
 func makeIntegrationSessionCoordinator(service: OpenTypeService) -> InputSessionCoordinator {
 InputSessionCoordinator(
 service: service,
 textProcessor: textProcessor,
 isUserWorkflowBusy: { [weak self] in
 self?.appState.isBusy ?? false
 }
Expand Down

7 changes: 7 additions & 0 deletions
Sources/App/AppState.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,11 @@ enum AppPhase: Equatable {
 case error(String)
}

enum AppCompletionKind: Equatable {
 case standard
 case espressoFallback
}

@MainActor
final class AppState: ObservableObject {
 @Published var phase: AppPhase = .idle
Expand All@@ -32,6 +37,7 @@ final class AppState: ObservableObject {
 @Published var lastFormattingDurationSeconds: Double = 0
 @Published var pendingReplacement: DeferredReplacement?
 @Published var activeInputMode: VoiceInputMode = .dictation
 @Published var completionKind: AppCompletionKind = .standard

 let settings = AppSettings.shared

Expand All@@ -54,6 +60,7 @@ final class AppState: ObservableObject {
 resetDownloadProgress()
 pendingReplacement = nil
 activeInputMode = .dictation
 completionKind = .standard
 }

 func clearPendingReplacement() {
Expand Down

8 changes: 7 additions & 1 deletion
Sources/App/OpenTypeApp.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 private var settingsWindowDelegate: SettingsWindowDelegate?
 private var onboardingWindow: NSWindow?
 private let popoverOutsideClickMonitor = PopoverOutsideClickMonitor()
 let textProcessor: TextProcessor
 var cancellables = Set<AnyCancellable>()
 var iconTimer: Timer?
 private var previousApp: NSRunningApplication?
Expand All@@ -39,6 +40,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 let registry = IntegrationClientRegistry()
 integrationClientRegistry = registry
 integrationService = OpenTypeService(registry: registry)
 textProcessor = TextProcessor()
 super.init()
 integrationSessionCoordinator = makeIntegrationSessionCoordinator(service: integrationService)
 }
Expand DownExpand Up@@ -95,7 +97,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 }

 private func setupPipeline() {
 pipeline = VoicePipeline(appState: appState)
 pipeline = VoicePipeline(appState: appState, textProcessor: textProcessor)
 Task { await pipeline?.warmUp() }
 }

Expand DownExpand Up@@ -202,6 +204,10 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 onUnloadWhisper: { [weak self] in self?.pipeline?.unloadWhisper() },
 onUnloadLLM: { [weak self] in self?.pipeline?.unloadLLM() },
 onLoadLLM: { [weak self] in self?.pipeline?.loadLLM() },
 onBenchmarkLLM: { [weak self] modelID in
 guard let self else { throw CancellationError() }
 return try await self.textProcessor.benchmarkLLM(modelID: modelID)
 },
 onUnloadLocalASR: { [weak self] in self?.pipeline?.unloadLocalASR() }
 )
 .environmentObject(appState)
Expand Down

103 changes: 103 additions & 0 deletions
Sources/App/VoicePipeline+EditCommandHelpers.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
import AppKit
import Foundation

@MainActor
extension VoicePipeline {
 func replacementInputContext(
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) -> InputContext {
 InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 }

 func finalizedReplacementText(
 _ text: String,
 settings: AppSettings
 ) -> String {
 textProcessor.cleanCommandGeneratedOutput(
 text,
 inputLanguage: settings.inputLanguage
 )
 }

 func rewriteSelectedText(
 raw: String,
 intent: SelectionRewriteIntent,
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) async {
 cancelScreenContextCapture()

 guard let selectedText = await textInserter.selectedText(targetApp: targetApp),
 !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showErrorHint(L("pipeline.no_selected_text_to_replace"))
 return
 }

 let context = InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 selectedTextOverride: selectedText,
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 var options = TextProcessingOptions(settings: settings)
 options.llmModel = settings.llmModel
 let memoryContext = VoicePipelinePolicy.memoryContext(
 for: .command,
 settings: settings,
 currentContext: context
 )

 appState.phase = .processing
 appState.statusMessage = L("pipeline.formatting")
 let rewrittenText = await textProcessor.processSelectionEdit(
 selectedText: selectedText,
 intent: intent,
 options: options,
 spokenCommand: raw,
 memoryContext: memoryContext,
 inputContext: context
 )
 guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showNoSpeechDetected(reason: "selection rewrite returned empty text")
 return
 }

 appState.processedText = rewrittenText
 appState.phase = .inserting
 appState.statusMessage = L("pipeline.replacing")

 let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp)
 appState.phase = .done
 appState.statusMessage = L("status.done")
 hideOverlayAfterDelay()

 if case .probablyFailed(let reason) = result {
 Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)")
 TextInserter.copyToClipboard(rewrittenText)
 showInsertionFailedAlert(text: rewrittenText, reason: reason)
 return
 }

 let recordID = InputHistory.shared.addRecord(
 rawText: raw,
 processedText: rewrittenText,
 wasProcessed: true,
 context: context
 )
 appState.lastInsertedText = rewrittenText
 beginCorrectionCapture(
 recordID: recordID,
 insertedText: rewrittenText,
 context: context
 )
 }
}

122 changes: 19 additions & 103 deletions
Sources/App/VoicePipeline+EditCommands.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,6 +8,7 @@ extension VoicePipeline {
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) async -> Bool {
 let expectedEspressoModelPath = settings.espressoModelPath
 guard let command = await resolvedSpokenEditCommand(
 raw: raw,
 settings: settings,
Expand All@@ -24,38 +25,50 @@ extension VoicePipeline {
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .replaceSelection(let replacementRaw):
 await replaceSelectedText(
 raw: raw,
 replacementRaw: replacementRaw,
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .rewriteLast(let intent):
 await rewriteLastInsertion(
 raw: raw,
 intent: intent,
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .rewriteSelection(let intent):
 await rewriteSelectedText(
 raw: raw,
 intent: intent,
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .deleteSelection:
 await deleteSelectedText(targetApp: targetApp)
 return true
 case .undoLastInsertion:
 await undoLastInsertion(targetApp: targetApp)
 return true
 }

 guard !Task.isCancelled else { return true }
 if let espressoOutcome = await consumeEspressoOutcome(
 settings: settings,
 expectedEspressoModelPath: expectedEspressoModelPath
 ) {
 if case .error = appState.phase, espressoOutcome == .fallback {
 Log.info("[VoicePipeline] preserving edit-command error after ANE-LM fallback")
 } else if espressoOutcome != .fallback {
 showErrorHint(espressoOutcome.message)
 } else {
 appState.completionKind = .espressoFallback
 appState.statusMessage = espressoOutcome.message
 showOverlay()
 hideOverlayAfterDelay()
 }
 }
 return true
 }

 private func replaceLastInsertion(
Expand DownExpand Up@@ -161,29 +174,6 @@ extension VoicePipeline {
 )
 }

 private func replacementInputContext(
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) -> InputContext {
 InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 }

 private func finalizedReplacementText(
 _ text: String,
 settings: AppSettings
 ) -> String {
 textProcessor.cleanCommandGeneratedOutput(
 text,
 inputLanguage: settings.inputLanguage
 )
 }

 private func deleteSelectedText(targetApp: NSRunningApplication?) async {
 cancelScreenContextCapture()

Expand DownExpand Up@@ -236,78 +226,4 @@ extension VoicePipeline {
 hideOverlayAfterDelay()
 }

 private func rewriteSelectedText(
 raw: String,
 intent: SelectionRewriteIntent,
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) async {
 cancelScreenContextCapture()

 guard let selectedText = await textInserter.selectedText(targetApp: targetApp),
 !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showErrorHint(L("pipeline.no_selected_text_to_replace"))
 return
 }

 let context = InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 selectedTextOverride: selectedText,
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 var options = TextProcessingOptions(settings: settings)
 options.llmModel = settings.llmModel
 let memoryContext = VoicePipelinePolicy.memoryContext(
 for: .command,
 settings: settings,
 currentContext: context
 )

 appState.phase = .processing
 appState.statusMessage = L("pipeline.formatting")
 let rewrittenText = await textProcessor.processSelectionEdit(
 selectedText: selectedText,
 intent: intent,
 options: options,
 spokenCommand: raw,
 memoryContext: memoryContext,
 inputContext: context
 )
 guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showNoSpeechDetected(reason: "selection rewrite returned empty text")
 return
 }

 appState.processedText = rewrittenText
 appState.phase = .inserting
 appState.statusMessage = L("pipeline.replacing")

 let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp)
 appState.phase = .done
 appState.statusMessage = L("status.done")
 hideOverlayAfterDelay()

 if case .probablyFailed(let reason) = result {
 Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)")
 TextInserter.copyToClipboard(rewrittenText)
 showInsertionFailedAlert(text: rewrittenText, reason: reason)
 return
 }

 let recordID = InputHistory.shared.addRecord(
 rawText: raw,
 processedText: rewrittenText,
 wasProcessed: true,
 context: context
 )
 appState.lastInsertedText = rewrittenText
 beginCorrectionCapture(
 recordID: recordID,
 insertedText: rewrittenText,
 context: context
 )
 }
}
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Replace Espresso with a hardened ANE-LM backend#86

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

IchenDEV
merged 17 commits into

from

Aug 31, 2026
 Merged


Replace Espresso with a hardened ANE-LM backend
#86
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


17 commits


Select commit
Hold shift + click to select a range
6128fb7

Add Espresso ANE inference backend


IchenDEV Aug 27, 2026
def962c

Merge remote-tracking branch 'origin/main' into lava-topaz


IchenDEV Aug 28, 2026
10bf158

Document Espresso ANE verification


IchenDEV Aug 29, 2026
11318c0

Handle Espresso runtime failures safely


IchenDEV Aug 30, 2026
ef4872e

Fix Espresso linkage on Swift 6.2


IchenDEV Aug 30, 2026
50427a4

Record hosted Espresso linkage verification


IchenDEV Aug 30, 2026
68eaa6b

fix: fall back to MLX when Espresso fails


IchenDEV Aug 30, 2026
d0e6629

fix: close local model lifecycle leaks


IchenDEV Aug 30, 2026
9307cf6

Merge remote-tracking branch 'origin/main' into lava-topaz


IchenDEV Aug 30, 2026
9b94621

fix: skip MLX cache clear before model load


IchenDEV Aug 30, 2026
3534e08

fix: make Espresso fallback user-selectable


IchenDEV Aug 30, 2026
d8307eb

feat: replace Espresso with hardened ANE-LM runtime


IchenDEV Aug 30, 2026
ee8b3be

Merge origin/main into lava-topaz


IchenDEV Aug 30, 2026
2f477b8

docs: record intent approval


IchenDEV Aug 31, 2026
41158c2

docs: record spec approval


IchenDEV Aug 31, 2026
61d53a6

docs: record plan approval


IchenDEV Aug 31, 2026
829d0a5

docs: record verification approval


IchenDEV Aug 31, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


10 changes: 9 additions & 1 deletion
Package.resolved

View file


Open in desktop


Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.


7 changes: 6 additions & 1 deletion
Package.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
// swift-tools-version: 6.0
// swift-tools-version: 6.2
import PackageDescription

let package = Package(
Expand All@@ -13,6 +13,10 @@ let package = Package(
 ],
 dependencies: [
 .package(url: "https://github.com/argmaxinc/argmax-oss-swift.git", from: "1.0.0"),
 .package(
 url: "https://github.com/IchenDEV/ANE-LM.git",
 revision: "033472ec12ea796fc7ea4f8cefd7ed456f69900b"
 ),
 .package(url: "https://github.com/Blaizzy/mlx-audio-swift.git", exact: "0.1.3"),
 .package(url: "https://github.com/huggingface/swift-transformers", from: "1.3.3"),
 .package(url: "https://github.com/ml-explore/mlx-swift-lm", exact: "3.31.4"),
Expand All@@ -22,6 +26,7 @@ let package = Package(
 name: "OpenType",
 dependencies: [
 .product(name: "WhisperKit", package: "argmax-oss-swift"),
 .product(name: "ANELMRuntime", package: "ANE-LM"),
 .product(name: "MLXAudioCore", package: "mlx-audio-swift"),
 .product(name: "MLXAudioSTT", package: "mlx-audio-swift"),
 .product(name: "Hub", package: "swift-transformers"),
Expand Down

1 change: 1 addition & 0 deletions
Sources/App/AppDelegate+Integrations.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -117,6 +117,7 @@ extension AppDelegate {
 func makeIntegrationSessionCoordinator(service: OpenTypeService) -> InputSessionCoordinator {
 InputSessionCoordinator(
 service: service,
 textProcessor: textProcessor,
 isUserWorkflowBusy: { [weak self] in
 self?.appState.isBusy ?? false
 }
Expand Down

7 changes: 7 additions & 0 deletions
Sources/App/AppState.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,11 @@ enum AppPhase: Equatable {
 case error(String)
}

enum AppCompletionKind: Equatable {
 case standard
 case espressoFallback
}

@MainActor
final class AppState: ObservableObject {
 @Published var phase: AppPhase = .idle
Expand All@@ -32,6 +37,7 @@ final class AppState: ObservableObject {
 @Published var lastFormattingDurationSeconds: Double = 0
 @Published var pendingReplacement: DeferredReplacement?
 @Published var activeInputMode: VoiceInputMode = .dictation
 @Published var completionKind: AppCompletionKind = .standard

 let settings = AppSettings.shared

Expand All@@ -54,6 +60,7 @@ final class AppState: ObservableObject {
 resetDownloadProgress()
 pendingReplacement = nil
 activeInputMode = .dictation
 completionKind = .standard
 }

 func clearPendingReplacement() {
Expand Down

8 changes: 7 additions & 1 deletion
Sources/App/OpenTypeApp.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 private var settingsWindowDelegate: SettingsWindowDelegate?
 private var onboardingWindow: NSWindow?
 private let popoverOutsideClickMonitor = PopoverOutsideClickMonitor()
 let textProcessor: TextProcessor
 var cancellables = Set<AnyCancellable>()
 var iconTimer: Timer?
 private var previousApp: NSRunningApplication?
Expand All@@ -39,6 +40,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 let registry = IntegrationClientRegistry()
 integrationClientRegistry = registry
 integrationService = OpenTypeService(registry: registry)
 textProcessor = TextProcessor()
 super.init()
 integrationSessionCoordinator = makeIntegrationSessionCoordinator(service: integrationService)
 }
Expand DownExpand Up@@ -95,7 +97,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 }

 private func setupPipeline() {
 pipeline = VoicePipeline(appState: appState)
 pipeline = VoicePipeline(appState: appState, textProcessor: textProcessor)
 Task { await pipeline?.warmUp() }
 }

Expand DownExpand Up@@ -202,6 +204,10 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 onUnloadWhisper: { [weak self] in self?.pipeline?.unloadWhisper() },
 onUnloadLLM: { [weak self] in self?.pipeline?.unloadLLM() },
 onLoadLLM: { [weak self] in self?.pipeline?.loadLLM() },
 onBenchmarkLLM: { [weak self] modelID in
 guard let self else { throw CancellationError() }
 return try await self.textProcessor.benchmarkLLM(modelID: modelID)
 },
 onUnloadLocalASR: { [weak self] in self?.pipeline?.unloadLocalASR() }
 )
 .environmentObject(appState)
Expand Down

103 changes: 103 additions & 0 deletions
Sources/App/VoicePipeline+EditCommandHelpers.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
import AppKit
import Foundation

@MainActor
extension VoicePipeline {
 func replacementInputContext(
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) -> InputContext {
 InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 }

 func finalizedReplacementText(
 _ text: String,
 settings: AppSettings
 ) -> String {
 textProcessor.cleanCommandGeneratedOutput(
 text,
 inputLanguage: settings.inputLanguage
 )
 }

 func rewriteSelectedText(
 raw: String,
 intent: SelectionRewriteIntent,
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) async {
 cancelScreenContextCapture()

 guard let selectedText = await textInserter.selectedText(targetApp: targetApp),
 !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showErrorHint(L("pipeline.no_selected_text_to_replace"))
 return
 }

 let context = InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 selectedTextOverride: selectedText,
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 var options = TextProcessingOptions(settings: settings)
 options.llmModel = settings.llmModel
 let memoryContext = VoicePipelinePolicy.memoryContext(
 for: .command,
 settings: settings,
 currentContext: context
 )

 appState.phase = .processing
 appState.statusMessage = L("pipeline.formatting")
 let rewrittenText = await textProcessor.processSelectionEdit(
 selectedText: selectedText,
 intent: intent,
 options: options,
 spokenCommand: raw,
 memoryContext: memoryContext,
 inputContext: context
 )
 guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showNoSpeechDetected(reason: "selection rewrite returned empty text")
 return
 }

 appState.processedText = rewrittenText
 appState.phase = .inserting
 appState.statusMessage = L("pipeline.replacing")

 let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp)
 appState.phase = .done
 appState.statusMessage = L("status.done")
 hideOverlayAfterDelay()

 if case .probablyFailed(let reason) = result {
 Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)")
 TextInserter.copyToClipboard(rewrittenText)
 showInsertionFailedAlert(text: rewrittenText, reason: reason)
 return
 }

 let recordID = InputHistory.shared.addRecord(
 rawText: raw,
 processedText: rewrittenText,
 wasProcessed: true,
 context: context
 )
 appState.lastInsertedText = rewrittenText
 beginCorrectionCapture(
 recordID: recordID,
 insertedText: rewrittenText,
 context: context
 )
 }
}

122 changes: 19 additions & 103 deletions
Sources/App/VoicePipeline+EditCommands.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,6 +8,7 @@ extension VoicePipeline {
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) async -> Bool {
 let expectedEspressoModelPath = settings.espressoModelPath
 guard let command = await resolvedSpokenEditCommand(
 raw: raw,
 settings: settings,
Expand All@@ -24,38 +25,50 @@ extension VoicePipeline {
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .replaceSelection(let replacementRaw):
 await replaceSelectedText(
 raw: raw,
 replacementRaw: replacementRaw,
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .rewriteLast(let intent):
 await rewriteLastInsertion(
 raw: raw,
 intent: intent,
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .rewriteSelection(let intent):
 await rewriteSelectedText(
 raw: raw,
 intent: intent,
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .deleteSelection:
 await deleteSelectedText(targetApp: targetApp)
 return true
 case .undoLastInsertion:
 await undoLastInsertion(targetApp: targetApp)
 return true
 }

 guard !Task.isCancelled else { return true }
 if let espressoOutcome = await consumeEspressoOutcome(
 settings: settings,
 expectedEspressoModelPath: expectedEspressoModelPath
 ) {
 if case .error = appState.phase, espressoOutcome == .fallback {
 Log.info("[VoicePipeline] preserving edit-command error after ANE-LM fallback")
 } else if espressoOutcome != .fallback {
 showErrorHint(espressoOutcome.message)
 } else {
 appState.completionKind = .espressoFallback
 appState.statusMessage = espressoOutcome.message
 showOverlay()
 hideOverlayAfterDelay()
 }
 }
 return true
 }

 private func replaceLastInsertion(
Expand DownExpand Up@@ -161,29 +174,6 @@ extension VoicePipeline {
 )
 }

 private func replacementInputContext(
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) -> InputContext {
 InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 }

 private func finalizedReplacementText(
 _ text: String,
 settings: AppSettings
 ) -> String {
 textProcessor.cleanCommandGeneratedOutput(
 text,
 inputLanguage: settings.inputLanguage
 )
 }

 private func deleteSelectedText(targetApp: NSRunningApplication?) async {
 cancelScreenContextCapture()

Expand DownExpand Up@@ -236,78 +226,4 @@ extension VoicePipeline {
 hideOverlayAfterDelay()
 }

 private func rewriteSelectedText(
 raw: String,
 intent: SelectionRewriteIntent,
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) async {
 cancelScreenContextCapture()

 guard let selectedText = await textInserter.selectedText(targetApp: targetApp),
 !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showErrorHint(L("pipeline.no_selected_text_to_replace"))
 return
 }

 let context = InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 selectedTextOverride: selectedText,
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 var options = TextProcessingOptions(settings: settings)
 options.llmModel = settings.llmModel
 let memoryContext = VoicePipelinePolicy.memoryContext(
 for: .command,
 settings: settings,
 currentContext: context
 )

 appState.phase = .processing
 appState.statusMessage = L("pipeline.formatting")
 let rewrittenText = await textProcessor.processSelectionEdit(
 selectedText: selectedText,
 intent: intent,
 options: options,
 spokenCommand: raw,
 memoryContext: memoryContext,
 inputContext: context
 )
 guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showNoSpeechDetected(reason: "selection rewrite returned empty text")
 return
 }

 appState.processedText = rewrittenText
 appState.phase = .inserting
 appState.statusMessage = L("pipeline.replacing")

 let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp)
 appState.phase = .done
 appState.statusMessage = L("status.done")
 hideOverlayAfterDelay()

 if case .probablyFailed(let reason) = result {
 Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)")
 TextInserter.copyToClipboard(rewrittenText)
 showInsertionFailedAlert(text: rewrittenText, reason: reason)
 return
 }

 let recordID = InputHistory.shared.addRecord(
 rawText: raw,
 processedText: rewrittenText,
 wasProcessed: true,
 context: context
 )
 appState.lastInsertedText = rewrittenText
 beginCorrectionCapture(
 recordID: recordID,
 insertedText: rewrittenText,
 context: context
 )
 }
}
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
Replace Espresso with a hardened ANE-LM backend#86

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

IchenDEV
merged 17 commits into

from

Aug 31, 2026
 Merged


Replace Espresso with a hardened ANE-LM backend
#86
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


17 commits


Select commit
Hold shift + click to select a range
6128fb7

Add Espresso ANE inference backend


IchenDEV Aug 27, 2026
def962c

Merge remote-tracking branch 'origin/main' into lava-topaz


IchenDEV Aug 28, 2026
10bf158

Document Espresso ANE verification


IchenDEV Aug 29, 2026
11318c0

Handle Espresso runtime failures safely


IchenDEV Aug 30, 2026
ef4872e

Fix Espresso linkage on Swift 6.2


IchenDEV Aug 30, 2026
50427a4

Record hosted Espresso linkage verification


IchenDEV Aug 30, 2026
68eaa6b

fix: fall back to MLX when Espresso fails


IchenDEV Aug 30, 2026
d0e6629

fix: close local model lifecycle leaks


IchenDEV Aug 30, 2026
9307cf6

Merge remote-tracking branch 'origin/main' into lava-topaz


IchenDEV Aug 30, 2026
9b94621

fix: skip MLX cache clear before model load


IchenDEV Aug 30, 2026
3534e08

fix: make Espresso fallback user-selectable


IchenDEV Aug 30, 2026
d8307eb

feat: replace Espresso with hardened ANE-LM runtime


IchenDEV Aug 30, 2026
ee8b3be

Merge origin/main into lava-topaz


IchenDEV Aug 30, 2026
2f477b8

docs: record intent approval


IchenDEV Aug 31, 2026
41158c2

docs: record spec approval


IchenDEV Aug 31, 2026
61d53a6

docs: record plan approval


IchenDEV Aug 31, 2026
829d0a5

docs: record verification approval


IchenDEV Aug 31, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


10 changes: 9 additions & 1 deletion
Package.resolved

View file


Open in desktop


Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.


7 changes: 6 additions & 1 deletion
Package.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
// swift-tools-version: 6.0
// swift-tools-version: 6.2
import PackageDescription

let package = Package(
Expand All@@ -13,6 +13,10 @@ let package = Package(
 ],
 dependencies: [
 .package(url: "https://github.com/argmaxinc/argmax-oss-swift.git", from: "1.0.0"),
 .package(
 url: "https://github.com/IchenDEV/ANE-LM.git",
 revision: "033472ec12ea796fc7ea4f8cefd7ed456f69900b"
 ),
 .package(url: "https://github.com/Blaizzy/mlx-audio-swift.git", exact: "0.1.3"),
 .package(url: "https://github.com/huggingface/swift-transformers", from: "1.3.3"),
 .package(url: "https://github.com/ml-explore/mlx-swift-lm", exact: "3.31.4"),
Expand All@@ -22,6 +26,7 @@ let package = Package(
 name: "OpenType",
 dependencies: [
 .product(name: "WhisperKit", package: "argmax-oss-swift"),
 .product(name: "ANELMRuntime", package: "ANE-LM"),
 .product(name: "MLXAudioCore", package: "mlx-audio-swift"),
 .product(name: "MLXAudioSTT", package: "mlx-audio-swift"),
 .product(name: "Hub", package: "swift-transformers"),
Expand Down

1 change: 1 addition & 0 deletions
Sources/App/AppDelegate+Integrations.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -117,6 +117,7 @@ extension AppDelegate {
 func makeIntegrationSessionCoordinator(service: OpenTypeService) -> InputSessionCoordinator {
 InputSessionCoordinator(
 service: service,
 textProcessor: textProcessor,
 isUserWorkflowBusy: { [weak self] in
 self?.appState.isBusy ?? false
 }
Expand Down

7 changes: 7 additions & 0 deletions
Sources/App/AppState.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,11 @@ enum AppPhase: Equatable {
 case error(String)
}

enum AppCompletionKind: Equatable {
 case standard
 case espressoFallback
}

@MainActor
final class AppState: ObservableObject {
 @Published var phase: AppPhase = .idle
Expand All@@ -32,6 +37,7 @@ final class AppState: ObservableObject {
 @Published var lastFormattingDurationSeconds: Double = 0
 @Published var pendingReplacement: DeferredReplacement?
 @Published var activeInputMode: VoiceInputMode = .dictation
 @Published var completionKind: AppCompletionKind = .standard

 let settings = AppSettings.shared

Expand All@@ -54,6 +60,7 @@ final class AppState: ObservableObject {
 resetDownloadProgress()
 pendingReplacement = nil
 activeInputMode = .dictation
 completionKind = .standard
 }

 func clearPendingReplacement() {
Expand Down

8 changes: 7 additions & 1 deletion
Sources/App/OpenTypeApp.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 private var settingsWindowDelegate: SettingsWindowDelegate?
 private var onboardingWindow: NSWindow?
 private let popoverOutsideClickMonitor = PopoverOutsideClickMonitor()
 let textProcessor: TextProcessor
 var cancellables = Set<AnyCancellable>()
 var iconTimer: Timer?
 private var previousApp: NSRunningApplication?
Expand All@@ -39,6 +40,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 let registry = IntegrationClientRegistry()
 integrationClientRegistry = registry
 integrationService = OpenTypeService(registry: registry)
 textProcessor = TextProcessor()
 super.init()
 integrationSessionCoordinator = makeIntegrationSessionCoordinator(service: integrationService)
 }
Expand DownExpand Up@@ -95,7 +97,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 }

 private func setupPipeline() {
 pipeline = VoicePipeline(appState: appState)
 pipeline = VoicePipeline(appState: appState, textProcessor: textProcessor)
 Task { await pipeline?.warmUp() }
 }

Expand DownExpand Up@@ -202,6 +204,10 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 onUnloadWhisper: { [weak self] in self?.pipeline?.unloadWhisper() },
 onUnloadLLM: { [weak self] in self?.pipeline?.unloadLLM() },
 onLoadLLM: { [weak self] in self?.pipeline?.loadLLM() },
 onBenchmarkLLM: { [weak self] modelID in
 guard let self else { throw CancellationError() }
 return try await self.textProcessor.benchmarkLLM(modelID: modelID)
 },
 onUnloadLocalASR: { [weak self] in self?.pipeline?.unloadLocalASR() }
 )
 .environmentObject(appState)
Expand Down

103 changes: 103 additions & 0 deletions
Sources/App/VoicePipeline+EditCommandHelpers.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
import AppKit
import Foundation

@MainActor
extension VoicePipeline {
 func replacementInputContext(
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) -> InputContext {
 InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 }

 func finalizedReplacementText(
 _ text: String,
 settings: AppSettings
 ) -> String {
 textProcessor.cleanCommandGeneratedOutput(
 text,
 inputLanguage: settings.inputLanguage
 )
 }

 func rewriteSelectedText(
 raw: String,
 intent: SelectionRewriteIntent,
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) async {
 cancelScreenContextCapture()

 guard let selectedText = await textInserter.selectedText(targetApp: targetApp),
 !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showErrorHint(L("pipeline.no_selected_text_to_replace"))
 return
 }

 let context = InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 selectedTextOverride: selectedText,
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 var options = TextProcessingOptions(settings: settings)
 options.llmModel = settings.llmModel
 let memoryContext = VoicePipelinePolicy.memoryContext(
 for: .command,
 settings: settings,
 currentContext: context
 )

 appState.phase = .processing
 appState.statusMessage = L("pipeline.formatting")
 let rewrittenText = await textProcessor.processSelectionEdit(
 selectedText: selectedText,
 intent: intent,
 options: options,
 spokenCommand: raw,
 memoryContext: memoryContext,
 inputContext: context
 )
 guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showNoSpeechDetected(reason: "selection rewrite returned empty text")
 return
 }

 appState.processedText = rewrittenText
 appState.phase = .inserting
 appState.statusMessage = L("pipeline.replacing")

 let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp)
 appState.phase = .done
 appState.statusMessage = L("status.done")
 hideOverlayAfterDelay()

 if case .probablyFailed(let reason) = result {
 Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)")
 TextInserter.copyToClipboard(rewrittenText)
 showInsertionFailedAlert(text: rewrittenText, reason: reason)
 return
 }

 let recordID = InputHistory.shared.addRecord(
 rawText: raw,
 processedText: rewrittenText,
 wasProcessed: true,
 context: context
 )
 appState.lastInsertedText = rewrittenText
 beginCorrectionCapture(
 recordID: recordID,
 insertedText: rewrittenText,
 context: context
 )
 }
}

122 changes: 19 additions & 103 deletions
Sources/App/VoicePipeline+EditCommands.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,6 +8,7 @@ extension VoicePipeline {
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) async -> Bool {
 let expectedEspressoModelPath = settings.espressoModelPath
 guard let command = await resolvedSpokenEditCommand(
 raw: raw,
 settings: settings,
Expand All@@ -24,38 +25,50 @@ extension VoicePipeline {
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .replaceSelection(let replacementRaw):
 await replaceSelectedText(
 raw: raw,
 replacementRaw: replacementRaw,
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .rewriteLast(let intent):
 await rewriteLastInsertion(
 raw: raw,
 intent: intent,
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .rewriteSelection(let intent):
 await rewriteSelectedText(
 raw: raw,
 intent: intent,
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .deleteSelection:
 await deleteSelectedText(targetApp: targetApp)
 return true
 case .undoLastInsertion:
 await undoLastInsertion(targetApp: targetApp)
 return true
 }

 guard !Task.isCancelled else { return true }
 if let espressoOutcome = await consumeEspressoOutcome(
 settings: settings,
 expectedEspressoModelPath: expectedEspressoModelPath
 ) {
 if case .error = appState.phase, espressoOutcome == .fallback {
 Log.info("[VoicePipeline] preserving edit-command error after ANE-LM fallback")
 } else if espressoOutcome != .fallback {
 showErrorHint(espressoOutcome.message)
 } else {
 appState.completionKind = .espressoFallback
 appState.statusMessage = espressoOutcome.message
 showOverlay()
 hideOverlayAfterDelay()
 }
 }
 return true
 }

 private func replaceLastInsertion(
Expand DownExpand Up@@ -161,29 +174,6 @@ extension VoicePipeline {
 )
 }

 private func replacementInputContext(
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) -> InputContext {
 InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 }

 private func finalizedReplacementText(
 _ text: String,
 settings: AppSettings
 ) -> String {
 textProcessor.cleanCommandGeneratedOutput(
 text,
 inputLanguage: settings.inputLanguage
 )
 }

 private func deleteSelectedText(targetApp: NSRunningApplication?) async {
 cancelScreenContextCapture()

Expand DownExpand Up@@ -236,78 +226,4 @@ extension VoicePipeline {
 hideOverlayAfterDelay()
 }

 private func rewriteSelectedText(
 raw: String,
 intent: SelectionRewriteIntent,
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) async {
 cancelScreenContextCapture()

 guard let selectedText = await textInserter.selectedText(targetApp: targetApp),
 !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showErrorHint(L("pipeline.no_selected_text_to_replace"))
 return
 }

 let context = InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 selectedTextOverride: selectedText,
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 var options = TextProcessingOptions(settings: settings)
 options.llmModel = settings.llmModel
 let memoryContext = VoicePipelinePolicy.memoryContext(
 for: .command,
 settings: settings,
 currentContext: context
 )

 appState.phase = .processing
 appState.statusMessage = L("pipeline.formatting")
 let rewrittenText = await textProcessor.processSelectionEdit(
 selectedText: selectedText,
 intent: intent,
 options: options,
 spokenCommand: raw,
 memoryContext: memoryContext,
 inputContext: context
 )
 guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showNoSpeechDetected(reason: "selection rewrite returned empty text")
 return
 }

 appState.processedText = rewrittenText
 appState.phase = .inserting
 appState.statusMessage = L("pipeline.replacing")

 let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp)
 appState.phase = .done
 appState.statusMessage = L("status.done")
 hideOverlayAfterDelay()

 if case .probablyFailed(let reason) = result {
 Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)")
 TextInserter.copyToClipboard(rewrittenText)
 showInsertionFailedAlert(text: rewrittenText, reason: reason)
 return
 }

 let recordID = InputHistory.shared.addRecord(
 rawText: raw,
 processedText: rewrittenText,
 wasProcessed: true,
 context: context
 )
 appState.lastInsertedText = rewrittenText
 beginCorrectionCapture(
 recordID: recordID,
 insertedText: rewrittenText,
 context: context
 )
 }
}
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Replace Espresso with a hardened ANE-LM backend#86

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

IchenDEV
merged 17 commits into

from

Aug 31, 2026
 Merged


Replace Espresso with a hardened ANE-LM backend
#86
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


17 commits


Select commit
Hold shift + click to select a range
6128fb7

Add Espresso ANE inference backend


IchenDEV Aug 27, 2026
def962c

Merge remote-tracking branch 'origin/main' into lava-topaz


IchenDEV Aug 28, 2026
10bf158

Document Espresso ANE verification


IchenDEV Aug 29, 2026
11318c0

Handle Espresso runtime failures safely


IchenDEV Aug 30, 2026
ef4872e

Fix Espresso linkage on Swift 6.2


IchenDEV Aug 30, 2026
50427a4

Record hosted Espresso linkage verification


IchenDEV Aug 30, 2026
68eaa6b

fix: fall back to MLX when Espresso fails


IchenDEV Aug 30, 2026
d0e6629

fix: close local model lifecycle leaks


IchenDEV Aug 30, 2026
9307cf6

Merge remote-tracking branch 'origin/main' into lava-topaz


IchenDEV Aug 30, 2026
9b94621

fix: skip MLX cache clear before model load


IchenDEV Aug 30, 2026
3534e08

fix: make Espresso fallback user-selectable


IchenDEV Aug 30, 2026
d8307eb

feat: replace Espresso with hardened ANE-LM runtime


IchenDEV Aug 30, 2026
ee8b3be

Merge origin/main into lava-topaz


IchenDEV Aug 30, 2026
2f477b8

docs: record intent approval


IchenDEV Aug 31, 2026
41158c2

docs: record spec approval


IchenDEV Aug 31, 2026
61d53a6

docs: record plan approval


IchenDEV Aug 31, 2026
829d0a5

docs: record verification approval


IchenDEV Aug 31, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


10 changes: 9 additions & 1 deletion
Package.resolved

View file


Open in desktop


Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.


7 changes: 6 additions & 1 deletion
Package.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
// swift-tools-version: 6.0
// swift-tools-version: 6.2
import PackageDescription

let package = Package(
Expand All@@ -13,6 +13,10 @@ let package = Package(
 ],
 dependencies: [
 .package(url: "https://github.com/argmaxinc/argmax-oss-swift.git", from: "1.0.0"),
 .package(
 url: "https://github.com/IchenDEV/ANE-LM.git",
 revision: "033472ec12ea796fc7ea4f8cefd7ed456f69900b"
 ),
 .package(url: "https://github.com/Blaizzy/mlx-audio-swift.git", exact: "0.1.3"),
 .package(url: "https://github.com/huggingface/swift-transformers", from: "1.3.3"),
 .package(url: "https://github.com/ml-explore/mlx-swift-lm", exact: "3.31.4"),
Expand All@@ -22,6 +26,7 @@ let package = Package(
 name: "OpenType",
 dependencies: [
 .product(name: "WhisperKit", package: "argmax-oss-swift"),
 .product(name: "ANELMRuntime", package: "ANE-LM"),
 .product(name: "MLXAudioCore", package: "mlx-audio-swift"),
 .product(name: "MLXAudioSTT", package: "mlx-audio-swift"),
 .product(name: "Hub", package: "swift-transformers"),
Expand Down

1 change: 1 addition & 0 deletions
Sources/App/AppDelegate+Integrations.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -117,6 +117,7 @@ extension AppDelegate {
 func makeIntegrationSessionCoordinator(service: OpenTypeService) -> InputSessionCoordinator {
 InputSessionCoordinator(
 service: service,
 textProcessor: textProcessor,
 isUserWorkflowBusy: { [weak self] in
 self?.appState.isBusy ?? false
 }
Expand Down

7 changes: 7 additions & 0 deletions
Sources/App/AppState.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,11 @@ enum AppPhase: Equatable {
 case error(String)
}

enum AppCompletionKind: Equatable {
 case standard
 case espressoFallback
}

@MainActor
final class AppState: ObservableObject {
 @Published var phase: AppPhase = .idle
Expand All@@ -32,6 +37,7 @@ final class AppState: ObservableObject {
 @Published var lastFormattingDurationSeconds: Double = 0
 @Published var pendingReplacement: DeferredReplacement?
 @Published var activeInputMode: VoiceInputMode = .dictation
 @Published var completionKind: AppCompletionKind = .standard

 let settings = AppSettings.shared

Expand All@@ -54,6 +60,7 @@ final class AppState: ObservableObject {
 resetDownloadProgress()
 pendingReplacement = nil
 activeInputMode = .dictation
 completionKind = .standard
 }

 func clearPendingReplacement() {
Expand Down

8 changes: 7 additions & 1 deletion
Sources/App/OpenTypeApp.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 private var settingsWindowDelegate: SettingsWindowDelegate?
 private var onboardingWindow: NSWindow?
 private let popoverOutsideClickMonitor = PopoverOutsideClickMonitor()
 let textProcessor: TextProcessor
 var cancellables = Set<AnyCancellable>()
 var iconTimer: Timer?
 private var previousApp: NSRunningApplication?
Expand All@@ -39,6 +40,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 let registry = IntegrationClientRegistry()
 integrationClientRegistry = registry
 integrationService = OpenTypeService(registry: registry)
 textProcessor = TextProcessor()
 super.init()
 integrationSessionCoordinator = makeIntegrationSessionCoordinator(service: integrationService)
 }
Expand DownExpand Up@@ -95,7 +97,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 }

 private func setupPipeline() {
 pipeline = VoicePipeline(appState: appState)
 pipeline = VoicePipeline(appState: appState, textProcessor: textProcessor)
 Task { await pipeline?.warmUp() }
 }

Expand DownExpand Up@@ -202,6 +204,10 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 onUnloadWhisper: { [weak self] in self?.pipeline?.unloadWhisper() },
 onUnloadLLM: { [weak self] in self?.pipeline?.unloadLLM() },
 onLoadLLM: { [weak self] in self?.pipeline?.loadLLM() },
 onBenchmarkLLM: { [weak self] modelID in
 guard let self else { throw CancellationError() }
 return try await self.textProcessor.benchmarkLLM(modelID: modelID)
 },
 onUnloadLocalASR: { [weak self] in self?.pipeline?.unloadLocalASR() }
 )
 .environmentObject(appState)
Expand Down

103 changes: 103 additions & 0 deletions
Sources/App/VoicePipeline+EditCommandHelpers.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
import AppKit
import Foundation

@MainActor
extension VoicePipeline {
 func replacementInputContext(
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) -> InputContext {
 InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 }

 func finalizedReplacementText(
 _ text: String,
 settings: AppSettings
 ) -> String {
 textProcessor.cleanCommandGeneratedOutput(
 text,
 inputLanguage: settings.inputLanguage
 )
 }

 func rewriteSelectedText(
 raw: String,
 intent: SelectionRewriteIntent,
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) async {
 cancelScreenContextCapture()

 guard let selectedText = await textInserter.selectedText(targetApp: targetApp),
 !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showErrorHint(L("pipeline.no_selected_text_to_replace"))
 return
 }

 let context = InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 selectedTextOverride: selectedText,
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 var options = TextProcessingOptions(settings: settings)
 options.llmModel = settings.llmModel
 let memoryContext = VoicePipelinePolicy.memoryContext(
 for: .command,
 settings: settings,
 currentContext: context
 )

 appState.phase = .processing
 appState.statusMessage = L("pipeline.formatting")
 let rewrittenText = await textProcessor.processSelectionEdit(
 selectedText: selectedText,
 intent: intent,
 options: options,
 spokenCommand: raw,
 memoryContext: memoryContext,
 inputContext: context
 )
 guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showNoSpeechDetected(reason: "selection rewrite returned empty text")
 return
 }

 appState.processedText = rewrittenText
 appState.phase = .inserting
 appState.statusMessage = L("pipeline.replacing")

 let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp)
 appState.phase = .done
 appState.statusMessage = L("status.done")
 hideOverlayAfterDelay()

 if case .probablyFailed(let reason) = result {
 Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)")
 TextInserter.copyToClipboard(rewrittenText)
 showInsertionFailedAlert(text: rewrittenText, reason: reason)
 return
 }

 let recordID = InputHistory.shared.addRecord(
 rawText: raw,
 processedText: rewrittenText,
 wasProcessed: true,
 context: context
 )
 appState.lastInsertedText = rewrittenText
 beginCorrectionCapture(
 recordID: recordID,
 insertedText: rewrittenText,
 context: context
 )
 }
}

122 changes: 19 additions & 103 deletions
Sources/App/VoicePipeline+EditCommands.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,6 +8,7 @@ extension VoicePipeline {
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) async -> Bool {
 let expectedEspressoModelPath = settings.espressoModelPath
 guard let command = await resolvedSpokenEditCommand(
 raw: raw,
 settings: settings,
Expand All@@ -24,38 +25,50 @@ extension VoicePipeline {
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .replaceSelection(let replacementRaw):
 await replaceSelectedText(
 raw: raw,
 replacementRaw: replacementRaw,
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .rewriteLast(let intent):
 await rewriteLastInsertion(
 raw: raw,
 intent: intent,
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .rewriteSelection(let intent):
 await rewriteSelectedText(
 raw: raw,
 intent: intent,
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .deleteSelection:
 await deleteSelectedText(targetApp: targetApp)
 return true
 case .undoLastInsertion:
 await undoLastInsertion(targetApp: targetApp)
 return true
 }

 guard !Task.isCancelled else { return true }
 if let espressoOutcome = await consumeEspressoOutcome(
 settings: settings,
 expectedEspressoModelPath: expectedEspressoModelPath
 ) {
 if case .error = appState.phase, espressoOutcome == .fallback {
 Log.info("[VoicePipeline] preserving edit-command error after ANE-LM fallback")
 } else if espressoOutcome != .fallback {
 showErrorHint(espressoOutcome.message)
 } else {
 appState.completionKind = .espressoFallback
 appState.statusMessage = espressoOutcome.message
 showOverlay()
 hideOverlayAfterDelay()
 }
 }
 return true
 }

 private func replaceLastInsertion(
Expand DownExpand Up@@ -161,29 +174,6 @@ extension VoicePipeline {
 )
 }

 private func replacementInputContext(
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) -> InputContext {
 InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 }

 private func finalizedReplacementText(
 _ text: String,
 settings: AppSettings
 ) -> String {
 textProcessor.cleanCommandGeneratedOutput(
 text,
 inputLanguage: settings.inputLanguage
 )
 }

 private func deleteSelectedText(targetApp: NSRunningApplication?) async {
 cancelScreenContextCapture()

Expand DownExpand Up@@ -236,78 +226,4 @@ extension VoicePipeline {
 hideOverlayAfterDelay()
 }

 private func rewriteSelectedText(
 raw: String,
 intent: SelectionRewriteIntent,
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) async {
 cancelScreenContextCapture()

 guard let selectedText = await textInserter.selectedText(targetApp: targetApp),
 !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showErrorHint(L("pipeline.no_selected_text_to_replace"))
 return
 }

 let context = InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 selectedTextOverride: selectedText,
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 var options = TextProcessingOptions(settings: settings)
 options.llmModel = settings.llmModel
 let memoryContext = VoicePipelinePolicy.memoryContext(
 for: .command,
 settings: settings,
 currentContext: context
 )

 appState.phase = .processing
 appState.statusMessage = L("pipeline.formatting")
 let rewrittenText = await textProcessor.processSelectionEdit(
 selectedText: selectedText,
 intent: intent,
 options: options,
 spokenCommand: raw,
 memoryContext: memoryContext,
 inputContext: context
 )
 guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showNoSpeechDetected(reason: "selection rewrite returned empty text")
 return
 }

 appState.processedText = rewrittenText
 appState.phase = .inserting
 appState.statusMessage = L("pipeline.replacing")

 let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp)
 appState.phase = .done
 appState.statusMessage = L("status.done")
 hideOverlayAfterDelay()

 if case .probablyFailed(let reason) = result {
 Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)")
 TextInserter.copyToClipboard(rewrittenText)
 showInsertionFailedAlert(text: rewrittenText, reason: reason)
 return
 }

 let recordID = InputHistory.shared.addRecord(
 rawText: raw,
 processedText: rewrittenText,
 wasProcessed: true,
 context: context
 )
 appState.lastInsertedText = rewrittenText
 beginCorrectionCapture(
 recordID: recordID,
 insertedText: rewrittenText,
 context: context
 )
 }
}
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Replace Espresso with a hardened ANE-LM backend#86

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

IchenDEV
merged 17 commits into

from

Aug 31, 2026
 Merged


Replace Espresso with a hardened ANE-LM backend
#86
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


17 commits


Select commit
Hold shift + click to select a range
6128fb7

Add Espresso ANE inference backend


IchenDEV Aug 27, 2026
def962c

Merge remote-tracking branch 'origin/main' into lava-topaz


IchenDEV Aug 28, 2026
10bf158

Document Espresso ANE verification


IchenDEV Aug 29, 2026
11318c0

Handle Espresso runtime failures safely


IchenDEV Aug 30, 2026
ef4872e

Fix Espresso linkage on Swift 6.2


IchenDEV Aug 30, 2026
50427a4

Record hosted Espresso linkage verification


IchenDEV Aug 30, 2026
68eaa6b

fix: fall back to MLX when Espresso fails


IchenDEV Aug 30, 2026
d0e6629

fix: close local model lifecycle leaks


IchenDEV Aug 30, 2026
9307cf6

Merge remote-tracking branch 'origin/main' into lava-topaz


IchenDEV Aug 30, 2026
9b94621

fix: skip MLX cache clear before model load


IchenDEV Aug 30, 2026
3534e08

fix: make Espresso fallback user-selectable


IchenDEV Aug 30, 2026
d8307eb

feat: replace Espresso with hardened ANE-LM runtime


IchenDEV Aug 30, 2026
ee8b3be

Merge origin/main into lava-topaz


IchenDEV Aug 30, 2026
2f477b8

docs: record intent approval


IchenDEV Aug 31, 2026
41158c2

docs: record spec approval


IchenDEV Aug 31, 2026
61d53a6

docs: record plan approval


IchenDEV Aug 31, 2026
829d0a5

docs: record verification approval


IchenDEV Aug 31, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


10 changes: 9 additions & 1 deletion
Package.resolved

View file


Open in desktop


Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.


7 changes: 6 additions & 1 deletion
Package.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
// swift-tools-version: 6.0
// swift-tools-version: 6.2
import PackageDescription

let package = Package(
Expand All@@ -13,6 +13,10 @@ let package = Package(
 ],
 dependencies: [
 .package(url: "https://github.com/argmaxinc/argmax-oss-swift.git", from: "1.0.0"),
 .package(
 url: "https://github.com/IchenDEV/ANE-LM.git",
 revision: "033472ec12ea796fc7ea4f8cefd7ed456f69900b"
 ),
 .package(url: "https://github.com/Blaizzy/mlx-audio-swift.git", exact: "0.1.3"),
 .package(url: "https://github.com/huggingface/swift-transformers", from: "1.3.3"),
 .package(url: "https://github.com/ml-explore/mlx-swift-lm", exact: "3.31.4"),
Expand All@@ -22,6 +26,7 @@ let package = Package(
 name: "OpenType",
 dependencies: [
 .product(name: "WhisperKit", package: "argmax-oss-swift"),
 .product(name: "ANELMRuntime", package: "ANE-LM"),
 .product(name: "MLXAudioCore", package: "mlx-audio-swift"),
 .product(name: "MLXAudioSTT", package: "mlx-audio-swift"),
 .product(name: "Hub", package: "swift-transformers"),
Expand Down

1 change: 1 addition & 0 deletions
Sources/App/AppDelegate+Integrations.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -117,6 +117,7 @@ extension AppDelegate {
 func makeIntegrationSessionCoordinator(service: OpenTypeService) -> InputSessionCoordinator {
 InputSessionCoordinator(
 service: service,
 textProcessor: textProcessor,
 isUserWorkflowBusy: { [weak self] in
 self?.appState.isBusy ?? false
 }
Expand Down

7 changes: 7 additions & 0 deletions
Sources/App/AppState.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,11 @@ enum AppPhase: Equatable {
 case error(String)
}

enum AppCompletionKind: Equatable {
 case standard
 case espressoFallback
}

@MainActor
final class AppState: ObservableObject {
 @Published var phase: AppPhase = .idle
Expand All@@ -32,6 +37,7 @@ final class AppState: ObservableObject {
 @Published var lastFormattingDurationSeconds: Double = 0
 @Published var pendingReplacement: DeferredReplacement?
 @Published var activeInputMode: VoiceInputMode = .dictation
 @Published var completionKind: AppCompletionKind = .standard

 let settings = AppSettings.shared

Expand All@@ -54,6 +60,7 @@ final class AppState: ObservableObject {
 resetDownloadProgress()
 pendingReplacement = nil
 activeInputMode = .dictation
 completionKind = .standard
 }

 func clearPendingReplacement() {
Expand Down

8 changes: 7 additions & 1 deletion
Sources/App/OpenTypeApp.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 private var settingsWindowDelegate: SettingsWindowDelegate?
 private var onboardingWindow: NSWindow?
 private let popoverOutsideClickMonitor = PopoverOutsideClickMonitor()
 let textProcessor: TextProcessor
 var cancellables = Set<AnyCancellable>()
 var iconTimer: Timer?
 private var previousApp: NSRunningApplication?
Expand All@@ -39,6 +40,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 let registry = IntegrationClientRegistry()
 integrationClientRegistry = registry
 integrationService = OpenTypeService(registry: registry)
 textProcessor = TextProcessor()
 super.init()
 integrationSessionCoordinator = makeIntegrationSessionCoordinator(service: integrationService)
 }
Expand DownExpand Up@@ -95,7 +97,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 }

 private func setupPipeline() {
 pipeline = VoicePipeline(appState: appState)
 pipeline = VoicePipeline(appState: appState, textProcessor: textProcessor)
 Task { await pipeline?.warmUp() }
 }

Expand DownExpand Up@@ -202,6 +204,10 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 onUnloadWhisper: { [weak self] in self?.pipeline?.unloadWhisper() },
 onUnloadLLM: { [weak self] in self?.pipeline?.unloadLLM() },
 onLoadLLM: { [weak self] in self?.pipeline?.loadLLM() },
 onBenchmarkLLM: { [weak self] modelID in
 guard let self else { throw CancellationError() }
 return try await self.textProcessor.benchmarkLLM(modelID: modelID)
 },
 onUnloadLocalASR: { [weak self] in self?.pipeline?.unloadLocalASR() }
 )
 .environmentObject(appState)
Expand Down

103 changes: 103 additions & 0 deletions
Sources/App/VoicePipeline+EditCommandHelpers.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
import AppKit
import Foundation

@MainActor
extension VoicePipeline {
 func replacementInputContext(
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) -> InputContext {
 InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 }

 func finalizedReplacementText(
 _ text: String,
 settings: AppSettings
 ) -> String {
 textProcessor.cleanCommandGeneratedOutput(
 text,
 inputLanguage: settings.inputLanguage
 )
 }

 func rewriteSelectedText(
 raw: String,
 intent: SelectionRewriteIntent,
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) async {
 cancelScreenContextCapture()

 guard let selectedText = await textInserter.selectedText(targetApp: targetApp),
 !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showErrorHint(L("pipeline.no_selected_text_to_replace"))
 return
 }

 let context = InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 selectedTextOverride: selectedText,
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 var options = TextProcessingOptions(settings: settings)
 options.llmModel = settings.llmModel
 let memoryContext = VoicePipelinePolicy.memoryContext(
 for: .command,
 settings: settings,
 currentContext: context
 )

 appState.phase = .processing
 appState.statusMessage = L("pipeline.formatting")
 let rewrittenText = await textProcessor.processSelectionEdit(
 selectedText: selectedText,
 intent: intent,
 options: options,
 spokenCommand: raw,
 memoryContext: memoryContext,
 inputContext: context
 )
 guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showNoSpeechDetected(reason: "selection rewrite returned empty text")
 return
 }

 appState.processedText = rewrittenText
 appState.phase = .inserting
 appState.statusMessage = L("pipeline.replacing")

 let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp)
 appState.phase = .done
 appState.statusMessage = L("status.done")
 hideOverlayAfterDelay()

 if case .probablyFailed(let reason) = result {
 Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)")
 TextInserter.copyToClipboard(rewrittenText)
 showInsertionFailedAlert(text: rewrittenText, reason: reason)
 return
 }

 let recordID = InputHistory.shared.addRecord(
 rawText: raw,
 processedText: rewrittenText,
 wasProcessed: true,
 context: context
 )
 appState.lastInsertedText = rewrittenText
 beginCorrectionCapture(
 recordID: recordID,
 insertedText: rewrittenText,
 context: context
 )
 }
}

122 changes: 19 additions & 103 deletions
Sources/App/VoicePipeline+EditCommands.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,6 +8,7 @@ extension VoicePipeline {
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) async -> Bool {
 let expectedEspressoModelPath = settings.espressoModelPath
 guard let command = await resolvedSpokenEditCommand(
 raw: raw,
 settings: settings,
Expand All@@ -24,38 +25,50 @@ extension VoicePipeline {
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .replaceSelection(let replacementRaw):
 await replaceSelectedText(
 raw: raw,
 replacementRaw: replacementRaw,
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .rewriteLast(let intent):
 await rewriteLastInsertion(
 raw: raw,
 intent: intent,
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .rewriteSelection(let intent):
 await rewriteSelectedText(
 raw: raw,
 intent: intent,
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .deleteSelection:
 await deleteSelectedText(targetApp: targetApp)
 return true
 case .undoLastInsertion:
 await undoLastInsertion(targetApp: targetApp)
 return true
 }

 guard !Task.isCancelled else { return true }
 if let espressoOutcome = await consumeEspressoOutcome(
 settings: settings,
 expectedEspressoModelPath: expectedEspressoModelPath
 ) {
 if case .error = appState.phase, espressoOutcome == .fallback {
 Log.info("[VoicePipeline] preserving edit-command error after ANE-LM fallback")
 } else if espressoOutcome != .fallback {
 showErrorHint(espressoOutcome.message)
 } else {
 appState.completionKind = .espressoFallback
 appState.statusMessage = espressoOutcome.message
 showOverlay()
 hideOverlayAfterDelay()
 }
 }
 return true
 }

 private func replaceLastInsertion(
Expand DownExpand Up@@ -161,29 +174,6 @@ extension VoicePipeline {
 )
 }

 private func replacementInputContext(
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) -> InputContext {
 InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 }

 private func finalizedReplacementText(
 _ text: String,
 settings: AppSettings
 ) -> String {
 textProcessor.cleanCommandGeneratedOutput(
 text,
 inputLanguage: settings.inputLanguage
 )
 }

 private func deleteSelectedText(targetApp: NSRunningApplication?) async {
 cancelScreenContextCapture()

Expand DownExpand Up@@ -236,78 +226,4 @@ extension VoicePipeline {
 hideOverlayAfterDelay()
 }

 private func rewriteSelectedText(
 raw: String,
 intent: SelectionRewriteIntent,
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) async {
 cancelScreenContextCapture()

 guard let selectedText = await textInserter.selectedText(targetApp: targetApp),
 !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showErrorHint(L("pipeline.no_selected_text_to_replace"))
 return
 }

 let context = InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 selectedTextOverride: selectedText,
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 var options = TextProcessingOptions(settings: settings)
 options.llmModel = settings.llmModel
 let memoryContext = VoicePipelinePolicy.memoryContext(
 for: .command,
 settings: settings,
 currentContext: context
 )

 appState.phase = .processing
 appState.statusMessage = L("pipeline.formatting")
 let rewrittenText = await textProcessor.processSelectionEdit(
 selectedText: selectedText,
 intent: intent,
 options: options,
 spokenCommand: raw,
 memoryContext: memoryContext,
 inputContext: context
 )
 guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showNoSpeechDetected(reason: "selection rewrite returned empty text")
 return
 }

 appState.processedText = rewrittenText
 appState.phase = .inserting
 appState.statusMessage = L("pipeline.replacing")

 let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp)
 appState.phase = .done
 appState.statusMessage = L("status.done")
 hideOverlayAfterDelay()

 if case .probablyFailed(let reason) = result {
 Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)")
 TextInserter.copyToClipboard(rewrittenText)
 showInsertionFailedAlert(text: rewrittenText, reason: reason)
 return
 }

 let recordID = InputHistory.shared.addRecord(
 rawText: raw,
 processedText: rewrittenText,
 wasProcessed: true,
 context: context
 )
 appState.lastInsertedText = rewrittenText
 beginCorrectionCapture(
 recordID: recordID,
 insertedText: rewrittenText,
 context: context
 )
 }
}
Loading
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
Replace Espresso with a hardened ANE-LM backend#86

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

IchenDEV
merged 17 commits into

from

Aug 31, 2026
 Merged


Replace Espresso with a hardened ANE-LM backend
#86
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


17 commits


Select commit
Hold shift + click to select a range
6128fb7

Add Espresso ANE inference backend


IchenDEV Aug 27, 2026
def962c

Merge remote-tracking branch 'origin/main' into lava-topaz


IchenDEV Aug 28, 2026
10bf158

Document Espresso ANE verification


IchenDEV Aug 29, 2026
11318c0

Handle Espresso runtime failures safely


IchenDEV Aug 30, 2026
ef4872e

Fix Espresso linkage on Swift 6.2


IchenDEV Aug 30, 2026
50427a4

Record hosted Espresso linkage verification


IchenDEV Aug 30, 2026
68eaa6b

fix: fall back to MLX when Espresso fails


IchenDEV Aug 30, 2026
d0e6629

fix: close local model lifecycle leaks


IchenDEV Aug 30, 2026
9307cf6

Merge remote-tracking branch 'origin/main' into lava-topaz


IchenDEV Aug 30, 2026
9b94621

fix: skip MLX cache clear before model load


IchenDEV Aug 30, 2026
3534e08

fix: make Espresso fallback user-selectable


IchenDEV Aug 30, 2026
d8307eb

feat: replace Espresso with hardened ANE-LM runtime


IchenDEV Aug 30, 2026
ee8b3be

Merge origin/main into lava-topaz


IchenDEV Aug 30, 2026
2f477b8

docs: record intent approval


IchenDEV Aug 31, 2026
41158c2

docs: record spec approval


IchenDEV Aug 31, 2026
61d53a6

docs: record plan approval


IchenDEV Aug 31, 2026
829d0a5

docs: record verification approval


IchenDEV Aug 31, 2026

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


10 changes: 9 additions & 1 deletion
Package.resolved

View file


Open in desktop


Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.


7 changes: 6 additions & 1 deletion
Package.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
// swift-tools-version: 6.0
// swift-tools-version: 6.2
import PackageDescription

let package = Package(
Expand All@@ -13,6 +13,10 @@ let package = Package(
 ],
 dependencies: [
 .package(url: "https://github.com/argmaxinc/argmax-oss-swift.git", from: "1.0.0"),
 .package(
 url: "https://github.com/IchenDEV/ANE-LM.git",
 revision: "033472ec12ea796fc7ea4f8cefd7ed456f69900b"
 ),
 .package(url: "https://github.com/Blaizzy/mlx-audio-swift.git", exact: "0.1.3"),
 .package(url: "https://github.com/huggingface/swift-transformers", from: "1.3.3"),
 .package(url: "https://github.com/ml-explore/mlx-swift-lm", exact: "3.31.4"),
Expand All@@ -22,6 +26,7 @@ let package = Package(
 name: "OpenType",
 dependencies: [
 .product(name: "WhisperKit", package: "argmax-oss-swift"),
 .product(name: "ANELMRuntime", package: "ANE-LM"),
 .product(name: "MLXAudioCore", package: "mlx-audio-swift"),
 .product(name: "MLXAudioSTT", package: "mlx-audio-swift"),
 .product(name: "Hub", package: "swift-transformers"),
Expand Down

1 change: 1 addition & 0 deletions
Sources/App/AppDelegate+Integrations.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -117,6 +117,7 @@ extension AppDelegate {
 func makeIntegrationSessionCoordinator(service: OpenTypeService) -> InputSessionCoordinator {
 InputSessionCoordinator(
 service: service,
 textProcessor: textProcessor,
 isUserWorkflowBusy: { [weak self] in
 self?.appState.isBusy ?? false
 }
Expand Down

7 changes: 7 additions & 0 deletions
Sources/App/AppState.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -13,6 +13,11 @@ enum AppPhase: Equatable {
 case error(String)
}

enum AppCompletionKind: Equatable {
 case standard
 case espressoFallback
}

@MainActor
final class AppState: ObservableObject {
 @Published var phase: AppPhase = .idle
Expand All@@ -32,6 +37,7 @@ final class AppState: ObservableObject {
 @Published var lastFormattingDurationSeconds: Double = 0
 @Published var pendingReplacement: DeferredReplacement?
 @Published var activeInputMode: VoiceInputMode = .dictation
 @Published var completionKind: AppCompletionKind = .standard

 let settings = AppSettings.shared

Expand All@@ -54,6 +60,7 @@ final class AppState: ObservableObject {
 resetDownloadProgress()
 pendingReplacement = nil
 activeInputMode = .dictation
 completionKind = .standard
 }

 func clearPendingReplacement() {
Expand Down

8 changes: 7 additions & 1 deletion
Sources/App/OpenTypeApp.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 private var settingsWindowDelegate: SettingsWindowDelegate?
 private var onboardingWindow: NSWindow?
 private let popoverOutsideClickMonitor = PopoverOutsideClickMonitor()
 let textProcessor: TextProcessor
 var cancellables = Set<AnyCancellable>()
 var iconTimer: Timer?
 private var previousApp: NSRunningApplication?
Expand All@@ -39,6 +40,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 let registry = IntegrationClientRegistry()
 integrationClientRegistry = registry
 integrationService = OpenTypeService(registry: registry)
 textProcessor = TextProcessor()
 super.init()
 integrationSessionCoordinator = makeIntegrationSessionCoordinator(service: integrationService)
 }
Expand DownExpand Up@@ -95,7 +97,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 }

 private func setupPipeline() {
 pipeline = VoicePipeline(appState: appState)
 pipeline = VoicePipeline(appState: appState, textProcessor: textProcessor)
 Task { await pipeline?.warmUp() }
 }

Expand DownExpand Up@@ -202,6 +204,10 @@ final class AppDelegate: NSObject, NSApplicationDelegate, ObservableObject {
 onUnloadWhisper: { [weak self] in self?.pipeline?.unloadWhisper() },
 onUnloadLLM: { [weak self] in self?.pipeline?.unloadLLM() },
 onLoadLLM: { [weak self] in self?.pipeline?.loadLLM() },
 onBenchmarkLLM: { [weak self] modelID in
 guard let self else { throw CancellationError() }
 return try await self.textProcessor.benchmarkLLM(modelID: modelID)
 },
 onUnloadLocalASR: { [weak self] in self?.pipeline?.unloadLocalASR() }
 )
 .environmentObject(appState)
Expand Down

103 changes: 103 additions & 0 deletions
Sources/App/VoicePipeline+EditCommandHelpers.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
import AppKit
import Foundation

@MainActor
extension VoicePipeline {
 func replacementInputContext(
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) -> InputContext {
 InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 }

 func finalizedReplacementText(
 _ text: String,
 settings: AppSettings
 ) -> String {
 textProcessor.cleanCommandGeneratedOutput(
 text,
 inputLanguage: settings.inputLanguage
 )
 }

 func rewriteSelectedText(
 raw: String,
 intent: SelectionRewriteIntent,
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) async {
 cancelScreenContextCapture()

 guard let selectedText = await textInserter.selectedText(targetApp: targetApp),
 !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showErrorHint(L("pipeline.no_selected_text_to_replace"))
 return
 }

 let context = InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 selectedTextOverride: selectedText,
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 var options = TextProcessingOptions(settings: settings)
 options.llmModel = settings.llmModel
 let memoryContext = VoicePipelinePolicy.memoryContext(
 for: .command,
 settings: settings,
 currentContext: context
 )

 appState.phase = .processing
 appState.statusMessage = L("pipeline.formatting")
 let rewrittenText = await textProcessor.processSelectionEdit(
 selectedText: selectedText,
 intent: intent,
 options: options,
 spokenCommand: raw,
 memoryContext: memoryContext,
 inputContext: context
 )
 guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showNoSpeechDetected(reason: "selection rewrite returned empty text")
 return
 }

 appState.processedText = rewrittenText
 appState.phase = .inserting
 appState.statusMessage = L("pipeline.replacing")

 let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp)
 appState.phase = .done
 appState.statusMessage = L("status.done")
 hideOverlayAfterDelay()

 if case .probablyFailed(let reason) = result {
 Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)")
 TextInserter.copyToClipboard(rewrittenText)
 showInsertionFailedAlert(text: rewrittenText, reason: reason)
 return
 }

 let recordID = InputHistory.shared.addRecord(
 rawText: raw,
 processedText: rewrittenText,
 wasProcessed: true,
 context: context
 )
 appState.lastInsertedText = rewrittenText
 beginCorrectionCapture(
 recordID: recordID,
 insertedText: rewrittenText,
 context: context
 )
 }
}

122 changes: 19 additions & 103 deletions
Sources/App/VoicePipeline+EditCommands.swift

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -8,6 +8,7 @@ extension VoicePipeline {
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) async -> Bool {
 let expectedEspressoModelPath = settings.espressoModelPath
 guard let command = await resolvedSpokenEditCommand(
 raw: raw,
 settings: settings,
Expand All@@ -24,38 +25,50 @@ extension VoicePipeline {
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .replaceSelection(let replacementRaw):
 await replaceSelectedText(
 raw: raw,
 replacementRaw: replacementRaw,
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .rewriteLast(let intent):
 await rewriteLastInsertion(
 raw: raw,
 intent: intent,
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .rewriteSelection(let intent):
 await rewriteSelectedText(
 raw: raw,
 intent: intent,
 settings: settings,
 targetApp: targetApp
 )
 return true
 case .deleteSelection:
 await deleteSelectedText(targetApp: targetApp)
 return true
 case .undoLastInsertion:
 await undoLastInsertion(targetApp: targetApp)
 return true
 }

 guard !Task.isCancelled else { return true }
 if let espressoOutcome = await consumeEspressoOutcome(
 settings: settings,
 expectedEspressoModelPath: expectedEspressoModelPath
 ) {
 if case .error = appState.phase, espressoOutcome == .fallback {
 Log.info("[VoicePipeline] preserving edit-command error after ANE-LM fallback")
 } else if espressoOutcome != .fallback {
 showErrorHint(espressoOutcome.message)
 } else {
 appState.completionKind = .espressoFallback
 appState.statusMessage = espressoOutcome.message
 showOverlay()
 hideOverlayAfterDelay()
 }
 }
 return true
 }

 private func replaceLastInsertion(
Expand DownExpand Up@@ -161,29 +174,6 @@ extension VoicePipeline {
 )
 }

 private func replacementInputContext(
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) -> InputContext {
 InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 }

 private func finalizedReplacementText(
 _ text: String,
 settings: AppSettings
 ) -> String {
 textProcessor.cleanCommandGeneratedOutput(
 text,
 inputLanguage: settings.inputLanguage
 )
 }

 private func deleteSelectedText(targetApp: NSRunningApplication?) async {
 cancelScreenContextCapture()

Expand DownExpand Up@@ -236,78 +226,4 @@ extension VoicePipeline {
 hideOverlayAfterDelay()
 }

 private func rewriteSelectedText(
 raw: String,
 intent: SelectionRewriteIntent,
 settings: AppSettings,
 targetApp: NSRunningApplication?
 ) async {
 cancelScreenContextCapture()

 guard let selectedText = await textInserter.selectedText(targetApp: targetApp),
 !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showErrorHint(L("pipeline.no_selected_text_to_replace"))
 return
 }

 let context = InputContext.capture(
 targetApp: targetApp,
 screenContext: "",
 selectedTextOverride: selectedText,
 outputMode: .command,
 inputLanguage: settings.inputLanguage,
 source: .menuBar
 )
 var options = TextProcessingOptions(settings: settings)
 options.llmModel = settings.llmModel
 let memoryContext = VoicePipelinePolicy.memoryContext(
 for: .command,
 settings: settings,
 currentContext: context
 )

 appState.phase = .processing
 appState.statusMessage = L("pipeline.formatting")
 let rewrittenText = await textProcessor.processSelectionEdit(
 selectedText: selectedText,
 intent: intent,
 options: options,
 spokenCommand: raw,
 memoryContext: memoryContext,
 inputContext: context
 )
 guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
 showNoSpeechDetected(reason: "selection rewrite returned empty text")
 return
 }

 appState.processedText = rewrittenText
 appState.phase = .inserting
 appState.statusMessage = L("pipeline.replacing")

 let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp)
 appState.phase = .done
 appState.statusMessage = L("status.done")
 hideOverlayAfterDelay()

 if case .probablyFailed(let reason) = result {
 Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)")
 TextInserter.copyToClipboard(rewrittenText)
 showInsertionFailedAlert(text: rewrittenText, reason: reason)
 return
 }

 let recordID = InputHistory.shared.addRecord(
 rawText: raw,
 processedText: rewrittenText,
 wasProcessed: true,
 context: context
 )
 appState.lastInsertedText = rewrittenText
 beginCorrectionCapture(
 recordID: recordID,
 insertedText: rewrittenText,
 context: context
 )
 }
}
Loading
Loading