[codex] Improve LLM voice processing

README.md

@@ -39,7 +39,10 @@ Three output modes are available:
 | Feature | Description |
 |---|---|
 | **Multiple Speech Engines** | Apple Speech, WhisperKit, Doubao ASR, Qwen3-ASR, or MiMo-V2.5-ASR |
-| **Smart Text Processing** | Local MLX Qwen2.5/Qwen3 or remote LLM — contextual cleanup, self-correction handling, structured list formatting |
+| **Smart Text Processing** | Local MLX Qwen2.5/Qwen3 or remote LLM infers spoken intent — contextual cleanup, "scratch that" restarts, self-correction handling, spoken punctuation, technical terms, numbers/ranges/units, and structured formatting |
+| **LLM-Owned Spoken Formatting** | Spoken casing, no-space dictation, identifiers, file paths, shortcuts, emoji, Markdown tasks, dates/times, quantities, units, formulas, fractions, and digit sequences are handled by the Smart Format / Voice Command prompts instead of local hardcoded rewrite rules |
+| **Voice Edit Commands** | In Voice Command mode, an LLM classifies safe structured actions for replacing, undoing, proofreading, titling, summarizing, drafting replies, making meeting notes, extracting key points/decisions/questions/risks/deadlines/owners/action items, rewriting tone, expanding, making tables/lists, or deleting the previous OpenType insertion or selected text |
+| **Verbatim & Preview Boundary** | Verbatim mode, streaming HUD, integration partials, and instant-insert drafts keep ASR text close to raw output with only dictionary, whitespace, duplicate, and non-speech-artifact cleanup |
 | **Remote LLM Support** | OpenAI, Claude (Anthropic format), Gemini, OpenRouter, SiliconFlow, Doubao, Bailian, MiniMax (CN & Global) |
 | **Global Hotkey** | Configurable key (Fn/Ctrl/Shift/Option) with long-press, double-tap, or single-tap activation |
 | **Screen Context OCR** | Captures on-screen text via ScreenCaptureKit + Vision to help the LLM correct homophones |

README_zh.md

@@ -39,7 +39,10 @@
 | 功能 | 说明 |
 |---|---|
 | **多语音引擎** | Apple 语音识别、WhisperKit、豆包语音识别、Qwen3-ASR 或 MiMo-V2.5-ASR |
-| **智能文字处理** | 本地 MLX Qwen2.5/Qwen3 或远程 LLM — 上下文感知的语气词清理、自动纠正、列表格式化 |
+| **智能文字处理** | 本地 MLX Qwen2.5/Qwen3 或远程 LLM 理解口述意图 — 上下文感知的语气词清理、“算了/删掉刚才”重说处理、自动纠正、口述标点、技术词、数字/范围/单位和列表格式化 |
+| **LLM 负责口述格式** | 大小写、无空格、标识符、文件路径、快捷键、表情、Markdown 任务、日期时间、数量、单位、公式、分数和数字串都由智能整理/语音指令提示词交给 LLM 判断，不在本地写死替换规则 |
+| **语音编辑口令** | 在语音指令模式下，由 LLM 分类安全结构化动作，支持上一段/选区替换、撤销、校对、跨语言回复起草、接受/拒绝/追问回复、会议纪要、关键要点/结论/问题/风险/截止时间/负责人/行动项提取、标题化、摘要、语气改写、扩写、表格化、列表化、删除与改写口令 |
+| **直出与预览边界** | 原文直出、流式 HUD、集成 partial 和快速插入草稿尽量保留 ASR 原文，只做词库、空白、重复转写和非语音垃圾过滤 |
 | **远程 LLM** | 支持 OpenAI、Claude（Anthropic 格式）、Gemini、OpenRouter、硅基流动、豆包、百炼、MiniMax（国内/海外） |
 | **全局快捷键** | 可配置按键（Fn/Ctrl/Shift/Option），支持长按、双击、单击三种触发模式 |
 | **屏幕上下文 OCR** | 通过 ScreenCaptureKit + Vision 截取屏幕文字，辅助 LLM 纠正同音字 |

Sources/App/VoicePipeline+EditCommandResolution.swift

@@ -0,0 +1,61 @@
+import AppKit
+import Foundation
+
+@MainActor
+extension VoicePipeline {
+ func resolvedSpokenEditCommand(
+ raw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async -> SpokenEditCommand? {
+ guard VoicePipelinePolicy.shouldResolveEditCommandWithLLMFirst(outputMode: settings.outputMode) else {
+ return nil
+ }
+
+ let resolution = await resolveSpokenEditCommandWithLLM(
+ raw: raw,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return VoicePipelinePolicy.editCommand(from: resolution)
+ }
+
+ func spokenEditCommandResolutionContext(
+ targetApp: NSRunningApplication?
+ ) async -> SpokenEditCommandResolutionContext {
+ let lastInsertedText = appState.lastInsertedText.trimmingCharacters(in: .whitespacesAndNewlines)
+ let lastInsertion = lastInsertedText.isEmpty
+ ? SpokenEditCommandTargetAvailability.unavailable
+ : .available
+ let selectedText = await textInserter.selectedText(targetApp: targetApp)
+ let selectionAvailability: SpokenEditCommandTargetAvailability
+ if let selectedText {
+ selectionAvailability = selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
+ ? .unavailable
+ : .available
+ } else {
+ selectionAvailability = .unknown
+ }
+
+ return SpokenEditCommandResolutionContext(
+ lastInsertion: lastInsertion,
+ selectedText: selectionAvailability,
+ lastInsertionPreview: SpokenEditCommandResolutionContext.preview(lastInsertedText),
+ selectedTextPreview: SpokenEditCommandResolutionContext.preview(selectedText)
+ )
+ }
+
+ private func resolveSpokenEditCommandWithLLM(
+ raw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async -> SpokenEditCommandLLMResolution? {
+ var options = TextProcessingOptions(settings: settings)
+ options.llmModel = settings.llmModel
+ return await textProcessor.resolveSpokenEditCommandResolution(
+ text: raw,
+ options: options,
+ context: await spokenEditCommandResolutionContext(targetApp: targetApp)
+ )
+ }
+}

Sources/App/VoicePipeline+EditCommands.swift

@@ -0,0 +1,282 @@
+import AppKit
+import Foundation
+
+@MainActor
+extension VoicePipeline {
+ func handleSpokenEditCommandIfNeeded(
+ raw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async -> Bool {
+ guard let command = await resolvedSpokenEditCommand(
+ raw: raw,
+ settings: settings,
+ targetApp: targetApp
+ ) else {
+ return false
+ }
+
+ switch command {
+ case .replaceLast(let replacementRaw):
+ await replaceLastInsertion(
+ raw: raw,
+ replacementRaw: replacementRaw,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .replaceSelection(let replacementRaw):
+ await replaceSelectedText(
+ raw: raw,
+ replacementRaw: replacementRaw,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .rewriteLast(let intent):
+ await rewriteLastInsertion(
+ raw: raw,
+ intent: intent,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .rewriteSelection(let intent):
+ await rewriteSelectedText(
+ raw: raw,
+ intent: intent,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .deleteSelection:
+ await deleteSelectedText(targetApp: targetApp)
+ return true
+ case .undoLastInsertion:
+ await undoLastInsertion(targetApp: targetApp)
+ return true
+ }
+ }
+
+ private func replaceLastInsertion(
+ raw: String,
+ replacementRaw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async {
+ cancelScreenContextCapture()
+
+ guard !appState.lastInsertedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showErrorHint(L("pipeline.no_previous_insert_to_replace"))
+ return
+ }
+
+ let context = replacementInputContext(settings: settings, targetApp: targetApp)
+ let replacementText = finalizedReplacementText(replacementRaw, settings: settings)
+ guard !replacementText.isEmpty else {
+ showNoSpeechDetected(reason: "spoken edit command has empty replacement text")
+ return
+ }
+
+ appState.processedText = replacementText
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ Log.sensitive("[VoicePipeline] voice edit replace \(replacementText.count) chars")
+ let result = await textInserter.replaceRecentInsertion(text: replacementText, targetApp: targetApp)
+
+ InputHistory.shared.addRecord(
+ rawText: raw,
+ processedText: replacementText,
+ wasProcessed: true,
+ context: context
+ )
+
+ appState.lastInsertedText = replacementText
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit replacement probably failed: \(reason)")
+ TextInserter.copyToClipboard(replacementText)
+ showInsertionFailedAlert(text: replacementText, reason: reason)
+ }
+ }
+
+ private func replaceSelectedText(
+ raw: String,
+ replacementRaw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async {
+ cancelScreenContextCapture()
+
+ let context = replacementInputContext(settings: settings, targetApp: targetApp)
+ let replacementText = finalizedReplacementText(replacementRaw, settings: settings)
+ guard !replacementText.isEmpty else {
+ showNoSpeechDetected(reason: "spoken edit command has empty selection replacement text")
+ return
+ }
+
+ appState.processedText = replacementText
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ Log.sensitive("[VoicePipeline] voice edit replace selection \(replacementText.count) chars")
+ let result = await textInserter.replaceSelectedText(text: replacementText, targetApp: targetApp)
+
+ InputHistory.shared.addRecord(
+ rawText: raw,
+ processedText: replacementText,
+ wasProcessed: true,
+ context: context
+ )
+
+ appState.lastInsertedText = replacementText
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit selection replacement probably failed: \(reason)")
+ TextInserter.copyToClipboard(replacementText)
+ showInsertionFailedAlert(text: replacementText, reason: reason)
+ }
+ }
+
+ private func replacementInputContext(
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) -> InputContext {
+ InputContext.capture(
+ targetApp: targetApp,
+ screenContext: "",
+ outputMode: .command,
+ inputLanguage: settings.inputLanguage,
+ source: .menuBar
+ )
+ }
+
+ private func finalizedReplacementText(
+ _ text: String,
+ settings: AppSettings
+ ) -> String {
+ textProcessor.cleanCommandGeneratedOutput(
+ text,
+ inputLanguage: settings.inputLanguage
+ )
+ }
+
+ private func deleteSelectedText(targetApp: NSRunningApplication?) async {
+ cancelScreenContextCapture()
+
+ appState.processedText = ""
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ Log.info("[VoicePipeline] voice edit delete selection")
+ let result = await textInserter.deleteSelectedText(targetApp: targetApp)
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit delete selection probably failed: \(reason)")
+ showErrorHint(reason)
+ return
+ }
+
+ appState.lastInsertedText = ""
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+ }
+
+ private func undoLastInsertion(targetApp: NSRunningApplication?) async {
+ cancelScreenContextCapture()
+
+ guard !appState.lastInsertedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showErrorHint(L("pipeline.no_previous_insert_to_replace"))
+ return
+ }
+
+ appState.processedText = ""
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.undoing")
+
+ Log.info("[VoicePipeline] voice edit undo last insertion")
+ let result = await textInserter.undoRecentInsertion(targetApp: targetApp)
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit undo probably failed: \(reason)")
+ showErrorHint(reason)
+ return
+ }
+
+ appState.lastInsertedText = ""
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+ }
+
+ private func rewriteSelectedText(
+ raw: String,
+ intent: SelectionRewriteIntent,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async {
+ cancelScreenContextCapture()
+
+ guard let selectedText = await textInserter.selectedText(targetApp: targetApp),
+ !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showErrorHint(L("pipeline.no_selected_text_to_replace"))
+ return
+ }
+
+ let context = InputContext.capture(
+ targetApp: targetApp,
+ screenContext: selectedText,
+ outputMode: .command,
+ inputLanguage: settings.inputLanguage,
+ source: .menuBar
+ )
+ var options = TextProcessingOptions(settings: settings)
+ options.llmModel = settings.llmModel
+ let memoryContext = VoicePipelinePolicy.memoryContext(
+ for: .command,
+ settings: settings,
+ currentContext: context
+ )
+
+ appState.phase = .processing
+ appState.statusMessage = L("pipeline.formatting")
+ let rewrittenText = await textProcessor.processSelectionEdit(
+ selectedText: selectedText,
+ intent: intent,
+ options: options,
+ spokenCommand: raw,
+ memoryContext: memoryContext,
+ inputContext: context
+ )
+ guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showNoSpeechDetected(reason: "selection rewrite returned empty text")
+ return
+ }
+
+ appState.processedText = rewrittenText
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp)
+ InputHistory.shared.addRecord(rawText: raw, processedText: rewrittenText, wasProcessed: true, context: context)
+
+ appState.lastInsertedText = rewrittenText
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)")
+ TextInserter.copyToClipboard(rewrittenText)
+ showInsertionFailedAlert(text: rewrittenText, reason: reason)
+ }
+ }
+}

README.md

@@ -39,7 +39,10 @@ Three output modes are available:
 | Feature | Description |
 |---|---|
 | **Multiple Speech Engines** | Apple Speech, WhisperKit, Doubao ASR, Qwen3-ASR, or MiMo-V2.5-ASR |
-| **Smart Text Processing** | Local MLX Qwen2.5/Qwen3 or remote LLM — contextual cleanup, self-correction handling, structured list formatting |
+| **Smart Text Processing** | Local MLX Qwen2.5/Qwen3 or remote LLM infers spoken intent — contextual cleanup, "scratch that" restarts, self-correction handling, spoken punctuation, technical terms, numbers/ranges/units, and structured formatting |
+| **LLM-Owned Spoken Formatting** | Spoken casing, no-space dictation, identifiers, file paths, shortcuts, emoji, Markdown tasks, dates/times, quantities, units, formulas, fractions, and digit sequences are handled by the Smart Format / Voice Command prompts instead of local hardcoded rewrite rules |
+| **Voice Edit Commands** | In Voice Command mode, an LLM classifies safe structured actions for replacing, undoing, proofreading, titling, summarizing, drafting replies, making meeting notes, extracting key points/decisions/questions/risks/deadlines/owners/action items, rewriting tone, expanding, making tables/lists, or deleting the previous OpenType insertion or selected text |
+| **Verbatim & Preview Boundary** | Verbatim mode, streaming HUD, integration partials, and instant-insert drafts keep ASR text close to raw output with only dictionary, whitespace, duplicate, and non-speech-artifact cleanup |
 | **Remote LLM Support** | OpenAI, Claude (Anthropic format), Gemini, OpenRouter, SiliconFlow, Doubao, Bailian, MiniMax (CN & Global) |
 | **Global Hotkey** | Configurable key (Fn/Ctrl/Shift/Option) with long-press, double-tap, or single-tap activation |
 | **Screen Context OCR** | Captures on-screen text via ScreenCaptureKit + Vision to help the LLM correct homophones |

README_zh.md

@@ -39,7 +39,10 @@
 | 功能 | 说明 |
 |---|---|
 | **多语音引擎** | Apple 语音识别、WhisperKit、豆包语音识别、Qwen3-ASR 或 MiMo-V2.5-ASR |
-| **智能文字处理** | 本地 MLX Qwen2.5/Qwen3 或远程 LLM — 上下文感知的语气词清理、自动纠正、列表格式化 |
+| **智能文字处理** | 本地 MLX Qwen2.5/Qwen3 或远程 LLM 理解口述意图 — 上下文感知的语气词清理、“算了/删掉刚才”重说处理、自动纠正、口述标点、技术词、数字/范围/单位和列表格式化 |
+| **LLM 负责口述格式** | 大小写、无空格、标识符、文件路径、快捷键、表情、Markdown 任务、日期时间、数量、单位、公式、分数和数字串都由智能整理/语音指令提示词交给 LLM 判断，不在本地写死替换规则 |
+| **语音编辑口令** | 在语音指令模式下，由 LLM 分类安全结构化动作，支持上一段/选区替换、撤销、校对、跨语言回复起草、接受/拒绝/追问回复、会议纪要、关键要点/结论/问题/风险/截止时间/负责人/行动项提取、标题化、摘要、语气改写、扩写、表格化、列表化、删除与改写口令 |
+| **直出与预览边界** | 原文直出、流式 HUD、集成 partial 和快速插入草稿尽量保留 ASR 原文，只做词库、空白、重复转写和非语音垃圾过滤 |
 | **远程 LLM** | 支持 OpenAI、Claude（Anthropic 格式）、Gemini、OpenRouter、硅基流动、豆包、百炼、MiniMax（国内/海外） |
 | **全局快捷键** | 可配置按键（Fn/Ctrl/Shift/Option），支持长按、双击、单击三种触发模式 |
 | **屏幕上下文 OCR** | 通过 ScreenCaptureKit + Vision 截取屏幕文字，辅助 LLM 纠正同音字 |

Sources/App/VoicePipeline+EditCommandResolution.swift

@@ -0,0 +1,61 @@
+import AppKit
+import Foundation
+
+@MainActor
+extension VoicePipeline {
+ func resolvedSpokenEditCommand(
+ raw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async -> SpokenEditCommand? {
+ guard VoicePipelinePolicy.shouldResolveEditCommandWithLLMFirst(outputMode: settings.outputMode) else {
+ return nil
+ }
+
+ let resolution = await resolveSpokenEditCommandWithLLM(
+ raw: raw,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return VoicePipelinePolicy.editCommand(from: resolution)
+ }
+
+ func spokenEditCommandResolutionContext(
+ targetApp: NSRunningApplication?
+ ) async -> SpokenEditCommandResolutionContext {
+ let lastInsertedText = appState.lastInsertedText.trimmingCharacters(in: .whitespacesAndNewlines)
+ let lastInsertion = lastInsertedText.isEmpty
+ ? SpokenEditCommandTargetAvailability.unavailable
+ : .available
+ let selectedText = await textInserter.selectedText(targetApp: targetApp)
+ let selectionAvailability: SpokenEditCommandTargetAvailability
+ if let selectedText {
+ selectionAvailability = selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
+ ? .unavailable
+ : .available
+ } else {
+ selectionAvailability = .unknown
+ }
+
+ return SpokenEditCommandResolutionContext(
+ lastInsertion: lastInsertion,
+ selectedText: selectionAvailability,
+ lastInsertionPreview: SpokenEditCommandResolutionContext.preview(lastInsertedText),
+ selectedTextPreview: SpokenEditCommandResolutionContext.preview(selectedText)
+ )
+ }
+
+ private func resolveSpokenEditCommandWithLLM(
+ raw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async -> SpokenEditCommandLLMResolution? {
+ var options = TextProcessingOptions(settings: settings)
+ options.llmModel = settings.llmModel
+ return await textProcessor.resolveSpokenEditCommandResolution(
+ text: raw,
+ options: options,
+ context: await spokenEditCommandResolutionContext(targetApp: targetApp)
+ )
+ }
+}

Sources/App/VoicePipeline+EditCommands.swift

@@ -0,0 +1,282 @@
+import AppKit
+import Foundation
+
+@MainActor
+extension VoicePipeline {
+ func handleSpokenEditCommandIfNeeded(
+ raw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async -> Bool {
+ guard let command = await resolvedSpokenEditCommand(
+ raw: raw,
+ settings: settings,
+ targetApp: targetApp
+ ) else {
+ return false
+ }
+
+ switch command {
+ case .replaceLast(let replacementRaw):
+ await replaceLastInsertion(
+ raw: raw,
+ replacementRaw: replacementRaw,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .replaceSelection(let replacementRaw):
+ await replaceSelectedText(
+ raw: raw,
+ replacementRaw: replacementRaw,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .rewriteLast(let intent):
+ await rewriteLastInsertion(
+ raw: raw,
+ intent: intent,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .rewriteSelection(let intent):
+ await rewriteSelectedText(
+ raw: raw,
+ intent: intent,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .deleteSelection:
+ await deleteSelectedText(targetApp: targetApp)
+ return true
+ case .undoLastInsertion:
+ await undoLastInsertion(targetApp: targetApp)
+ return true
+ }
+ }
+
+ private func replaceLastInsertion(
+ raw: String,
+ replacementRaw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async {
+ cancelScreenContextCapture()
+
+ guard !appState.lastInsertedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showErrorHint(L("pipeline.no_previous_insert_to_replace"))
+ return
+ }
+
+ let context = replacementInputContext(settings: settings, targetApp: targetApp)
+ let replacementText = finalizedReplacementText(replacementRaw, settings: settings)
+ guard !replacementText.isEmpty else {
+ showNoSpeechDetected(reason: "spoken edit command has empty replacement text")
+ return
+ }
+
+ appState.processedText = replacementText
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ Log.sensitive("[VoicePipeline] voice edit replace \(replacementText.count) chars")
+ let result = await textInserter.replaceRecentInsertion(text: replacementText, targetApp: targetApp)
+
+ InputHistory.shared.addRecord(
+ rawText: raw,
+ processedText: replacementText,
+ wasProcessed: true,
+ context: context
+ )
+
+ appState.lastInsertedText = replacementText
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit replacement probably failed: \(reason)")
+ TextInserter.copyToClipboard(replacementText)
+ showInsertionFailedAlert(text: replacementText, reason: reason)
+ }
+ }
+
+ private func replaceSelectedText(
+ raw: String,
+ replacementRaw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async {
+ cancelScreenContextCapture()
+
+ let context = replacementInputContext(settings: settings, targetApp: targetApp)
+ let replacementText = finalizedReplacementText(replacementRaw, settings: settings)
+ guard !replacementText.isEmpty else {
+ showNoSpeechDetected(reason: "spoken edit command has empty selection replacement text")
+ return
+ }
+
+ appState.processedText = replacementText
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ Log.sensitive("[VoicePipeline] voice edit replace selection \(replacementText.count) chars")
+ let result = await textInserter.replaceSelectedText(text: replacementText, targetApp: targetApp)
+
+ InputHistory.shared.addRecord(
+ rawText: raw,
+ processedText: replacementText,
+ wasProcessed: true,
+ context: context
+ )
+
+ appState.lastInsertedText = replacementText
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit selection replacement probably failed: \(reason)")
+ TextInserter.copyToClipboard(replacementText)
+ showInsertionFailedAlert(text: replacementText, reason: reason)
+ }
+ }
+
+ private func replacementInputContext(
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) -> InputContext {
+ InputContext.capture(
+ targetApp: targetApp,
+ screenContext: "",
+ outputMode: .command,
+ inputLanguage: settings.inputLanguage,
+ source: .menuBar
+ )
+ }
+
+ private func finalizedReplacementText(
+ _ text: String,
+ settings: AppSettings
+ ) -> String {
+ textProcessor.cleanCommandGeneratedOutput(
+ text,
+ inputLanguage: settings.inputLanguage
+ )
+ }
+
+ private func deleteSelectedText(targetApp: NSRunningApplication?) async {
+ cancelScreenContextCapture()
+
+ appState.processedText = ""
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ Log.info("[VoicePipeline] voice edit delete selection")
+ let result = await textInserter.deleteSelectedText(targetApp: targetApp)
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit delete selection probably failed: \(reason)")
+ showErrorHint(reason)
+ return
+ }
+
+ appState.lastInsertedText = ""
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+ }
+
+ private func undoLastInsertion(targetApp: NSRunningApplication?) async {
+ cancelScreenContextCapture()
+
+ guard !appState.lastInsertedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showErrorHint(L("pipeline.no_previous_insert_to_replace"))
+ return
+ }
+
+ appState.processedText = ""
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.undoing")
+
+ Log.info("[VoicePipeline] voice edit undo last insertion")
+ let result = await textInserter.undoRecentInsertion(targetApp: targetApp)
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit undo probably failed: \(reason)")
+ showErrorHint(reason)
+ return
+ }
+
+ appState.lastInsertedText = ""
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+ }
+
+ private func rewriteSelectedText(
+ raw: String,
+ intent: SelectionRewriteIntent,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async {
+ cancelScreenContextCapture()
+
+ guard let selectedText = await textInserter.selectedText(targetApp: targetApp),
+ !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showErrorHint(L("pipeline.no_selected_text_to_replace"))
+ return
+ }
+
+ let context = InputContext.capture(
+ targetApp: targetApp,
+ screenContext: selectedText,
+ outputMode: .command,
+ inputLanguage: settings.inputLanguage,
+ source: .menuBar
+ )
+ var options = TextProcessingOptions(settings: settings)
+ options.llmModel = settings.llmModel
+ let memoryContext = VoicePipelinePolicy.memoryContext(
+ for: .command,
+ settings: settings,
+ currentContext: context
+ )
+
+ appState.phase = .processing
+ appState.statusMessage = L("pipeline.formatting")
+ let rewrittenText = await textProcessor.processSelectionEdit(
+ selectedText: selectedText,
+ intent: intent,
+ options: options,
+ spokenCommand: raw,
+ memoryContext: memoryContext,
+ inputContext: context
+ )
+ guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showNoSpeechDetected(reason: "selection rewrite returned empty text")
+ return
+ }
+
+ appState.processedText = rewrittenText
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp)
+ InputHistory.shared.addRecord(rawText: raw, processedText: rewrittenText, wasProcessed: true, context: context)
+
+ appState.lastInsertedText = rewrittenText
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)")
+ TextInserter.copyToClipboard(rewrittenText)
+ showInsertionFailedAlert(text: rewrittenText, reason: reason)
+ }
+ }
+}

README.md

@@ -39,7 +39,10 @@ Three output modes are available:
 | Feature | Description |
 |---|---|
 | **Multiple Speech Engines** | Apple Speech, WhisperKit, Doubao ASR, Qwen3-ASR, or MiMo-V2.5-ASR |
-| **Smart Text Processing** | Local MLX Qwen2.5/Qwen3 or remote LLM — contextual cleanup, self-correction handling, structured list formatting |
+| **Smart Text Processing** | Local MLX Qwen2.5/Qwen3 or remote LLM infers spoken intent — contextual cleanup, "scratch that" restarts, self-correction handling, spoken punctuation, technical terms, numbers/ranges/units, and structured formatting |
+| **LLM-Owned Spoken Formatting** | Spoken casing, no-space dictation, identifiers, file paths, shortcuts, emoji, Markdown tasks, dates/times, quantities, units, formulas, fractions, and digit sequences are handled by the Smart Format / Voice Command prompts instead of local hardcoded rewrite rules |
+| **Voice Edit Commands** | In Voice Command mode, an LLM classifies safe structured actions for replacing, undoing, proofreading, titling, summarizing, drafting replies, making meeting notes, extracting key points/decisions/questions/risks/deadlines/owners/action items, rewriting tone, expanding, making tables/lists, or deleting the previous OpenType insertion or selected text |
+| **Verbatim & Preview Boundary** | Verbatim mode, streaming HUD, integration partials, and instant-insert drafts keep ASR text close to raw output with only dictionary, whitespace, duplicate, and non-speech-artifact cleanup |
 | **Remote LLM Support** | OpenAI, Claude (Anthropic format), Gemini, OpenRouter, SiliconFlow, Doubao, Bailian, MiniMax (CN & Global) |
 | **Global Hotkey** | Configurable key (Fn/Ctrl/Shift/Option) with long-press, double-tap, or single-tap activation |
 | **Screen Context OCR** | Captures on-screen text via ScreenCaptureKit + Vision to help the LLM correct homophones |

README_zh.md

@@ -39,7 +39,10 @@
 | 功能 | 说明 |
 |---|---|
 | **多语音引擎** | Apple 语音识别、WhisperKit、豆包语音识别、Qwen3-ASR 或 MiMo-V2.5-ASR |
-| **智能文字处理** | 本地 MLX Qwen2.5/Qwen3 或远程 LLM — 上下文感知的语气词清理、自动纠正、列表格式化 |
+| **智能文字处理** | 本地 MLX Qwen2.5/Qwen3 或远程 LLM 理解口述意图 — 上下文感知的语气词清理、“算了/删掉刚才”重说处理、自动纠正、口述标点、技术词、数字/范围/单位和列表格式化 |
+| **LLM 负责口述格式** | 大小写、无空格、标识符、文件路径、快捷键、表情、Markdown 任务、日期时间、数量、单位、公式、分数和数字串都由智能整理/语音指令提示词交给 LLM 判断，不在本地写死替换规则 |
+| **语音编辑口令** | 在语音指令模式下，由 LLM 分类安全结构化动作，支持上一段/选区替换、撤销、校对、跨语言回复起草、接受/拒绝/追问回复、会议纪要、关键要点/结论/问题/风险/截止时间/负责人/行动项提取、标题化、摘要、语气改写、扩写、表格化、列表化、删除与改写口令 |
+| **直出与预览边界** | 原文直出、流式 HUD、集成 partial 和快速插入草稿尽量保留 ASR 原文，只做词库、空白、重复转写和非语音垃圾过滤 |
 | **远程 LLM** | 支持 OpenAI、Claude（Anthropic 格式）、Gemini、OpenRouter、硅基流动、豆包、百炼、MiniMax（国内/海外） |
 | **全局快捷键** | 可配置按键（Fn/Ctrl/Shift/Option），支持长按、双击、单击三种触发模式 |
 | **屏幕上下文 OCR** | 通过 ScreenCaptureKit + Vision 截取屏幕文字，辅助 LLM 纠正同音字 |

Sources/App/VoicePipeline+EditCommandResolution.swift

@@ -0,0 +1,61 @@
+import AppKit
+import Foundation
+
+@MainActor
+extension VoicePipeline {
+ func resolvedSpokenEditCommand(
+ raw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async -> SpokenEditCommand? {
+ guard VoicePipelinePolicy.shouldResolveEditCommandWithLLMFirst(outputMode: settings.outputMode) else {
+ return nil
+ }
+
+ let resolution = await resolveSpokenEditCommandWithLLM(
+ raw: raw,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return VoicePipelinePolicy.editCommand(from: resolution)
+ }
+
+ func spokenEditCommandResolutionContext(
+ targetApp: NSRunningApplication?
+ ) async -> SpokenEditCommandResolutionContext {
+ let lastInsertedText = appState.lastInsertedText.trimmingCharacters(in: .whitespacesAndNewlines)
+ let lastInsertion = lastInsertedText.isEmpty
+ ? SpokenEditCommandTargetAvailability.unavailable
+ : .available
+ let selectedText = await textInserter.selectedText(targetApp: targetApp)
+ let selectionAvailability: SpokenEditCommandTargetAvailability
+ if let selectedText {
+ selectionAvailability = selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
+ ? .unavailable
+ : .available
+ } else {
+ selectionAvailability = .unknown
+ }
+
+ return SpokenEditCommandResolutionContext(
+ lastInsertion: lastInsertion,
+ selectedText: selectionAvailability,
+ lastInsertionPreview: SpokenEditCommandResolutionContext.preview(lastInsertedText),
+ selectedTextPreview: SpokenEditCommandResolutionContext.preview(selectedText)
+ )
+ }
+
+ private func resolveSpokenEditCommandWithLLM(
+ raw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async -> SpokenEditCommandLLMResolution? {
+ var options = TextProcessingOptions(settings: settings)
+ options.llmModel = settings.llmModel
+ return await textProcessor.resolveSpokenEditCommandResolution(
+ text: raw,
+ options: options,
+ context: await spokenEditCommandResolutionContext(targetApp: targetApp)
+ )
+ }
+}

Sources/App/VoicePipeline+EditCommands.swift

@@ -0,0 +1,282 @@
+import AppKit
+import Foundation
+
+@MainActor
+extension VoicePipeline {
+ func handleSpokenEditCommandIfNeeded(
+ raw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async -> Bool {
+ guard let command = await resolvedSpokenEditCommand(
+ raw: raw,
+ settings: settings,
+ targetApp: targetApp
+ ) else {
+ return false
+ }
+
+ switch command {
+ case .replaceLast(let replacementRaw):
+ await replaceLastInsertion(
+ raw: raw,
+ replacementRaw: replacementRaw,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .replaceSelection(let replacementRaw):
+ await replaceSelectedText(
+ raw: raw,
+ replacementRaw: replacementRaw,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .rewriteLast(let intent):
+ await rewriteLastInsertion(
+ raw: raw,
+ intent: intent,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .rewriteSelection(let intent):
+ await rewriteSelectedText(
+ raw: raw,
+ intent: intent,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .deleteSelection:
+ await deleteSelectedText(targetApp: targetApp)
+ return true
+ case .undoLastInsertion:
+ await undoLastInsertion(targetApp: targetApp)
+ return true
+ }
+ }
+
+ private func replaceLastInsertion(
+ raw: String,
+ replacementRaw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async {
+ cancelScreenContextCapture()
+
+ guard !appState.lastInsertedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showErrorHint(L("pipeline.no_previous_insert_to_replace"))
+ return
+ }
+
+ let context = replacementInputContext(settings: settings, targetApp: targetApp)
+ let replacementText = finalizedReplacementText(replacementRaw, settings: settings)
+ guard !replacementText.isEmpty else {
+ showNoSpeechDetected(reason: "spoken edit command has empty replacement text")
+ return
+ }
+
+ appState.processedText = replacementText
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ Log.sensitive("[VoicePipeline] voice edit replace \(replacementText.count) chars")
+ let result = await textInserter.replaceRecentInsertion(text: replacementText, targetApp: targetApp)
+
+ InputHistory.shared.addRecord(
+ rawText: raw,
+ processedText: replacementText,
+ wasProcessed: true,
+ context: context
+ )
+
+ appState.lastInsertedText = replacementText
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit replacement probably failed: \(reason)")
+ TextInserter.copyToClipboard(replacementText)
+ showInsertionFailedAlert(text: replacementText, reason: reason)
+ }
+ }
+
+ private func replaceSelectedText(
+ raw: String,
+ replacementRaw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async {
+ cancelScreenContextCapture()
+
+ let context = replacementInputContext(settings: settings, targetApp: targetApp)
+ let replacementText = finalizedReplacementText(replacementRaw, settings: settings)
+ guard !replacementText.isEmpty else {
+ showNoSpeechDetected(reason: "spoken edit command has empty selection replacement text")
+ return
+ }
+
+ appState.processedText = replacementText
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ Log.sensitive("[VoicePipeline] voice edit replace selection \(replacementText.count) chars")
+ let result = await textInserter.replaceSelectedText(text: replacementText, targetApp: targetApp)
+
+ InputHistory.shared.addRecord(
+ rawText: raw,
+ processedText: replacementText,
+ wasProcessed: true,
+ context: context
+ )
+
+ appState.lastInsertedText = replacementText
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit selection replacement probably failed: \(reason)")
+ TextInserter.copyToClipboard(replacementText)
+ showInsertionFailedAlert(text: replacementText, reason: reason)
+ }
+ }
+
+ private func replacementInputContext(
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) -> InputContext {
+ InputContext.capture(
+ targetApp: targetApp,
+ screenContext: "",
+ outputMode: .command,
+ inputLanguage: settings.inputLanguage,
+ source: .menuBar
+ )
+ }
+
+ private func finalizedReplacementText(
+ _ text: String,
+ settings: AppSettings
+ ) -> String {
+ textProcessor.cleanCommandGeneratedOutput(
+ text,
+ inputLanguage: settings.inputLanguage
+ )
+ }
+
+ private func deleteSelectedText(targetApp: NSRunningApplication?) async {
+ cancelScreenContextCapture()
+
+ appState.processedText = ""
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ Log.info("[VoicePipeline] voice edit delete selection")
+ let result = await textInserter.deleteSelectedText(targetApp: targetApp)
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit delete selection probably failed: \(reason)")
+ showErrorHint(reason)
+ return
+ }
+
+ appState.lastInsertedText = ""
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+ }
+
+ private func undoLastInsertion(targetApp: NSRunningApplication?) async {
+ cancelScreenContextCapture()
+
+ guard !appState.lastInsertedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showErrorHint(L("pipeline.no_previous_insert_to_replace"))
+ return
+ }
+
+ appState.processedText = ""
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.undoing")
+
+ Log.info("[VoicePipeline] voice edit undo last insertion")
+ let result = await textInserter.undoRecentInsertion(targetApp: targetApp)
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit undo probably failed: \(reason)")
+ showErrorHint(reason)
+ return
+ }
+
+ appState.lastInsertedText = ""
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+ }
+
+ private func rewriteSelectedText(
+ raw: String,
+ intent: SelectionRewriteIntent,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async {
+ cancelScreenContextCapture()
+
+ guard let selectedText = await textInserter.selectedText(targetApp: targetApp),
+ !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showErrorHint(L("pipeline.no_selected_text_to_replace"))
+ return
+ }
+
+ let context = InputContext.capture(
+ targetApp: targetApp,
+ screenContext: selectedText,
+ outputMode: .command,
+ inputLanguage: settings.inputLanguage,
+ source: .menuBar
+ )
+ var options = TextProcessingOptions(settings: settings)
+ options.llmModel = settings.llmModel
+ let memoryContext = VoicePipelinePolicy.memoryContext(
+ for: .command,
+ settings: settings,
+ currentContext: context
+ )
+
+ appState.phase = .processing
+ appState.statusMessage = L("pipeline.formatting")
+ let rewrittenText = await textProcessor.processSelectionEdit(
+ selectedText: selectedText,
+ intent: intent,
+ options: options,
+ spokenCommand: raw,
+ memoryContext: memoryContext,
+ inputContext: context
+ )
+ guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showNoSpeechDetected(reason: "selection rewrite returned empty text")
+ return
+ }
+
+ appState.processedText = rewrittenText
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp)
+ InputHistory.shared.addRecord(rawText: raw, processedText: rewrittenText, wasProcessed: true, context: context)
+
+ appState.lastInsertedText = rewrittenText
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)")
+ TextInserter.copyToClipboard(rewrittenText)
+ showInsertionFailedAlert(text: rewrittenText, reason: reason)
+ }
+ }
+}

README.md

@@ -39,7 +39,10 @@ Three output modes are available:
 | Feature | Description |
 |---|---|
 | **Multiple Speech Engines** | Apple Speech, WhisperKit, Doubao ASR, Qwen3-ASR, or MiMo-V2.5-ASR |
-| **Smart Text Processing** | Local MLX Qwen2.5/Qwen3 or remote LLM — contextual cleanup, self-correction handling, structured list formatting |
+| **Smart Text Processing** | Local MLX Qwen2.5/Qwen3 or remote LLM infers spoken intent — contextual cleanup, "scratch that" restarts, self-correction handling, spoken punctuation, technical terms, numbers/ranges/units, and structured formatting |
+| **LLM-Owned Spoken Formatting** | Spoken casing, no-space dictation, identifiers, file paths, shortcuts, emoji, Markdown tasks, dates/times, quantities, units, formulas, fractions, and digit sequences are handled by the Smart Format / Voice Command prompts instead of local hardcoded rewrite rules |
+| **Voice Edit Commands** | In Voice Command mode, an LLM classifies safe structured actions for replacing, undoing, proofreading, titling, summarizing, drafting replies, making meeting notes, extracting key points/decisions/questions/risks/deadlines/owners/action items, rewriting tone, expanding, making tables/lists, or deleting the previous OpenType insertion or selected text |
+| **Verbatim & Preview Boundary** | Verbatim mode, streaming HUD, integration partials, and instant-insert drafts keep ASR text close to raw output with only dictionary, whitespace, duplicate, and non-speech-artifact cleanup |
 | **Remote LLM Support** | OpenAI, Claude (Anthropic format), Gemini, OpenRouter, SiliconFlow, Doubao, Bailian, MiniMax (CN & Global) |
 | **Global Hotkey** | Configurable key (Fn/Ctrl/Shift/Option) with long-press, double-tap, or single-tap activation |
 | **Screen Context OCR** | Captures on-screen text via ScreenCaptureKit + Vision to help the LLM correct homophones |

README_zh.md

@@ -39,7 +39,10 @@
 | 功能 | 说明 |
 |---|---|
 | **多语音引擎** | Apple 语音识别、WhisperKit、豆包语音识别、Qwen3-ASR 或 MiMo-V2.5-ASR |
-| **智能文字处理** | 本地 MLX Qwen2.5/Qwen3 或远程 LLM — 上下文感知的语气词清理、自动纠正、列表格式化 |
+| **智能文字处理** | 本地 MLX Qwen2.5/Qwen3 或远程 LLM 理解口述意图 — 上下文感知的语气词清理、“算了/删掉刚才”重说处理、自动纠正、口述标点、技术词、数字/范围/单位和列表格式化 |
+| **LLM 负责口述格式** | 大小写、无空格、标识符、文件路径、快捷键、表情、Markdown 任务、日期时间、数量、单位、公式、分数和数字串都由智能整理/语音指令提示词交给 LLM 判断，不在本地写死替换规则 |
+| **语音编辑口令** | 在语音指令模式下，由 LLM 分类安全结构化动作，支持上一段/选区替换、撤销、校对、跨语言回复起草、接受/拒绝/追问回复、会议纪要、关键要点/结论/问题/风险/截止时间/负责人/行动项提取、标题化、摘要、语气改写、扩写、表格化、列表化、删除与改写口令 |
+| **直出与预览边界** | 原文直出、流式 HUD、集成 partial 和快速插入草稿尽量保留 ASR 原文，只做词库、空白、重复转写和非语音垃圾过滤 |
 | **远程 LLM** | 支持 OpenAI、Claude（Anthropic 格式）、Gemini、OpenRouter、硅基流动、豆包、百炼、MiniMax（国内/海外） |
 | **全局快捷键** | 可配置按键（Fn/Ctrl/Shift/Option），支持长按、双击、单击三种触发模式 |
 | **屏幕上下文 OCR** | 通过 ScreenCaptureKit + Vision 截取屏幕文字，辅助 LLM 纠正同音字 |

Sources/App/VoicePipeline+EditCommandResolution.swift

@@ -0,0 +1,61 @@
+import AppKit
+import Foundation
+
+@MainActor
+extension VoicePipeline {
+ func resolvedSpokenEditCommand(
+ raw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async -> SpokenEditCommand? {
+ guard VoicePipelinePolicy.shouldResolveEditCommandWithLLMFirst(outputMode: settings.outputMode) else {
+ return nil
+ }
+
+ let resolution = await resolveSpokenEditCommandWithLLM(
+ raw: raw,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return VoicePipelinePolicy.editCommand(from: resolution)
+ }
+
+ func spokenEditCommandResolutionContext(
+ targetApp: NSRunningApplication?
+ ) async -> SpokenEditCommandResolutionContext {
+ let lastInsertedText = appState.lastInsertedText.trimmingCharacters(in: .whitespacesAndNewlines)
+ let lastInsertion = lastInsertedText.isEmpty
+ ? SpokenEditCommandTargetAvailability.unavailable
+ : .available
+ let selectedText = await textInserter.selectedText(targetApp: targetApp)
+ let selectionAvailability: SpokenEditCommandTargetAvailability
+ if let selectedText {
+ selectionAvailability = selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
+ ? .unavailable
+ : .available
+ } else {
+ selectionAvailability = .unknown
+ }
+
+ return SpokenEditCommandResolutionContext(
+ lastInsertion: lastInsertion,
+ selectedText: selectionAvailability,
+ lastInsertionPreview: SpokenEditCommandResolutionContext.preview(lastInsertedText),
+ selectedTextPreview: SpokenEditCommandResolutionContext.preview(selectedText)
+ )
+ }
+
+ private func resolveSpokenEditCommandWithLLM(
+ raw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async -> SpokenEditCommandLLMResolution? {
+ var options = TextProcessingOptions(settings: settings)
+ options.llmModel = settings.llmModel
+ return await textProcessor.resolveSpokenEditCommandResolution(
+ text: raw,
+ options: options,
+ context: await spokenEditCommandResolutionContext(targetApp: targetApp)
+ )
+ }
+}

Sources/App/VoicePipeline+EditCommands.swift

@@ -0,0 +1,282 @@
+import AppKit
+import Foundation
+
+@MainActor
+extension VoicePipeline {
+ func handleSpokenEditCommandIfNeeded(
+ raw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async -> Bool {
+ guard let command = await resolvedSpokenEditCommand(
+ raw: raw,
+ settings: settings,
+ targetApp: targetApp
+ ) else {
+ return false
+ }
+
+ switch command {
+ case .replaceLast(let replacementRaw):
+ await replaceLastInsertion(
+ raw: raw,
+ replacementRaw: replacementRaw,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .replaceSelection(let replacementRaw):
+ await replaceSelectedText(
+ raw: raw,
+ replacementRaw: replacementRaw,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .rewriteLast(let intent):
+ await rewriteLastInsertion(
+ raw: raw,
+ intent: intent,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .rewriteSelection(let intent):
+ await rewriteSelectedText(
+ raw: raw,
+ intent: intent,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .deleteSelection:
+ await deleteSelectedText(targetApp: targetApp)
+ return true
+ case .undoLastInsertion:
+ await undoLastInsertion(targetApp: targetApp)
+ return true
+ }
+ }
+
+ private func replaceLastInsertion(
+ raw: String,
+ replacementRaw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async {
+ cancelScreenContextCapture()
+
+ guard !appState.lastInsertedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showErrorHint(L("pipeline.no_previous_insert_to_replace"))
+ return
+ }
+
+ let context = replacementInputContext(settings: settings, targetApp: targetApp)
+ let replacementText = finalizedReplacementText(replacementRaw, settings: settings)
+ guard !replacementText.isEmpty else {
+ showNoSpeechDetected(reason: "spoken edit command has empty replacement text")
+ return
+ }
+
+ appState.processedText = replacementText
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ Log.sensitive("[VoicePipeline] voice edit replace \(replacementText.count) chars")
+ let result = await textInserter.replaceRecentInsertion(text: replacementText, targetApp: targetApp)
+
+ InputHistory.shared.addRecord(
+ rawText: raw,
+ processedText: replacementText,
+ wasProcessed: true,
+ context: context
+ )
+
+ appState.lastInsertedText = replacementText
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit replacement probably failed: \(reason)")
+ TextInserter.copyToClipboard(replacementText)
+ showInsertionFailedAlert(text: replacementText, reason: reason)
+ }
+ }
+
+ private func replaceSelectedText(
+ raw: String,
+ replacementRaw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async {
+ cancelScreenContextCapture()
+
+ let context = replacementInputContext(settings: settings, targetApp: targetApp)
+ let replacementText = finalizedReplacementText(replacementRaw, settings: settings)
+ guard !replacementText.isEmpty else {
+ showNoSpeechDetected(reason: "spoken edit command has empty selection replacement text")
+ return
+ }
+
+ appState.processedText = replacementText
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ Log.sensitive("[VoicePipeline] voice edit replace selection \(replacementText.count) chars")
+ let result = await textInserter.replaceSelectedText(text: replacementText, targetApp: targetApp)
+
+ InputHistory.shared.addRecord(
+ rawText: raw,
+ processedText: replacementText,
+ wasProcessed: true,
+ context: context
+ )
+
+ appState.lastInsertedText = replacementText
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit selection replacement probably failed: \(reason)")
+ TextInserter.copyToClipboard(replacementText)
+ showInsertionFailedAlert(text: replacementText, reason: reason)
+ }
+ }
+
+ private func replacementInputContext(
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) -> InputContext {
+ InputContext.capture(
+ targetApp: targetApp,
+ screenContext: "",
+ outputMode: .command,
+ inputLanguage: settings.inputLanguage,
+ source: .menuBar
+ )
+ }
+
+ private func finalizedReplacementText(
+ _ text: String,
+ settings: AppSettings
+ ) -> String {
+ textProcessor.cleanCommandGeneratedOutput(
+ text,
+ inputLanguage: settings.inputLanguage
+ )
+ }
+
+ private func deleteSelectedText(targetApp: NSRunningApplication?) async {
+ cancelScreenContextCapture()
+
+ appState.processedText = ""
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ Log.info("[VoicePipeline] voice edit delete selection")
+ let result = await textInserter.deleteSelectedText(targetApp: targetApp)
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit delete selection probably failed: \(reason)")
+ showErrorHint(reason)
+ return
+ }
+
+ appState.lastInsertedText = ""
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+ }
+
+ private func undoLastInsertion(targetApp: NSRunningApplication?) async {
+ cancelScreenContextCapture()
+
+ guard !appState.lastInsertedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showErrorHint(L("pipeline.no_previous_insert_to_replace"))
+ return
+ }
+
+ appState.processedText = ""
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.undoing")
+
+ Log.info("[VoicePipeline] voice edit undo last insertion")
+ let result = await textInserter.undoRecentInsertion(targetApp: targetApp)
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit undo probably failed: \(reason)")
+ showErrorHint(reason)
+ return
+ }
+
+ appState.lastInsertedText = ""
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+ }
+
+ private func rewriteSelectedText(
+ raw: String,
+ intent: SelectionRewriteIntent,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async {
+ cancelScreenContextCapture()
+
+ guard let selectedText = await textInserter.selectedText(targetApp: targetApp),
+ !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showErrorHint(L("pipeline.no_selected_text_to_replace"))
+ return
+ }
+
+ let context = InputContext.capture(
+ targetApp: targetApp,
+ screenContext: selectedText,
+ outputMode: .command,
+ inputLanguage: settings.inputLanguage,
+ source: .menuBar
+ )
+ var options = TextProcessingOptions(settings: settings)
+ options.llmModel = settings.llmModel
+ let memoryContext = VoicePipelinePolicy.memoryContext(
+ for: .command,
+ settings: settings,
+ currentContext: context
+ )
+
+ appState.phase = .processing
+ appState.statusMessage = L("pipeline.formatting")
+ let rewrittenText = await textProcessor.processSelectionEdit(
+ selectedText: selectedText,
+ intent: intent,
+ options: options,
+ spokenCommand: raw,
+ memoryContext: memoryContext,
+ inputContext: context
+ )
+ guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showNoSpeechDetected(reason: "selection rewrite returned empty text")
+ return
+ }
+
+ appState.processedText = rewrittenText
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp)
+ InputHistory.shared.addRecord(rawText: raw, processedText: rewrittenText, wasProcessed: true, context: context)
+
+ appState.lastInsertedText = rewrittenText
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)")
+ TextInserter.copyToClipboard(rewrittenText)
+ showInsertionFailedAlert(text: rewrittenText, reason: reason)
+ }
+ }
+}

README.md

@@ -39,7 +39,10 @@ Three output modes are available:
 | Feature | Description |
 |---|---|
 | **Multiple Speech Engines** | Apple Speech, WhisperKit, Doubao ASR, Qwen3-ASR, or MiMo-V2.5-ASR |
-| **Smart Text Processing** | Local MLX Qwen2.5/Qwen3 or remote LLM — contextual cleanup, self-correction handling, structured list formatting |
+| **Smart Text Processing** | Local MLX Qwen2.5/Qwen3 or remote LLM infers spoken intent — contextual cleanup, "scratch that" restarts, self-correction handling, spoken punctuation, technical terms, numbers/ranges/units, and structured formatting |
+| **LLM-Owned Spoken Formatting** | Spoken casing, no-space dictation, identifiers, file paths, shortcuts, emoji, Markdown tasks, dates/times, quantities, units, formulas, fractions, and digit sequences are handled by the Smart Format / Voice Command prompts instead of local hardcoded rewrite rules |
+| **Voice Edit Commands** | In Voice Command mode, an LLM classifies safe structured actions for replacing, undoing, proofreading, titling, summarizing, drafting replies, making meeting notes, extracting key points/decisions/questions/risks/deadlines/owners/action items, rewriting tone, expanding, making tables/lists, or deleting the previous OpenType insertion or selected text |
+| **Verbatim & Preview Boundary** | Verbatim mode, streaming HUD, integration partials, and instant-insert drafts keep ASR text close to raw output with only dictionary, whitespace, duplicate, and non-speech-artifact cleanup |
 | **Remote LLM Support** | OpenAI, Claude (Anthropic format), Gemini, OpenRouter, SiliconFlow, Doubao, Bailian, MiniMax (CN & Global) |
 | **Global Hotkey** | Configurable key (Fn/Ctrl/Shift/Option) with long-press, double-tap, or single-tap activation |
 | **Screen Context OCR** | Captures on-screen text via ScreenCaptureKit + Vision to help the LLM correct homophones |

README_zh.md

@@ -39,7 +39,10 @@
 | 功能 | 说明 |
 |---|---|
 | **多语音引擎** | Apple 语音识别、WhisperKit、豆包语音识别、Qwen3-ASR 或 MiMo-V2.5-ASR |
-| **智能文字处理** | 本地 MLX Qwen2.5/Qwen3 或远程 LLM — 上下文感知的语气词清理、自动纠正、列表格式化 |
+| **智能文字处理** | 本地 MLX Qwen2.5/Qwen3 或远程 LLM 理解口述意图 — 上下文感知的语气词清理、“算了/删掉刚才”重说处理、自动纠正、口述标点、技术词、数字/范围/单位和列表格式化 |
+| **LLM 负责口述格式** | 大小写、无空格、标识符、文件路径、快捷键、表情、Markdown 任务、日期时间、数量、单位、公式、分数和数字串都由智能整理/语音指令提示词交给 LLM 判断，不在本地写死替换规则 |
+| **语音编辑口令** | 在语音指令模式下，由 LLM 分类安全结构化动作，支持上一段/选区替换、撤销、校对、跨语言回复起草、接受/拒绝/追问回复、会议纪要、关键要点/结论/问题/风险/截止时间/负责人/行动项提取、标题化、摘要、语气改写、扩写、表格化、列表化、删除与改写口令 |
+| **直出与预览边界** | 原文直出、流式 HUD、集成 partial 和快速插入草稿尽量保留 ASR 原文，只做词库、空白、重复转写和非语音垃圾过滤 |
 | **远程 LLM** | 支持 OpenAI、Claude（Anthropic 格式）、Gemini、OpenRouter、硅基流动、豆包、百炼、MiniMax（国内/海外） |
 | **全局快捷键** | 可配置按键（Fn/Ctrl/Shift/Option），支持长按、双击、单击三种触发模式 |
 | **屏幕上下文 OCR** | 通过 ScreenCaptureKit + Vision 截取屏幕文字，辅助 LLM 纠正同音字 |

Sources/App/VoicePipeline+EditCommandResolution.swift

@@ -0,0 +1,61 @@
+import AppKit
+import Foundation
+
+@MainActor
+extension VoicePipeline {
+ func resolvedSpokenEditCommand(
+ raw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async -> SpokenEditCommand? {
+ guard VoicePipelinePolicy.shouldResolveEditCommandWithLLMFirst(outputMode: settings.outputMode) else {
+ return nil
+ }
+
+ let resolution = await resolveSpokenEditCommandWithLLM(
+ raw: raw,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return VoicePipelinePolicy.editCommand(from: resolution)
+ }
+
+ func spokenEditCommandResolutionContext(
+ targetApp: NSRunningApplication?
+ ) async -> SpokenEditCommandResolutionContext {
+ let lastInsertedText = appState.lastInsertedText.trimmingCharacters(in: .whitespacesAndNewlines)
+ let lastInsertion = lastInsertedText.isEmpty
+ ? SpokenEditCommandTargetAvailability.unavailable
+ : .available
+ let selectedText = await textInserter.selectedText(targetApp: targetApp)
+ let selectionAvailability: SpokenEditCommandTargetAvailability
+ if let selectedText {
+ selectionAvailability = selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
+ ? .unavailable
+ : .available
+ } else {
+ selectionAvailability = .unknown
+ }
+
+ return SpokenEditCommandResolutionContext(
+ lastInsertion: lastInsertion,
+ selectedText: selectionAvailability,
+ lastInsertionPreview: SpokenEditCommandResolutionContext.preview(lastInsertedText),
+ selectedTextPreview: SpokenEditCommandResolutionContext.preview(selectedText)
+ )
+ }
+
+ private func resolveSpokenEditCommandWithLLM(
+ raw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async -> SpokenEditCommandLLMResolution? {
+ var options = TextProcessingOptions(settings: settings)
+ options.llmModel = settings.llmModel
+ return await textProcessor.resolveSpokenEditCommandResolution(
+ text: raw,
+ options: options,
+ context: await spokenEditCommandResolutionContext(targetApp: targetApp)
+ )
+ }
+}

Sources/App/VoicePipeline+EditCommands.swift

@@ -0,0 +1,282 @@
+import AppKit
+import Foundation
+
+@MainActor
+extension VoicePipeline {
+ func handleSpokenEditCommandIfNeeded(
+ raw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async -> Bool {
+ guard let command = await resolvedSpokenEditCommand(
+ raw: raw,
+ settings: settings,
+ targetApp: targetApp
+ ) else {
+ return false
+ }
+
+ switch command {
+ case .replaceLast(let replacementRaw):
+ await replaceLastInsertion(
+ raw: raw,
+ replacementRaw: replacementRaw,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .replaceSelection(let replacementRaw):
+ await replaceSelectedText(
+ raw: raw,
+ replacementRaw: replacementRaw,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .rewriteLast(let intent):
+ await rewriteLastInsertion(
+ raw: raw,
+ intent: intent,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .rewriteSelection(let intent):
+ await rewriteSelectedText(
+ raw: raw,
+ intent: intent,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .deleteSelection:
+ await deleteSelectedText(targetApp: targetApp)
+ return true
+ case .undoLastInsertion:
+ await undoLastInsertion(targetApp: targetApp)
+ return true
+ }
+ }
+
+ private func replaceLastInsertion(
+ raw: String,
+ replacementRaw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async {
+ cancelScreenContextCapture()
+
+ guard !appState.lastInsertedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showErrorHint(L("pipeline.no_previous_insert_to_replace"))
+ return
+ }
+
+ let context = replacementInputContext(settings: settings, targetApp: targetApp)
+ let replacementText = finalizedReplacementText(replacementRaw, settings: settings)
+ guard !replacementText.isEmpty else {
+ showNoSpeechDetected(reason: "spoken edit command has empty replacement text")
+ return
+ }
+
+ appState.processedText = replacementText
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ Log.sensitive("[VoicePipeline] voice edit replace \(replacementText.count) chars")
+ let result = await textInserter.replaceRecentInsertion(text: replacementText, targetApp: targetApp)
+
+ InputHistory.shared.addRecord(
+ rawText: raw,
+ processedText: replacementText,
+ wasProcessed: true,
+ context: context
+ )
+
+ appState.lastInsertedText = replacementText
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit replacement probably failed: \(reason)")
+ TextInserter.copyToClipboard(replacementText)
+ showInsertionFailedAlert(text: replacementText, reason: reason)
+ }
+ }
+
+ private func replaceSelectedText(
+ raw: String,
+ replacementRaw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async {
+ cancelScreenContextCapture()
+
+ let context = replacementInputContext(settings: settings, targetApp: targetApp)
+ let replacementText = finalizedReplacementText(replacementRaw, settings: settings)
+ guard !replacementText.isEmpty else {
+ showNoSpeechDetected(reason: "spoken edit command has empty selection replacement text")
+ return
+ }
+
+ appState.processedText = replacementText
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ Log.sensitive("[VoicePipeline] voice edit replace selection \(replacementText.count) chars")
+ let result = await textInserter.replaceSelectedText(text: replacementText, targetApp: targetApp)
+
+ InputHistory.shared.addRecord(
+ rawText: raw,
+ processedText: replacementText,
+ wasProcessed: true,
+ context: context
+ )
+
+ appState.lastInsertedText = replacementText
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit selection replacement probably failed: \(reason)")
+ TextInserter.copyToClipboard(replacementText)
+ showInsertionFailedAlert(text: replacementText, reason: reason)
+ }
+ }
+
+ private func replacementInputContext(
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) -> InputContext {
+ InputContext.capture(
+ targetApp: targetApp,
+ screenContext: "",
+ outputMode: .command,
+ inputLanguage: settings.inputLanguage,
+ source: .menuBar
+ )
+ }
+
+ private func finalizedReplacementText(
+ _ text: String,
+ settings: AppSettings
+ ) -> String {
+ textProcessor.cleanCommandGeneratedOutput(
+ text,
+ inputLanguage: settings.inputLanguage
+ )
+ }
+
+ private func deleteSelectedText(targetApp: NSRunningApplication?) async {
+ cancelScreenContextCapture()
+
+ appState.processedText = ""
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ Log.info("[VoicePipeline] voice edit delete selection")
+ let result = await textInserter.deleteSelectedText(targetApp: targetApp)
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit delete selection probably failed: \(reason)")
+ showErrorHint(reason)
+ return
+ }
+
+ appState.lastInsertedText = ""
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+ }
+
+ private func undoLastInsertion(targetApp: NSRunningApplication?) async {
+ cancelScreenContextCapture()
+
+ guard !appState.lastInsertedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showErrorHint(L("pipeline.no_previous_insert_to_replace"))
+ return
+ }
+
+ appState.processedText = ""
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.undoing")
+
+ Log.info("[VoicePipeline] voice edit undo last insertion")
+ let result = await textInserter.undoRecentInsertion(targetApp: targetApp)
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit undo probably failed: \(reason)")
+ showErrorHint(reason)
+ return
+ }
+
+ appState.lastInsertedText = ""
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+ }
+
+ private func rewriteSelectedText(
+ raw: String,
+ intent: SelectionRewriteIntent,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async {
+ cancelScreenContextCapture()
+
+ guard let selectedText = await textInserter.selectedText(targetApp: targetApp),
+ !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showErrorHint(L("pipeline.no_selected_text_to_replace"))
+ return
+ }
+
+ let context = InputContext.capture(
+ targetApp: targetApp,
+ screenContext: selectedText,
+ outputMode: .command,
+ inputLanguage: settings.inputLanguage,
+ source: .menuBar
+ )
+ var options = TextProcessingOptions(settings: settings)
+ options.llmModel = settings.llmModel
+ let memoryContext = VoicePipelinePolicy.memoryContext(
+ for: .command,
+ settings: settings,
+ currentContext: context
+ )
+
+ appState.phase = .processing
+ appState.statusMessage = L("pipeline.formatting")
+ let rewrittenText = await textProcessor.processSelectionEdit(
+ selectedText: selectedText,
+ intent: intent,
+ options: options,
+ spokenCommand: raw,
+ memoryContext: memoryContext,
+ inputContext: context
+ )
+ guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showNoSpeechDetected(reason: "selection rewrite returned empty text")
+ return
+ }
+
+ appState.processedText = rewrittenText
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp)
+ InputHistory.shared.addRecord(rawText: raw, processedText: rewrittenText, wasProcessed: true, context: context)
+
+ appState.lastInsertedText = rewrittenText
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)")
+ TextInserter.copyToClipboard(rewrittenText)
+ showInsertionFailedAlert(text: rewrittenText, reason: reason)
+ }
+ }
+}

README.md

@@ -39,7 +39,10 @@ Three output modes are available:
 | Feature | Description |
 |---|---|
 | **Multiple Speech Engines** | Apple Speech, WhisperKit, Doubao ASR, Qwen3-ASR, or MiMo-V2.5-ASR |
-| **Smart Text Processing** | Local MLX Qwen2.5/Qwen3 or remote LLM — contextual cleanup, self-correction handling, structured list formatting |
+| **Smart Text Processing** | Local MLX Qwen2.5/Qwen3 or remote LLM infers spoken intent — contextual cleanup, "scratch that" restarts, self-correction handling, spoken punctuation, technical terms, numbers/ranges/units, and structured formatting |
+| **LLM-Owned Spoken Formatting** | Spoken casing, no-space dictation, identifiers, file paths, shortcuts, emoji, Markdown tasks, dates/times, quantities, units, formulas, fractions, and digit sequences are handled by the Smart Format / Voice Command prompts instead of local hardcoded rewrite rules |
+| **Voice Edit Commands** | In Voice Command mode, an LLM classifies safe structured actions for replacing, undoing, proofreading, titling, summarizing, drafting replies, making meeting notes, extracting key points/decisions/questions/risks/deadlines/owners/action items, rewriting tone, expanding, making tables/lists, or deleting the previous OpenType insertion or selected text |
+| **Verbatim & Preview Boundary** | Verbatim mode, streaming HUD, integration partials, and instant-insert drafts keep ASR text close to raw output with only dictionary, whitespace, duplicate, and non-speech-artifact cleanup |
 | **Remote LLM Support** | OpenAI, Claude (Anthropic format), Gemini, OpenRouter, SiliconFlow, Doubao, Bailian, MiniMax (CN & Global) |
 | **Global Hotkey** | Configurable key (Fn/Ctrl/Shift/Option) with long-press, double-tap, or single-tap activation |
 | **Screen Context OCR** | Captures on-screen text via ScreenCaptureKit + Vision to help the LLM correct homophones |

README_zh.md

@@ -39,7 +39,10 @@
 | 功能 | 说明 |
 |---|---|
 | **多语音引擎** | Apple 语音识别、WhisperKit、豆包语音识别、Qwen3-ASR 或 MiMo-V2.5-ASR |
-| **智能文字处理** | 本地 MLX Qwen2.5/Qwen3 或远程 LLM — 上下文感知的语气词清理、自动纠正、列表格式化 |
+| **智能文字处理** | 本地 MLX Qwen2.5/Qwen3 或远程 LLM 理解口述意图 — 上下文感知的语气词清理、“算了/删掉刚才”重说处理、自动纠正、口述标点、技术词、数字/范围/单位和列表格式化 |
+| **LLM 负责口述格式** | 大小写、无空格、标识符、文件路径、快捷键、表情、Markdown 任务、日期时间、数量、单位、公式、分数和数字串都由智能整理/语音指令提示词交给 LLM 判断，不在本地写死替换规则 |
+| **语音编辑口令** | 在语音指令模式下，由 LLM 分类安全结构化动作，支持上一段/选区替换、撤销、校对、跨语言回复起草、接受/拒绝/追问回复、会议纪要、关键要点/结论/问题/风险/截止时间/负责人/行动项提取、标题化、摘要、语气改写、扩写、表格化、列表化、删除与改写口令 |
+| **直出与预览边界** | 原文直出、流式 HUD、集成 partial 和快速插入草稿尽量保留 ASR 原文，只做词库、空白、重复转写和非语音垃圾过滤 |
 | **远程 LLM** | 支持 OpenAI、Claude（Anthropic 格式）、Gemini、OpenRouter、硅基流动、豆包、百炼、MiniMax（国内/海外） |
 | **全局快捷键** | 可配置按键（Fn/Ctrl/Shift/Option），支持长按、双击、单击三种触发模式 |
 | **屏幕上下文 OCR** | 通过 ScreenCaptureKit + Vision 截取屏幕文字，辅助 LLM 纠正同音字 |

Sources/App/VoicePipeline+EditCommandResolution.swift

@@ -0,0 +1,61 @@
+import AppKit
+import Foundation
+
+@MainActor
+extension VoicePipeline {
+ func resolvedSpokenEditCommand(
+ raw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async -> SpokenEditCommand? {
+ guard VoicePipelinePolicy.shouldResolveEditCommandWithLLMFirst(outputMode: settings.outputMode) else {
+ return nil
+ }
+
+ let resolution = await resolveSpokenEditCommandWithLLM(
+ raw: raw,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return VoicePipelinePolicy.editCommand(from: resolution)
+ }
+
+ func spokenEditCommandResolutionContext(
+ targetApp: NSRunningApplication?
+ ) async -> SpokenEditCommandResolutionContext {
+ let lastInsertedText = appState.lastInsertedText.trimmingCharacters(in: .whitespacesAndNewlines)
+ let lastInsertion = lastInsertedText.isEmpty
+ ? SpokenEditCommandTargetAvailability.unavailable
+ : .available
+ let selectedText = await textInserter.selectedText(targetApp: targetApp)
+ let selectionAvailability: SpokenEditCommandTargetAvailability
+ if let selectedText {
+ selectionAvailability = selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
+ ? .unavailable
+ : .available
+ } else {
+ selectionAvailability = .unknown
+ }
+
+ return SpokenEditCommandResolutionContext(
+ lastInsertion: lastInsertion,
+ selectedText: selectionAvailability,
+ lastInsertionPreview: SpokenEditCommandResolutionContext.preview(lastInsertedText),
+ selectedTextPreview: SpokenEditCommandResolutionContext.preview(selectedText)
+ )
+ }
+
+ private func resolveSpokenEditCommandWithLLM(
+ raw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async -> SpokenEditCommandLLMResolution? {
+ var options = TextProcessingOptions(settings: settings)
+ options.llmModel = settings.llmModel
+ return await textProcessor.resolveSpokenEditCommandResolution(
+ text: raw,
+ options: options,
+ context: await spokenEditCommandResolutionContext(targetApp: targetApp)
+ )
+ }
+}

Sources/App/VoicePipeline+EditCommands.swift

@@ -0,0 +1,282 @@
+import AppKit
+import Foundation
+
+@MainActor
+extension VoicePipeline {
+ func handleSpokenEditCommandIfNeeded(
+ raw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async -> Bool {
+ guard let command = await resolvedSpokenEditCommand(
+ raw: raw,
+ settings: settings,
+ targetApp: targetApp
+ ) else {
+ return false
+ }
+
+ switch command {
+ case .replaceLast(let replacementRaw):
+ await replaceLastInsertion(
+ raw: raw,
+ replacementRaw: replacementRaw,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .replaceSelection(let replacementRaw):
+ await replaceSelectedText(
+ raw: raw,
+ replacementRaw: replacementRaw,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .rewriteLast(let intent):
+ await rewriteLastInsertion(
+ raw: raw,
+ intent: intent,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .rewriteSelection(let intent):
+ await rewriteSelectedText(
+ raw: raw,
+ intent: intent,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .deleteSelection:
+ await deleteSelectedText(targetApp: targetApp)
+ return true
+ case .undoLastInsertion:
+ await undoLastInsertion(targetApp: targetApp)
+ return true
+ }
+ }
+
+ private func replaceLastInsertion(
+ raw: String,
+ replacementRaw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async {
+ cancelScreenContextCapture()
+
+ guard !appState.lastInsertedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showErrorHint(L("pipeline.no_previous_insert_to_replace"))
+ return
+ }
+
+ let context = replacementInputContext(settings: settings, targetApp: targetApp)
+ let replacementText = finalizedReplacementText(replacementRaw, settings: settings)
+ guard !replacementText.isEmpty else {
+ showNoSpeechDetected(reason: "spoken edit command has empty replacement text")
+ return
+ }
+
+ appState.processedText = replacementText
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ Log.sensitive("[VoicePipeline] voice edit replace \(replacementText.count) chars")
+ let result = await textInserter.replaceRecentInsertion(text: replacementText, targetApp: targetApp)
+
+ InputHistory.shared.addRecord(
+ rawText: raw,
+ processedText: replacementText,
+ wasProcessed: true,
+ context: context
+ )
+
+ appState.lastInsertedText = replacementText
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit replacement probably failed: \(reason)")
+ TextInserter.copyToClipboard(replacementText)
+ showInsertionFailedAlert(text: replacementText, reason: reason)
+ }
+ }
+
+ private func replaceSelectedText(
+ raw: String,
+ replacementRaw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async {
+ cancelScreenContextCapture()
+
+ let context = replacementInputContext(settings: settings, targetApp: targetApp)
+ let replacementText = finalizedReplacementText(replacementRaw, settings: settings)
+ guard !replacementText.isEmpty else {
+ showNoSpeechDetected(reason: "spoken edit command has empty selection replacement text")
+ return
+ }
+
+ appState.processedText = replacementText
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ Log.sensitive("[VoicePipeline] voice edit replace selection \(replacementText.count) chars")
+ let result = await textInserter.replaceSelectedText(text: replacementText, targetApp: targetApp)
+
+ InputHistory.shared.addRecord(
+ rawText: raw,
+ processedText: replacementText,
+ wasProcessed: true,
+ context: context
+ )
+
+ appState.lastInsertedText = replacementText
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit selection replacement probably failed: \(reason)")
+ TextInserter.copyToClipboard(replacementText)
+ showInsertionFailedAlert(text: replacementText, reason: reason)
+ }
+ }
+
+ private func replacementInputContext(
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) -> InputContext {
+ InputContext.capture(
+ targetApp: targetApp,
+ screenContext: "",
+ outputMode: .command,
+ inputLanguage: settings.inputLanguage,
+ source: .menuBar
+ )
+ }
+
+ private func finalizedReplacementText(
+ _ text: String,
+ settings: AppSettings
+ ) -> String {
+ textProcessor.cleanCommandGeneratedOutput(
+ text,
+ inputLanguage: settings.inputLanguage
+ )
+ }
+
+ private func deleteSelectedText(targetApp: NSRunningApplication?) async {
+ cancelScreenContextCapture()
+
+ appState.processedText = ""
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ Log.info("[VoicePipeline] voice edit delete selection")
+ let result = await textInserter.deleteSelectedText(targetApp: targetApp)
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit delete selection probably failed: \(reason)")
+ showErrorHint(reason)
+ return
+ }
+
+ appState.lastInsertedText = ""
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+ }
+
+ private func undoLastInsertion(targetApp: NSRunningApplication?) async {
+ cancelScreenContextCapture()
+
+ guard !appState.lastInsertedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showErrorHint(L("pipeline.no_previous_insert_to_replace"))
+ return
+ }
+
+ appState.processedText = ""
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.undoing")
+
+ Log.info("[VoicePipeline] voice edit undo last insertion")
+ let result = await textInserter.undoRecentInsertion(targetApp: targetApp)
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit undo probably failed: \(reason)")
+ showErrorHint(reason)
+ return
+ }
+
+ appState.lastInsertedText = ""
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+ }
+
+ private func rewriteSelectedText(
+ raw: String,
+ intent: SelectionRewriteIntent,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async {
+ cancelScreenContextCapture()
+
+ guard let selectedText = await textInserter.selectedText(targetApp: targetApp),
+ !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showErrorHint(L("pipeline.no_selected_text_to_replace"))
+ return
+ }
+
+ let context = InputContext.capture(
+ targetApp: targetApp,
+ screenContext: selectedText,
+ outputMode: .command,
+ inputLanguage: settings.inputLanguage,
+ source: .menuBar
+ )
+ var options = TextProcessingOptions(settings: settings)
+ options.llmModel = settings.llmModel
+ let memoryContext = VoicePipelinePolicy.memoryContext(
+ for: .command,
+ settings: settings,
+ currentContext: context
+ )
+
+ appState.phase = .processing
+ appState.statusMessage = L("pipeline.formatting")
+ let rewrittenText = await textProcessor.processSelectionEdit(
+ selectedText: selectedText,
+ intent: intent,
+ options: options,
+ spokenCommand: raw,
+ memoryContext: memoryContext,
+ inputContext: context
+ )
+ guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showNoSpeechDetected(reason: "selection rewrite returned empty text")
+ return
+ }
+
+ appState.processedText = rewrittenText
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp)
+ InputHistory.shared.addRecord(rawText: raw, processedText: rewrittenText, wasProcessed: true, context: context)
+
+ appState.lastInsertedText = rewrittenText
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)")
+ TextInserter.copyToClipboard(rewrittenText)
+ showInsertionFailedAlert(text: rewrittenText, reason: reason)
+ }
+ }
+}

README.md

@@ -39,7 +39,10 @@ Three output modes are available:
 | Feature | Description |
 |---|---|
 | **Multiple Speech Engines** | Apple Speech, WhisperKit, Doubao ASR, Qwen3-ASR, or MiMo-V2.5-ASR |
-| **Smart Text Processing** | Local MLX Qwen2.5/Qwen3 or remote LLM — contextual cleanup, self-correction handling, structured list formatting |
+| **Smart Text Processing** | Local MLX Qwen2.5/Qwen3 or remote LLM infers spoken intent — contextual cleanup, "scratch that" restarts, self-correction handling, spoken punctuation, technical terms, numbers/ranges/units, and structured formatting |
+| **LLM-Owned Spoken Formatting** | Spoken casing, no-space dictation, identifiers, file paths, shortcuts, emoji, Markdown tasks, dates/times, quantities, units, formulas, fractions, and digit sequences are handled by the Smart Format / Voice Command prompts instead of local hardcoded rewrite rules |
+| **Voice Edit Commands** | In Voice Command mode, an LLM classifies safe structured actions for replacing, undoing, proofreading, titling, summarizing, drafting replies, making meeting notes, extracting key points/decisions/questions/risks/deadlines/owners/action items, rewriting tone, expanding, making tables/lists, or deleting the previous OpenType insertion or selected text |
+| **Verbatim & Preview Boundary** | Verbatim mode, streaming HUD, integration partials, and instant-insert drafts keep ASR text close to raw output with only dictionary, whitespace, duplicate, and non-speech-artifact cleanup |
 | **Remote LLM Support** | OpenAI, Claude (Anthropic format), Gemini, OpenRouter, SiliconFlow, Doubao, Bailian, MiniMax (CN & Global) |
 | **Global Hotkey** | Configurable key (Fn/Ctrl/Shift/Option) with long-press, double-tap, or single-tap activation |
 | **Screen Context OCR** | Captures on-screen text via ScreenCaptureKit + Vision to help the LLM correct homophones |

README_zh.md

@@ -39,7 +39,10 @@
 | 功能 | 说明 |
 |---|---|
 | **多语音引擎** | Apple 语音识别、WhisperKit、豆包语音识别、Qwen3-ASR 或 MiMo-V2.5-ASR |
-| **智能文字处理** | 本地 MLX Qwen2.5/Qwen3 或远程 LLM — 上下文感知的语气词清理、自动纠正、列表格式化 |
+| **智能文字处理** | 本地 MLX Qwen2.5/Qwen3 或远程 LLM 理解口述意图 — 上下文感知的语气词清理、“算了/删掉刚才”重说处理、自动纠正、口述标点、技术词、数字/范围/单位和列表格式化 |
+| **LLM 负责口述格式** | 大小写、无空格、标识符、文件路径、快捷键、表情、Markdown 任务、日期时间、数量、单位、公式、分数和数字串都由智能整理/语音指令提示词交给 LLM 判断，不在本地写死替换规则 |
+| **语音编辑口令** | 在语音指令模式下，由 LLM 分类安全结构化动作，支持上一段/选区替换、撤销、校对、跨语言回复起草、接受/拒绝/追问回复、会议纪要、关键要点/结论/问题/风险/截止时间/负责人/行动项提取、标题化、摘要、语气改写、扩写、表格化、列表化、删除与改写口令 |
+| **直出与预览边界** | 原文直出、流式 HUD、集成 partial 和快速插入草稿尽量保留 ASR 原文，只做词库、空白、重复转写和非语音垃圾过滤 |
 | **远程 LLM** | 支持 OpenAI、Claude（Anthropic 格式）、Gemini、OpenRouter、硅基流动、豆包、百炼、MiniMax（国内/海外） |
 | **全局快捷键** | 可配置按键（Fn/Ctrl/Shift/Option），支持长按、双击、单击三种触发模式 |
 | **屏幕上下文 OCR** | 通过 ScreenCaptureKit + Vision 截取屏幕文字，辅助 LLM 纠正同音字 |

Sources/App/VoicePipeline+EditCommandResolution.swift

@@ -0,0 +1,61 @@
+import AppKit
+import Foundation
+
+@MainActor
+extension VoicePipeline {
+ func resolvedSpokenEditCommand(
+ raw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async -> SpokenEditCommand? {
+ guard VoicePipelinePolicy.shouldResolveEditCommandWithLLMFirst(outputMode: settings.outputMode) else {
+ return nil
+ }
+
+ let resolution = await resolveSpokenEditCommandWithLLM(
+ raw: raw,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return VoicePipelinePolicy.editCommand(from: resolution)
+ }
+
+ func spokenEditCommandResolutionContext(
+ targetApp: NSRunningApplication?
+ ) async -> SpokenEditCommandResolutionContext {
+ let lastInsertedText = appState.lastInsertedText.trimmingCharacters(in: .whitespacesAndNewlines)
+ let lastInsertion = lastInsertedText.isEmpty
+ ? SpokenEditCommandTargetAvailability.unavailable
+ : .available
+ let selectedText = await textInserter.selectedText(targetApp: targetApp)
+ let selectionAvailability: SpokenEditCommandTargetAvailability
+ if let selectedText {
+ selectionAvailability = selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
+ ? .unavailable
+ : .available
+ } else {
+ selectionAvailability = .unknown
+ }
+
+ return SpokenEditCommandResolutionContext(
+ lastInsertion: lastInsertion,
+ selectedText: selectionAvailability,
+ lastInsertionPreview: SpokenEditCommandResolutionContext.preview(lastInsertedText),
+ selectedTextPreview: SpokenEditCommandResolutionContext.preview(selectedText)
+ )
+ }
+
+ private func resolveSpokenEditCommandWithLLM(
+ raw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async -> SpokenEditCommandLLMResolution? {
+ var options = TextProcessingOptions(settings: settings)
+ options.llmModel = settings.llmModel
+ return await textProcessor.resolveSpokenEditCommandResolution(
+ text: raw,
+ options: options,
+ context: await spokenEditCommandResolutionContext(targetApp: targetApp)
+ )
+ }
+}

Sources/App/VoicePipeline+EditCommands.swift

@@ -0,0 +1,282 @@
+import AppKit
+import Foundation
+
+@MainActor
+extension VoicePipeline {
+ func handleSpokenEditCommandIfNeeded(
+ raw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async -> Bool {
+ guard let command = await resolvedSpokenEditCommand(
+ raw: raw,
+ settings: settings,
+ targetApp: targetApp
+ ) else {
+ return false
+ }
+
+ switch command {
+ case .replaceLast(let replacementRaw):
+ await replaceLastInsertion(
+ raw: raw,
+ replacementRaw: replacementRaw,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .replaceSelection(let replacementRaw):
+ await replaceSelectedText(
+ raw: raw,
+ replacementRaw: replacementRaw,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .rewriteLast(let intent):
+ await rewriteLastInsertion(
+ raw: raw,
+ intent: intent,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .rewriteSelection(let intent):
+ await rewriteSelectedText(
+ raw: raw,
+ intent: intent,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .deleteSelection:
+ await deleteSelectedText(targetApp: targetApp)
+ return true
+ case .undoLastInsertion:
+ await undoLastInsertion(targetApp: targetApp)
+ return true
+ }
+ }
+
+ private func replaceLastInsertion(
+ raw: String,
+ replacementRaw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async {
+ cancelScreenContextCapture()
+
+ guard !appState.lastInsertedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showErrorHint(L("pipeline.no_previous_insert_to_replace"))
+ return
+ }
+
+ let context = replacementInputContext(settings: settings, targetApp: targetApp)
+ let replacementText = finalizedReplacementText(replacementRaw, settings: settings)
+ guard !replacementText.isEmpty else {
+ showNoSpeechDetected(reason: "spoken edit command has empty replacement text")
+ return
+ }
+
+ appState.processedText = replacementText
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ Log.sensitive("[VoicePipeline] voice edit replace \(replacementText.count) chars")
+ let result = await textInserter.replaceRecentInsertion(text: replacementText, targetApp: targetApp)
+
+ InputHistory.shared.addRecord(
+ rawText: raw,
+ processedText: replacementText,
+ wasProcessed: true,
+ context: context
+ )
+
+ appState.lastInsertedText = replacementText
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit replacement probably failed: \(reason)")
+ TextInserter.copyToClipboard(replacementText)
+ showInsertionFailedAlert(text: replacementText, reason: reason)
+ }
+ }
+
+ private func replaceSelectedText(
+ raw: String,
+ replacementRaw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async {
+ cancelScreenContextCapture()
+
+ let context = replacementInputContext(settings: settings, targetApp: targetApp)
+ let replacementText = finalizedReplacementText(replacementRaw, settings: settings)
+ guard !replacementText.isEmpty else {
+ showNoSpeechDetected(reason: "spoken edit command has empty selection replacement text")
+ return
+ }
+
+ appState.processedText = replacementText
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ Log.sensitive("[VoicePipeline] voice edit replace selection \(replacementText.count) chars")
+ let result = await textInserter.replaceSelectedText(text: replacementText, targetApp: targetApp)
+
+ InputHistory.shared.addRecord(
+ rawText: raw,
+ processedText: replacementText,
+ wasProcessed: true,
+ context: context
+ )
+
+ appState.lastInsertedText = replacementText
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit selection replacement probably failed: \(reason)")
+ TextInserter.copyToClipboard(replacementText)
+ showInsertionFailedAlert(text: replacementText, reason: reason)
+ }
+ }
+
+ private func replacementInputContext(
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) -> InputContext {
+ InputContext.capture(
+ targetApp: targetApp,
+ screenContext: "",
+ outputMode: .command,
+ inputLanguage: settings.inputLanguage,
+ source: .menuBar
+ )
+ }
+
+ private func finalizedReplacementText(
+ _ text: String,
+ settings: AppSettings
+ ) -> String {
+ textProcessor.cleanCommandGeneratedOutput(
+ text,
+ inputLanguage: settings.inputLanguage
+ )
+ }
+
+ private func deleteSelectedText(targetApp: NSRunningApplication?) async {
+ cancelScreenContextCapture()
+
+ appState.processedText = ""
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ Log.info("[VoicePipeline] voice edit delete selection")
+ let result = await textInserter.deleteSelectedText(targetApp: targetApp)
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit delete selection probably failed: \(reason)")
+ showErrorHint(reason)
+ return
+ }
+
+ appState.lastInsertedText = ""
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+ }
+
+ private func undoLastInsertion(targetApp: NSRunningApplication?) async {
+ cancelScreenContextCapture()
+
+ guard !appState.lastInsertedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showErrorHint(L("pipeline.no_previous_insert_to_replace"))
+ return
+ }
+
+ appState.processedText = ""
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.undoing")
+
+ Log.info("[VoicePipeline] voice edit undo last insertion")
+ let result = await textInserter.undoRecentInsertion(targetApp: targetApp)
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit undo probably failed: \(reason)")
+ showErrorHint(reason)
+ return
+ }
+
+ appState.lastInsertedText = ""
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+ }
+
+ private func rewriteSelectedText(
+ raw: String,
+ intent: SelectionRewriteIntent,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async {
+ cancelScreenContextCapture()
+
+ guard let selectedText = await textInserter.selectedText(targetApp: targetApp),
+ !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showErrorHint(L("pipeline.no_selected_text_to_replace"))
+ return
+ }
+
+ let context = InputContext.capture(
+ targetApp: targetApp,
+ screenContext: selectedText,
+ outputMode: .command,
+ inputLanguage: settings.inputLanguage,
+ source: .menuBar
+ )
+ var options = TextProcessingOptions(settings: settings)
+ options.llmModel = settings.llmModel
+ let memoryContext = VoicePipelinePolicy.memoryContext(
+ for: .command,
+ settings: settings,
+ currentContext: context
+ )
+
+ appState.phase = .processing
+ appState.statusMessage = L("pipeline.formatting")
+ let rewrittenText = await textProcessor.processSelectionEdit(
+ selectedText: selectedText,
+ intent: intent,
+ options: options,
+ spokenCommand: raw,
+ memoryContext: memoryContext,
+ inputContext: context
+ )
+ guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showNoSpeechDetected(reason: "selection rewrite returned empty text")
+ return
+ }
+
+ appState.processedText = rewrittenText
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp)
+ InputHistory.shared.addRecord(rawText: raw, processedText: rewrittenText, wasProcessed: true, context: context)
+
+ appState.lastInsertedText = rewrittenText
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)")
+ TextInserter.copyToClipboard(rewrittenText)
+ showInsertionFailedAlert(text: rewrittenText, reason: reason)
+ }
+ }
+}

README.md

@@ -39,7 +39,10 @@ Three output modes are available:
 | Feature | Description |
 |---|---|
 | **Multiple Speech Engines** | Apple Speech, WhisperKit, Doubao ASR, Qwen3-ASR, or MiMo-V2.5-ASR |
-| **Smart Text Processing** | Local MLX Qwen2.5/Qwen3 or remote LLM — contextual cleanup, self-correction handling, structured list formatting |
+| **Smart Text Processing** | Local MLX Qwen2.5/Qwen3 or remote LLM infers spoken intent — contextual cleanup, "scratch that" restarts, self-correction handling, spoken punctuation, technical terms, numbers/ranges/units, and structured formatting |
+| **LLM-Owned Spoken Formatting** | Spoken casing, no-space dictation, identifiers, file paths, shortcuts, emoji, Markdown tasks, dates/times, quantities, units, formulas, fractions, and digit sequences are handled by the Smart Format / Voice Command prompts instead of local hardcoded rewrite rules |
+| **Voice Edit Commands** | In Voice Command mode, an LLM classifies safe structured actions for replacing, undoing, proofreading, titling, summarizing, drafting replies, making meeting notes, extracting key points/decisions/questions/risks/deadlines/owners/action items, rewriting tone, expanding, making tables/lists, or deleting the previous OpenType insertion or selected text |
+| **Verbatim & Preview Boundary** | Verbatim mode, streaming HUD, integration partials, and instant-insert drafts keep ASR text close to raw output with only dictionary, whitespace, duplicate, and non-speech-artifact cleanup |
 | **Remote LLM Support** | OpenAI, Claude (Anthropic format), Gemini, OpenRouter, SiliconFlow, Doubao, Bailian, MiniMax (CN & Global) |
 | **Global Hotkey** | Configurable key (Fn/Ctrl/Shift/Option) with long-press, double-tap, or single-tap activation |
 | **Screen Context OCR** | Captures on-screen text via ScreenCaptureKit + Vision to help the LLM correct homophones |

README_zh.md

@@ -39,7 +39,10 @@
 | 功能 | 说明 |
 |---|---|
 | **多语音引擎** | Apple 语音识别、WhisperKit、豆包语音识别、Qwen3-ASR 或 MiMo-V2.5-ASR |
-| **智能文字处理** | 本地 MLX Qwen2.5/Qwen3 或远程 LLM — 上下文感知的语气词清理、自动纠正、列表格式化 |
+| **智能文字处理** | 本地 MLX Qwen2.5/Qwen3 或远程 LLM 理解口述意图 — 上下文感知的语气词清理、“算了/删掉刚才”重说处理、自动纠正、口述标点、技术词、数字/范围/单位和列表格式化 |
+| **LLM 负责口述格式** | 大小写、无空格、标识符、文件路径、快捷键、表情、Markdown 任务、日期时间、数量、单位、公式、分数和数字串都由智能整理/语音指令提示词交给 LLM 判断，不在本地写死替换规则 |
+| **语音编辑口令** | 在语音指令模式下，由 LLM 分类安全结构化动作，支持上一段/选区替换、撤销、校对、跨语言回复起草、接受/拒绝/追问回复、会议纪要、关键要点/结论/问题/风险/截止时间/负责人/行动项提取、标题化、摘要、语气改写、扩写、表格化、列表化、删除与改写口令 |
+| **直出与预览边界** | 原文直出、流式 HUD、集成 partial 和快速插入草稿尽量保留 ASR 原文，只做词库、空白、重复转写和非语音垃圾过滤 |
 | **远程 LLM** | 支持 OpenAI、Claude（Anthropic 格式）、Gemini、OpenRouter、硅基流动、豆包、百炼、MiniMax（国内/海外） |
 | **全局快捷键** | 可配置按键（Fn/Ctrl/Shift/Option），支持长按、双击、单击三种触发模式 |
 | **屏幕上下文 OCR** | 通过 ScreenCaptureKit + Vision 截取屏幕文字，辅助 LLM 纠正同音字 |

Sources/App/VoicePipeline+EditCommandResolution.swift

@@ -0,0 +1,61 @@
+import AppKit
+import Foundation
+
+@MainActor
+extension VoicePipeline {
+ func resolvedSpokenEditCommand(
+ raw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async -> SpokenEditCommand? {
+ guard VoicePipelinePolicy.shouldResolveEditCommandWithLLMFirst(outputMode: settings.outputMode) else {
+ return nil
+ }
+
+ let resolution = await resolveSpokenEditCommandWithLLM(
+ raw: raw,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return VoicePipelinePolicy.editCommand(from: resolution)
+ }
+
+ func spokenEditCommandResolutionContext(
+ targetApp: NSRunningApplication?
+ ) async -> SpokenEditCommandResolutionContext {
+ let lastInsertedText = appState.lastInsertedText.trimmingCharacters(in: .whitespacesAndNewlines)
+ let lastInsertion = lastInsertedText.isEmpty
+ ? SpokenEditCommandTargetAvailability.unavailable
+ : .available
+ let selectedText = await textInserter.selectedText(targetApp: targetApp)
+ let selectionAvailability: SpokenEditCommandTargetAvailability
+ if let selectedText {
+ selectionAvailability = selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
+ ? .unavailable
+ : .available
+ } else {
+ selectionAvailability = .unknown
+ }
+
+ return SpokenEditCommandResolutionContext(
+ lastInsertion: lastInsertion,
+ selectedText: selectionAvailability,
+ lastInsertionPreview: SpokenEditCommandResolutionContext.preview(lastInsertedText),
+ selectedTextPreview: SpokenEditCommandResolutionContext.preview(selectedText)
+ )
+ }
+
+ private func resolveSpokenEditCommandWithLLM(
+ raw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async -> SpokenEditCommandLLMResolution? {
+ var options = TextProcessingOptions(settings: settings)
+ options.llmModel = settings.llmModel
+ return await textProcessor.resolveSpokenEditCommandResolution(
+ text: raw,
+ options: options,
+ context: await spokenEditCommandResolutionContext(targetApp: targetApp)
+ )
+ }
+}

Sources/App/VoicePipeline+EditCommands.swift

@@ -0,0 +1,282 @@
+import AppKit
+import Foundation
+
+@MainActor
+extension VoicePipeline {
+ func handleSpokenEditCommandIfNeeded(
+ raw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async -> Bool {
+ guard let command = await resolvedSpokenEditCommand(
+ raw: raw,
+ settings: settings,
+ targetApp: targetApp
+ ) else {
+ return false
+ }
+
+ switch command {
+ case .replaceLast(let replacementRaw):
+ await replaceLastInsertion(
+ raw: raw,
+ replacementRaw: replacementRaw,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .replaceSelection(let replacementRaw):
+ await replaceSelectedText(
+ raw: raw,
+ replacementRaw: replacementRaw,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .rewriteLast(let intent):
+ await rewriteLastInsertion(
+ raw: raw,
+ intent: intent,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .rewriteSelection(let intent):
+ await rewriteSelectedText(
+ raw: raw,
+ intent: intent,
+ settings: settings,
+ targetApp: targetApp
+ )
+ return true
+ case .deleteSelection:
+ await deleteSelectedText(targetApp: targetApp)
+ return true
+ case .undoLastInsertion:
+ await undoLastInsertion(targetApp: targetApp)
+ return true
+ }
+ }
+
+ private func replaceLastInsertion(
+ raw: String,
+ replacementRaw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async {
+ cancelScreenContextCapture()
+
+ guard !appState.lastInsertedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showErrorHint(L("pipeline.no_previous_insert_to_replace"))
+ return
+ }
+
+ let context = replacementInputContext(settings: settings, targetApp: targetApp)
+ let replacementText = finalizedReplacementText(replacementRaw, settings: settings)
+ guard !replacementText.isEmpty else {
+ showNoSpeechDetected(reason: "spoken edit command has empty replacement text")
+ return
+ }
+
+ appState.processedText = replacementText
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ Log.sensitive("[VoicePipeline] voice edit replace \(replacementText.count) chars")
+ let result = await textInserter.replaceRecentInsertion(text: replacementText, targetApp: targetApp)
+
+ InputHistory.shared.addRecord(
+ rawText: raw,
+ processedText: replacementText,
+ wasProcessed: true,
+ context: context
+ )
+
+ appState.lastInsertedText = replacementText
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit replacement probably failed: \(reason)")
+ TextInserter.copyToClipboard(replacementText)
+ showInsertionFailedAlert(text: replacementText, reason: reason)
+ }
+ }
+
+ private func replaceSelectedText(
+ raw: String,
+ replacementRaw: String,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async {
+ cancelScreenContextCapture()
+
+ let context = replacementInputContext(settings: settings, targetApp: targetApp)
+ let replacementText = finalizedReplacementText(replacementRaw, settings: settings)
+ guard !replacementText.isEmpty else {
+ showNoSpeechDetected(reason: "spoken edit command has empty selection replacement text")
+ return
+ }
+
+ appState.processedText = replacementText
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ Log.sensitive("[VoicePipeline] voice edit replace selection \(replacementText.count) chars")
+ let result = await textInserter.replaceSelectedText(text: replacementText, targetApp: targetApp)
+
+ InputHistory.shared.addRecord(
+ rawText: raw,
+ processedText: replacementText,
+ wasProcessed: true,
+ context: context
+ )
+
+ appState.lastInsertedText = replacementText
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit selection replacement probably failed: \(reason)")
+ TextInserter.copyToClipboard(replacementText)
+ showInsertionFailedAlert(text: replacementText, reason: reason)
+ }
+ }
+
+ private func replacementInputContext(
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) -> InputContext {
+ InputContext.capture(
+ targetApp: targetApp,
+ screenContext: "",
+ outputMode: .command,
+ inputLanguage: settings.inputLanguage,
+ source: .menuBar
+ )
+ }
+
+ private func finalizedReplacementText(
+ _ text: String,
+ settings: AppSettings
+ ) -> String {
+ textProcessor.cleanCommandGeneratedOutput(
+ text,
+ inputLanguage: settings.inputLanguage
+ )
+ }
+
+ private func deleteSelectedText(targetApp: NSRunningApplication?) async {
+ cancelScreenContextCapture()
+
+ appState.processedText = ""
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ Log.info("[VoicePipeline] voice edit delete selection")
+ let result = await textInserter.deleteSelectedText(targetApp: targetApp)
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit delete selection probably failed: \(reason)")
+ showErrorHint(reason)
+ return
+ }
+
+ appState.lastInsertedText = ""
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+ }
+
+ private func undoLastInsertion(targetApp: NSRunningApplication?) async {
+ cancelScreenContextCapture()
+
+ guard !appState.lastInsertedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showErrorHint(L("pipeline.no_previous_insert_to_replace"))
+ return
+ }
+
+ appState.processedText = ""
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.undoing")
+
+ Log.info("[VoicePipeline] voice edit undo last insertion")
+ let result = await textInserter.undoRecentInsertion(targetApp: targetApp)
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit undo probably failed: \(reason)")
+ showErrorHint(reason)
+ return
+ }
+
+ appState.lastInsertedText = ""
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+ }
+
+ private func rewriteSelectedText(
+ raw: String,
+ intent: SelectionRewriteIntent,
+ settings: AppSettings,
+ targetApp: NSRunningApplication?
+ ) async {
+ cancelScreenContextCapture()
+
+ guard let selectedText = await textInserter.selectedText(targetApp: targetApp),
+ !selectedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showErrorHint(L("pipeline.no_selected_text_to_replace"))
+ return
+ }
+
+ let context = InputContext.capture(
+ targetApp: targetApp,
+ screenContext: selectedText,
+ outputMode: .command,
+ inputLanguage: settings.inputLanguage,
+ source: .menuBar
+ )
+ var options = TextProcessingOptions(settings: settings)
+ options.llmModel = settings.llmModel
+ let memoryContext = VoicePipelinePolicy.memoryContext(
+ for: .command,
+ settings: settings,
+ currentContext: context
+ )
+
+ appState.phase = .processing
+ appState.statusMessage = L("pipeline.formatting")
+ let rewrittenText = await textProcessor.processSelectionEdit(
+ selectedText: selectedText,
+ intent: intent,
+ options: options,
+ spokenCommand: raw,
+ memoryContext: memoryContext,
+ inputContext: context
+ )
+ guard !rewrittenText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ showNoSpeechDetected(reason: "selection rewrite returned empty text")
+ return
+ }
+
+ appState.processedText = rewrittenText
+ appState.phase = .inserting
+ appState.statusMessage = L("pipeline.replacing")
+
+ let result = await textInserter.replaceSelectedText(text: rewrittenText, targetApp: targetApp)
+ InputHistory.shared.addRecord(rawText: raw, processedText: rewrittenText, wasProcessed: true, context: context)
+
+ appState.lastInsertedText = rewrittenText
+ appState.phase = .done
+ appState.statusMessage = L("status.done")
+ hideOverlayAfterDelay()
+
+ if case .probablyFailed(let reason) = result {
+ Log.info("[VoicePipeline] voice edit selection rewrite probably failed: \(reason)")
+ TextInserter.copyToClipboard(rewrittenText)
+ showInsertionFailedAlert(text: rewrittenText, reason: reason)
+ }
+ }
+}