diff --git a/Sources/App/VoicePipeline+Processing.swift b/Sources/App/VoicePipeline+Processing.swift index e58ef8fa..04d95104 100644 --- a/Sources/App/VoicePipeline+Processing.swift +++ b/Sources/App/VoicePipeline+Processing.swift @@ -154,29 +154,34 @@ extension VoicePipeline { appState.statusMessage = L("pipeline.formatting") let started = CFAbsoluteTimeGetCurrent() + let processingOptions = TextProcessingOptions(settings: settings) + let dictionarySnapshot = PersonalDictionary.shared.snapshot() + let enableMemory = settings.enableMemory + let memoryWindowMinutes = settings.memoryWindowMinutes let screenContext = await finishScreenContextCapture() let inputContext = InputContext.capture( targetApp: targetApp, screenContext: screenContext.text, outputMode: .processed, - inputLanguage: settings.inputLanguage, + inputLanguage: processingOptions.inputLanguage, source: .menuBar ) guard !Task.isCancelled else { return VoicePipelineOutput(text: "", context: inputContext) } let memoryContext = VoicePipelinePolicy.memoryContext( for: .processed, - settings: settings, + enableMemory: enableMemory, + memoryWindowMinutes: memoryWindowMinutes, currentContext: inputContext ) let text = await textProcessor.process( text: raw, - stylePrompt: settings.customStylePrompt, - model: settings.llmModel, + options: processingOptions, screenContext: screenContext.text, screenImage: screenContext.image, memoryContext: memoryContext, - inputContext: inputContext + inputContext: inputContext, + dictionarySnapshot: dictionarySnapshot ) recordFormattingDuration(started, label: "Smart Format") return VoicePipelineOutput(text: text, context: inputContext) @@ -191,28 +196,34 @@ extension VoicePipeline { appState.statusMessage = L("pipeline.formatting") let started = CFAbsoluteTimeGetCurrent() + let processingOptions = TextProcessingOptions(settings: settings) + let dictionarySnapshot = PersonalDictionary.shared.snapshot() + let enableMemory = settings.enableMemory + let memoryWindowMinutes = settings.memoryWindowMinutes let screenContext = await finishScreenContextCapture() let inputContext = InputContext.capture( targetApp: targetApp, screenContext: screenContext.text, outputMode: .command, - inputLanguage: settings.inputLanguage, + inputLanguage: processingOptions.inputLanguage, source: .menuBar ) guard !Task.isCancelled else { return VoicePipelineOutput(text: "", context: inputContext) } let memoryContext = VoicePipelinePolicy.memoryContext( for: .command, - settings: settings, + enableMemory: enableMemory, + memoryWindowMinutes: memoryWindowMinutes, currentContext: inputContext ) let text = await textProcessor.processCommand( text: raw, - model: settings.llmModel, + options: processingOptions, screenContext: screenContext.text, screenImage: screenContext.image, memoryContext: memoryContext, - inputContext: inputContext + inputContext: inputContext, + dictionarySnapshot: dictionarySnapshot ) recordFormattingDuration(started, label: "Voice Command formatting") return VoicePipelineOutput(text: text, context: inputContext) diff --git a/Sources/App/VoicePipeline+Replacement.swift b/Sources/App/VoicePipeline+Replacement.swift index 37d05dea..9121bf69 100644 --- a/Sources/App/VoicePipeline+Replacement.swift +++ b/Sources/App/VoicePipeline+Replacement.swift @@ -38,12 +38,20 @@ extension VoicePipeline { settings: AppSettings, targetApp: NSRunningApplication? ) async { - let quickText = immediateInsertText(from: raw, settings: settings) + let processingOptions = TextProcessingOptions(settings: settings) + let dictionarySnapshot = PersonalDictionary.shared.snapshot() + let enableMemory = settings.enableMemory + let memoryWindowMinutes = settings.memoryWindowMinutes + let quickText = immediateInsertText( + from: raw, + inputLanguage: processingOptions.inputLanguage, + dictionarySnapshot: dictionarySnapshot + ) let quickContext = InputContext.capture( targetApp: targetApp, screenContext: "", outputMode: .processed, - inputLanguage: settings.inputLanguage, + inputLanguage: processingOptions.inputLanguage, source: .menuBar ) let ocrTask = screenOCRTask @@ -51,6 +59,14 @@ extension VoicePipeline { screenOCRTask = nil screenOCRStartedAt = nil + guard !quickText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + ocrTask?.cancel() + showNoSpeechDetected( + reason: "instant-insert preprocessing produced empty text" + ) + return + } + appState.processedText = quickText appState.phase = .inserting appState.statusMessage = L("pipeline.inserting") @@ -94,7 +110,10 @@ extension VoicePipeline { await self?.finishDeferredSmartFormat( replacementID: replacement.id, raw: raw, - settings: settings, + processingOptions: processingOptions, + dictionarySnapshot: dictionarySnapshot, + enableMemory: enableMemory, + memoryWindowMinutes: memoryWindowMinutes, ocrTask: ocrTask, ocrStartedAt: ocrStartedAt ) @@ -153,7 +172,10 @@ extension VoicePipeline { private func finishDeferredSmartFormat( replacementID: UUID, raw: String, - settings: AppSettings, + processingOptions: TextProcessingOptions, + dictionarySnapshot: PersonalDictionarySnapshot, + enableMemory: Bool, + memoryWindowMinutes: Int, ocrTask: Task?, ocrStartedAt: CFAbsoluteTime? ) async { @@ -167,30 +189,28 @@ extension VoicePipeline { guard !Task.isCancelled else { return } guard let currentReplacement = appState.pendingReplacement, currentReplacement.id == replacementID else { return } - let inputContext = InputContext( - appName: currentReplacement.targetAppName, - bundleIdentifier: currentReplacement.targetBundleIdentifier, - windowTitle: currentReplacement.context?.windowTitle, + let inputContext = Self.deferredInputContext( + for: currentReplacement, screenContext: screenContext.text, - outputMode: .processed, - inputLanguage: settings.inputLanguage, - source: .menuBar + inputLanguage: processingOptions.inputLanguage ) let memoryContext = VoicePipelinePolicy.memoryContext( for: .processed, - settings: settings, + enableMemory: enableMemory, + memoryWindowMinutes: memoryWindowMinutes, currentContext: inputContext ) let formattedText = await textProcessor.process( text: raw, - stylePrompt: settings.customStylePrompt, - model: settings.llmModel, + options: processingOptions, screenContext: screenContext.text, screenImage: screenContext.image, memoryContext: memoryContext, inputContext: inputContext, - allowsPreparedFallback: false + allowsPreparedFallback: false, + allowsGuardFallback: false, + dictionarySnapshot: dictionarySnapshot ) let elapsed = CFAbsoluteTimeGetCurrent() - started appState.lastFormattingDurationSeconds = elapsed @@ -215,12 +235,43 @@ extension VoicePipeline { appState.pendingReplacement = replacement } - private func immediateInsertText(from raw: String, settings: AppSettings) -> String { - let cleaned = textProcessor.prepareForFormatting(text: raw, inputLanguage: settings.inputLanguage) - let fallback = textProcessor.basicClean(text: raw, inputLanguage: settings.inputLanguage) + static func deferredInputContext( + for replacement: DeferredReplacement, + screenContext: String, + inputLanguage: InputLanguage + ) -> InputContext { + InputContext( + appName: replacement.targetAppName, + bundleIdentifier: replacement.targetBundleIdentifier, + windowTitle: replacement.context?.windowTitle, + screenContext: screenContext, + textBeforeSelection: replacement.context?.textBeforeSelection, + selectedText: replacement.context?.selectedText, + textAfterSelection: replacement.context?.textAfterSelection, + outputMode: .processed, + inputLanguage: inputLanguage, + source: .menuBar + ) + } + + private func immediateInsertText( + from raw: String, + inputLanguage: InputLanguage, + dictionarySnapshot: PersonalDictionarySnapshot + ) -> String { + let cleaned = textProcessor.prepareForFormatting( + text: raw, + inputLanguage: inputLanguage, + dictionarySnapshot: dictionarySnapshot + ) + let fallback = textProcessor.basicClean( + text: raw, + inputLanguage: inputLanguage, + dictionarySnapshot: dictionarySnapshot + ) if !cleaned.isEmpty { return cleaned } if !fallback.isEmpty { return fallback } - return raw.trimmingCharacters(in: .whitespacesAndNewlines) + return "" } private func replacementCopyMessage(for reason: DeferredReplacementCopyReason) -> String { diff --git a/Sources/App/VoicePipeline.swift b/Sources/App/VoicePipeline.swift index 9b29abb8..6138a2c1 100644 --- a/Sources/App/VoicePipeline.swift +++ b/Sources/App/VoicePipeline.swift @@ -116,6 +116,11 @@ final class VoicePipeline { let micID = appState.settings.microphoneID let language = appState.settings.inputLanguage.whisperCode let streamingEnabled = appState.settings.enableStreamingRecognitionBeta + currentEngine?.configureRecognition( + context: SpeechRecognitionContext( + dictionaryEntries: PersonalDictionary.shared.entries + ) + ) if streamingEnabled { currentEngine?.startListening(language: language) { [weak self] partialText in Task { @MainActor in diff --git a/Sources/App/VoicePipelinePolicy.swift b/Sources/App/VoicePipelinePolicy.swift index 1bed2171..ec32116e 100644 --- a/Sources/App/VoicePipelinePolicy.swift +++ b/Sources/App/VoicePipelinePolicy.swift @@ -28,9 +28,26 @@ enum VoicePipelinePolicy { currentContext: InputContext? = nil, recentContextProvider: ((Int, InputContext?) -> String)? = nil ) -> String { - guard settings.enableMemory else { return "" } + memoryContext( + for: outputMode, + enableMemory: settings.enableMemory, + memoryWindowMinutes: settings.memoryWindowMinutes, + currentContext: currentContext, + recentContextProvider: recentContextProvider + ) + } + + @MainActor + static func memoryContext( + for outputMode: OutputMode, + enableMemory: Bool, + memoryWindowMinutes: Int, + currentContext: InputContext? = nil, + recentContextProvider: ((Int, InputContext?) -> String)? = nil + ) -> String { + guard enableMemory else { return "" } guard outputMode != .direct else { return "" } let provider = recentContextProvider ?? { MemoryStore.recentContext(windowMinutes: $0, currentContext: $1) } - return provider(settings.memoryWindowMinutes, currentContext) + return provider(memoryWindowMinutes, currentContext) } } diff --git a/Sources/Config/AppSettings.swift b/Sources/Config/AppSettings.swift index 206010b6..949be66f 100644 --- a/Sources/Config/AppSettings.swift +++ b/Sources/Config/AppSettings.swift @@ -224,7 +224,8 @@ enum InputLanguage: String, Codable, CaseIterable { var localeIdentifier: String { switch self { - case .auto, .chinese: return "zh-CN" + case .auto: return Locale.current.identifier + case .chinese: return "zh-CN" case .english: return "en-US" case .japanese: return "ja-JP" case .korean: return "ko-KR" @@ -235,6 +236,7 @@ enum InputLanguage: String, Codable, CaseIterable { final class AppSettings: ObservableObject { static let shared = AppSettings() + static let defaultLLMModelID = "mlx-community/Qwen3.5-2B-4bit" @Published var hotkeyType: HotkeyType { didSet { @@ -343,7 +345,7 @@ final class AppSettings: ObservableObject { ?? (savedEngine.contains("Whisper") || savedEngine.contains("whisper") ? .whisper : nil) ?? .apple whisperModel = ud.string(forKey: Key.whisperModel.rawValue) ?? "large-v3" - llmModel = ud.string(forKey: Key.llmModel.rawValue) ?? "mlx-community/Qwen3.5-2B-4bit" + llmModel = ud.string(forKey: Key.llmModel.rawValue) ?? Self.defaultLLMModelID microphoneID = ud.string(forKey: Key.microphoneID.rawValue) let savedOutput = ud.string(forKey: Key.outputMode.rawValue) ?? "" outputMode = OutputMode(rawValue: savedOutput) diff --git a/Sources/Config/ModelCatalog.swift b/Sources/Config/ModelCatalog.swift index dae89737..2d5a5475 100644 --- a/Sources/Config/ModelCatalog.swift +++ b/Sources/Config/ModelCatalog.swift @@ -91,11 +91,11 @@ final class ModelCatalog: ObservableObject { let supported = Set(rec.supported) whisperModels = Self.curatedWhisperVariants.compactMap { variant in - let fullName = rec.supported.first { name in - guard let range = name.range(of: variant) else { return false } - let after = name[range.upperBound...] - return after.isEmpty || after.first == "_" - } + let fullName = WhisperModelSelection.matches(defaultID, variant: variant) + ? defaultID + : rec.supported.first { + WhisperModelSelection.matches($0, variant: variant) + } guard let fullName, supported.contains(fullName) else { return nil } return ModelEntry( id: fullName, @@ -106,8 +106,13 @@ final class ModelCatalog: ObservableObject { } appendLocalWhisperModels() - if !whisperModels.contains(where: { $0.id == settings.whisperModel }) { - settings.whisperModel = defaultID + let resolvedWhisperModel = WhisperModelSelection.resolve( + requested: settings.whisperModel, + available: whisperModels.map(\.id), + fallback: defaultID + ) + if settings.whisperModel != resolvedWhisperModel { + settings.whisperModel = resolvedWhisperModel } llmModels = Self.defaultLLMModels.map { @@ -115,7 +120,9 @@ final class ModelCatalog: ObservableObject { } appendLocalLLMModels() if !llmModels.contains(where: { $0.id == settings.llmModel }) { - settings.llmModel = llmModels.first?.id ?? "" + settings.llmModel = llmModels.first(where: { + $0.id == AppSettings.defaultLLMModelID + })?.id ?? llmModels.first?.id ?? "" } asrModels = Self.defaultASRModels.map { diff --git a/Sources/Integration/InputSessionCoordinator+AudioFile.swift b/Sources/Integration/InputSessionCoordinator+AudioFile.swift index 64ebb56c..0f3c1ce5 100644 --- a/Sources/Integration/InputSessionCoordinator+AudioFile.swift +++ b/Sources/Integration/InputSessionCoordinator+AudioFile.swift @@ -23,6 +23,11 @@ extension InputSessionCoordinator { guard let engine = await engineProvider.engine(settings: settings), engine.isReady else { throw IntegrationError.modelNotReady } + engine.configureRecognition( + context: SpeechRecognitionContext( + dictionaryEntries: PersonalDictionary.shared.entries + ) + ) do { try service.emitAudioReceived(sessionID: sessionID, clientID: clientID) diff --git a/Sources/Integration/InputSessionCoordinator+Output.swift b/Sources/Integration/InputSessionCoordinator+Output.swift index d4194cab..67cd7be8 100644 --- a/Sources/Integration/InputSessionCoordinator+Output.swift +++ b/Sources/Integration/InputSessionCoordinator+Output.swift @@ -4,6 +4,9 @@ import Foundation extension InputSessionCoordinator { func outputText(for raw: String, active: ActiveSession) async throws -> String { let options = TextProcessingOptions(settings: settings, inputLanguage: active.inputLanguage) + let dictionarySnapshot = PersonalDictionary.shared.snapshot() + let enableMemory = settings.enableMemory + let memoryWindowMinutes = settings.memoryWindowMinutes let text: String let context: InputContext @@ -11,13 +14,18 @@ extension InputSessionCoordinator { case .direct: active.screenContextTask?.cancel() context = inputContext(for: active, screenContext: "", mode: .direct) - text = textProcessor.basicClean(text: raw, inputLanguage: active.inputLanguage) + text = textProcessor.basicClean( + text: raw, + inputLanguage: active.inputLanguage, + dictionarySnapshot: dictionarySnapshot + ) case .processed: let screenContext = await screenContext(from: active) context = inputContext(for: active, screenContext: screenContext.text, mode: .processed) let memoryContext = VoicePipelinePolicy.memoryContext( for: .processed, - settings: settings, + enableMemory: enableMemory, + memoryWindowMinutes: memoryWindowMinutes, currentContext: context ) text = await textProcessor.process( @@ -26,14 +34,16 @@ extension InputSessionCoordinator { screenContext: screenContext.text, screenImage: screenContext.image, memoryContext: memoryContext, - inputContext: context + inputContext: context, + dictionarySnapshot: dictionarySnapshot ) case .command: let screenContext = await screenContext(from: active) context = inputContext(for: active, screenContext: screenContext.text, mode: .command) let memoryContext = VoicePipelinePolicy.memoryContext( for: .command, - settings: settings, + enableMemory: enableMemory, + memoryWindowMinutes: memoryWindowMinutes, currentContext: context ) text = await textProcessor.processCommand( @@ -42,7 +52,8 @@ extension InputSessionCoordinator { screenContext: screenContext.text, screenImage: screenContext.image, memoryContext: memoryContext, - inputContext: context + inputContext: context, + dictionarySnapshot: dictionarySnapshot ) } diff --git a/Sources/Integration/InputSessionCoordinator.swift b/Sources/Integration/InputSessionCoordinator.swift index 096dea8f..b0d3b0f7 100644 --- a/Sources/Integration/InputSessionCoordinator.swift +++ b/Sources/Integration/InputSessionCoordinator.swift @@ -53,6 +53,11 @@ final class InputSessionCoordinator { guard let engine = await engineProvider.engine(settings: settings), engine.isReady else { throw IntegrationError.modelNotReady } + engine.configureRecognition( + context: SpeechRecognitionContext( + dictionaryEntries: PersonalDictionary.shared.entries + ) + ) if effective.streamingEnabled, engine.supportsStreaming { engine.startListening(language: effective.languageCode) { [weak service] partialText in diff --git a/Sources/LLM/RemoteLLMClient.swift b/Sources/LLM/RemoteLLMClient.swift index 21ff696c..608fae79 100644 --- a/Sources/LLM/RemoteLLMClient.swift +++ b/Sources/LLM/RemoteLLMClient.swift @@ -29,6 +29,50 @@ actor RemoteLLMClient { ) async throws -> String { guard !apiKey.isEmpty else { throw RemoteLLMError.noAPIKey } + do { + return try await generateOnce( + prompt: prompt, + systemPrompt: systemPrompt, + baseURL: baseURL, + apiKey: apiKey, + model: model, + provider: provider, + maxTokens: maxTokens, + temperature: temperature + ) + } catch RemoteLLMError.requestFailed(let message) { + guard let retryTokens = Self.retryTokenBudget( + maxTokens: maxTokens, + failureMessage: message + ) else { + throw RemoteLLMError.requestFailed(message) + } + Log.info( + "[RemoteLLM] retrying token-limit failure with \(retryTokens) max tokens" + ) + return try await generateOnce( + prompt: prompt, + systemPrompt: systemPrompt, + baseURL: baseURL, + apiKey: apiKey, + model: model, + provider: provider, + maxTokens: retryTokens, + temperature: temperature + ) + } + } + + private func generateOnce( + prompt: String, + systemPrompt: String?, + baseURL: String, + apiKey: String, + model: String, + provider: RemoteProvider, + maxTokens: Int, + temperature: Double + ) async throws -> String { switch provider.apiFormat { case .anthropic: return try await generateAnthropic( @@ -46,6 +90,47 @@ actor RemoteLLMClient { } } + nonisolated static func retryTokenBudget( + maxTokens: Int, + failureMessage: String + ) -> Int? { + let message = failureMessage.lowercased() + let indicatesUnsupportedParameter = [ + "unsupported parameter", + "unknown parameter", + "unrecognized parameter", + "use max_completion_tokens", + ].contains { message.contains($0) } + guard !indicatesUnsupportedParameter else { return nil } + + if let statusRange = message.range( + of: #"http\s+(\d{3})"#, + options: .regularExpression + ) { + let status = Int(message[statusRange].filter(\.isNumber)) + guard status == 400 || status == 413 || status == 422 else { + return nil + } + } + + let indicatesContextLimit = [ + "maximum context", + "context length", + "context_length", + "too many tokens", + "token limit", + "max output", + "maximum number of tokens", + ].contains { message.contains($0) } + let indicatesOutputBudgetLimit = message.contains("max_tokens") + && ["must be less", "too large", "exceed", "limit"] + .contains { message.contains($0) } + guard indicatesContextLimit || indicatesOutputBudgetLimit else { return nil } + + let retryTokens = max(256, min(1_024, maxTokens / 2)) + return retryTokens < maxTokens ? retryTokens : nil + } + // MARK: - OpenAI-compatible format private func generateOpenAI( diff --git a/Sources/LLM/RemoteLLMResponseText.swift b/Sources/LLM/RemoteLLMResponseText.swift index f92457d1..bec6ce4a 100644 --- a/Sources/LLM/RemoteLLMResponseText.swift +++ b/Sources/LLM/RemoteLLMResponseText.swift @@ -3,12 +3,17 @@ import Foundation enum RemoteLLMResponseText { static func openAI(from data: Data) throws -> String { if let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] { + try RemoteLLMResponseTruncation.validateOpenAI( + data: data, + json: json + ) if let text = openAIText(in: json) { return resolveStructuredOutput(text) } throw RemoteLLMError.invalidResponse } + try RemoteLLMResponseTruncation.validateOpenAI(data: data, json: nil) if let text = RemoteLLMEventStreamText.openAI(from: data) { return resolveStructuredOutput(text) } @@ -34,12 +39,17 @@ enum RemoteLLMResponseText { static func anthropic(from data: Data) throws -> String { if let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] { + try RemoteLLMResponseTruncation.validateAnthropic( + data: data, + json: json + ) if let text = anthropicText(in: json) { return resolveStructuredOutput(text) } throw RemoteLLMError.invalidResponse } + try RemoteLLMResponseTruncation.validateAnthropic(data: data, json: nil) if let text = RemoteLLMEventStreamText.anthropic(from: data) { return resolveStructuredOutput(text) } diff --git a/Sources/LLM/RemoteLLMResponseTruncation.swift b/Sources/LLM/RemoteLLMResponseTruncation.swift new file mode 100644 index 00000000..c82dc5c4 --- /dev/null +++ b/Sources/LLM/RemoteLLMResponseTruncation.swift @@ -0,0 +1,108 @@ +import Foundation + +enum RemoteLLMResponseTruncation { + static func validateOpenAI( + data: Data, + json: [String: Any]? + ) throws { + if let json { + if contains( + key: "finish_reason", + value: "length", + in: json + ) || ( + contains(key: "status", value: "incomplete", in: json) + && contains( + key: "reason", + value: "max_output_tokens", + in: json + ) + ) { + throw RemoteLLMError.requestFailed( + "provider returned truncated output" + ) + } + return + } + try validateEventStream( + data, + patternGroups: [ + [#""finish_reason"\s*:\s*"length""#], + [ + #""status"\s*:\s*"incomplete""#, + #""reason"\s*:\s*"max_output_tokens""#, + ], + ] + ) + } + + static func validateAnthropic( + data: Data, + json: [String: Any]? + ) throws { + if let json { + if contains(key: "stop_reason", value: "max_tokens", in: json) { + throw RemoteLLMError.requestFailed( + "provider returned truncated output" + ) + } + return + } + try validateEventStream( + data, + patternGroups: [[#""stop_reason"\s*:\s*"max_tokens""#]] + ) + } + + private static func contains( + key expectedKey: String, + value expectedValue: String, + in value: Any + ) -> Bool { + if let object = value as? [String: Any] { + for (key, child) in object { + if key.caseInsensitiveCompare(expectedKey) == .orderedSame, + let string = child as? String, + string.caseInsensitiveCompare(expectedValue) == .orderedSame { + return true + } + if contains( + key: expectedKey, + value: expectedValue, + in: child + ) { + return true + } + } + } else if let array = value as? [Any] { + return array.contains { + contains( + key: expectedKey, + value: expectedValue, + in: $0 + ) + } + } + return false + } + + private static func validateEventStream( + _ data: Data, + patternGroups: [[String]] + ) throws { + guard let text = String(data: data, encoding: .utf8) else { return } + let hasTruncation = patternGroups.contains { patterns in + patterns.allSatisfy { pattern in + text.range( + of: pattern, + options: [.regularExpression, .caseInsensitive] + ) != nil + } + } + if hasTruncation { + throw RemoteLLMError.requestFailed( + "provider returned truncated output" + ) + } + } +} diff --git a/Sources/Processing/PersonalDictionary.swift b/Sources/Processing/PersonalDictionary.swift index c456bd6b..c7ac4b36 100644 --- a/Sources/Processing/PersonalDictionary.swift +++ b/Sources/Processing/PersonalDictionary.swift @@ -1,18 +1,126 @@ import Foundation -struct DictionaryEntry: Codable, Identifiable { +struct DictionaryEntry: Codable, Identifiable, Sendable { var id = UUID() var original: String var replacement: String var enabled: Bool = true } -struct EditRule: Codable, Identifiable { +struct EditRule: Codable, Identifiable, Sendable { var id = UUID() var description: String var enabled: Bool = true } +struct PersonalDictionarySnapshot: Sendable { + let entries: [DictionaryEntry] + let editRules: [EditRule] + + func applyReplacements(to text: String) -> String { + let rules = entries.enumerated().compactMap { offset, entry -> ReplacementRule? in + let original = entry.original + let replacement = entry.replacement + guard entry.enabled, + !original.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + return nil + } + return ReplacementRule( + original: original, + replacement: replacement, + insertionOrder: offset + ) + } + .sorted { + if $0.original.count != $1.original.count { + return $0.original.count > $1.original.count + } + return $0.insertionOrder < $1.insertionOrder + } + + guard !rules.isEmpty, !text.isEmpty else { return text } + + var result = "" + result.reserveCapacity(text.count) + var cursor = text.startIndex + while cursor < text.endIndex { + if let match = rules.first(where: { + Self.matches($0.original, in: text, at: cursor) + }) { + result += match.replacement + cursor = text.index(cursor, offsetBy: match.original.count) + } else { + result.append(text[cursor]) + cursor = text.index(after: cursor) + } + } + return result + } + + var activeEntriesDescription: String { + entries + .filter(\.enabled) + .compactMap { entry -> String? in + let original = entry.original.trimmingCharacters(in: .whitespacesAndNewlines) + let replacement = entry.replacement.trimmingCharacters(in: .whitespacesAndNewlines) + guard !original.isEmpty, !replacement.isEmpty else { return nil } + return "\(original) -> \(replacement)" + } + .joined(separator: "\n") + } + + var activeRulesDescription: String { + editRules + .filter(\.enabled) + .map { $0.description.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + .joined(separator: "\n") + } + + var protectedTerms: [String] { + var seen = Set() + return entries.compactMap { entry in + guard entry.enabled else { return nil } + let term = entry.replacement.trimmingCharacters(in: .whitespacesAndNewlines) + guard !term.isEmpty, + seen.insert(term.lowercased()).inserted else { + return nil + } + return term + } + } + + private static func matches( + _ original: String, + in text: String, + at start: String.Index + ) -> Bool { + guard let end = text.index(start, offsetBy: original.count, limitedBy: text.endIndex), + String(text[start.. text.startIndex, + isASCIIWordCharacter(text[text.index(before: start)]) { + return false + } + if let last = original.last, isASCIIWordCharacter(last), + end < text.endIndex, + isASCIIWordCharacter(text[end]) { + return false + } + return true + } + + private static func isASCIIWordCharacter(_ character: Character) -> Bool { + character.isASCIIWord + } +} + final class PersonalDictionary: ObservableObject { static let shared = PersonalDictionary() @@ -33,31 +141,19 @@ final class PersonalDictionary: ObservableObject { } func applyReplacements(to text: String) -> String { - var result = text - for entry in entries where entry.enabled { - result = result.replacingOccurrences(of: entry.original, with: entry.replacement) - } - return result + snapshot().applyReplacements(to: text) } func activeEntriesDescription() -> String { - entries - .filter(\.enabled) - .compactMap { entry -> String? in - let original = entry.original.trimmingCharacters(in: .whitespacesAndNewlines) - let replacement = entry.replacement.trimmingCharacters(in: .whitespacesAndNewlines) - guard !original.isEmpty, !replacement.isEmpty else { return nil } - return "\(original) -> \(replacement)" - } - .joined(separator: "\n") + snapshot().activeEntriesDescription } func activeRulesDescription() -> String { - editRules - .filter(\.enabled) - .map { $0.description.trimmingCharacters(in: .whitespacesAndNewlines) } - .filter { !$0.isEmpty } - .joined(separator: "\n") + snapshot().activeRulesDescription + } + + func snapshot() -> PersonalDictionarySnapshot { + PersonalDictionarySnapshot(entries: entries, editRules: editRules) } func addEntry(original: String, replacement: String) { @@ -101,4 +197,11 @@ final class PersonalDictionary: ObservableObject { editRules = decoded } } + +} + +private struct ReplacementRule { + let original: String + let replacement: String + let insertionOrder: Int } diff --git a/Sources/Processing/TextProcessingOptions.swift b/Sources/Processing/TextProcessingOptions.swift index 93123858..6f1b6e25 100644 --- a/Sources/Processing/TextProcessingOptions.swift +++ b/Sources/Processing/TextProcessingOptions.swift @@ -6,6 +6,11 @@ struct GenerationOptions { } struct TextProcessingOptions { + enum FidelityPolicy { + case faithfulCorrection + case boundedCustomTransformation + } + var inputLanguage: InputLanguage var languageStyle: LanguageStyle var customStylePrompt: String @@ -16,6 +21,16 @@ struct TextProcessingOptions { var remoteModel: String var remoteProvider: RemoteProvider var screenContextMode: ScreenContextMode + var useCustomSystemPrompt: Bool + var customSystemPrompt: String + + var fidelityPolicy: FidelityPolicy { + let hasCustomSystemPrompt = useCustomSystemPrompt + && !customSystemPrompt.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + return hasCustomSystemPrompt + ? .boundedCustomTransformation + : .faithfulCorrection + } init(settings: AppSettings, inputLanguage: InputLanguage? = nil) { self.inputLanguage = inputLanguage ?? settings.inputLanguage @@ -28,5 +43,7 @@ struct TextProcessingOptions { self.remoteModel = settings.remoteModel self.remoteProvider = settings.remoteProvider self.screenContextMode = settings.screenContextMode + self.useCustomSystemPrompt = settings.useCustomSystemPrompt + self.customSystemPrompt = settings.customSystemPrompt } } diff --git a/Sources/Processing/TextProcessor+Output.swift b/Sources/Processing/TextProcessor+Output.swift index 7ad6a2ac..0a9b235f 100644 --- a/Sources/Processing/TextProcessor+Output.swift +++ b/Sources/Processing/TextProcessor+Output.swift @@ -38,33 +38,84 @@ extension TextProcessor { func formattingOptions(for text: String, style: LanguageStyle) -> GenerationOptions { let characterCount = text.trimmingCharacters(in: .whitespacesAndNewlines).count - let maxTokens: Int + let minimumTokens: Int switch (style, characterCount) { case (.professional, 0...80), (.custom, 0...80): - maxTokens = 224 + minimumTokens = 224 case (.professional, 81...220), (.custom, 81...220): - maxTokens = 384 + minimumTokens = 384 case (.professional, _), (.custom, _): - maxTokens = 640 + minimumTokens = 640 case (.casual, 0...80): - maxTokens = 160 + minimumTokens = 160 case (.casual, 81...220): - maxTokens = 256 + minimumTokens = 256 case (.casual, _): - maxTokens = 384 + minimumTokens = 384 } - let temperature: Double - switch style { - case .casual: - temperature = 0.08 - case .professional, .custom: - temperature = 0.10 - } + // CJK transcripts can approach one token per character. Leave enough + // room for punctuation and formatting without letting requests grow + // beyond the supported generation ceiling. + let estimatedTokens = min(characterCount, 2_048) * 2 + let maxTokens = min(4_096, max(minimumTokens, estimatedTokens)) return GenerationOptions( maxTokens: maxTokens, - temperature: temperature + temperature: 0 ) } + + /// Dictionary replacements are applied once on the input side. Applying + /// them again could double-expand replacements that contain the original. + func cleanGeneratedOutput( + _ text: String, + inputLanguage: InputLanguage, + fallback: String = "" + ) -> String { + var result = stripThinkingTags(text) + result = FormattedOutputCleaner.clean(result) + if result.isEmpty { return FormattedOutputCleaner.clean(fallback) } + return result + } + + /// Command output is entirely model-generated, so parsing its advertised + /// final_text envelope cannot swallow dictated content. + func cleanCommandGeneratedOutput( + _ text: String, + inputLanguage: InputLanguage + ) -> String { + let stripped = stripThinkingTags(text) + if let finalText = LLMFinalTextOutput.text(from: stripped) { + return FormattedOutputCleaner.clean(finalText) + } + return cleanGeneratedOutput(text, inputLanguage: inputLanguage) + } + + func rejectedOutputFallback( + _ cleanedText: String, + allowsGuardFallback: Bool + ) -> String { + allowsGuardFallback ? cleanedText : "" + } + + /// Keeps line breaks while collapsing surrounding whitespace. + func normalizeWhitespace(_ text: String) -> String { + TranscriptionSanitizer.normalizeInput(text) + .replacingOccurrences( + of: "[^\\S\\n]*\\n[^\\S\\n]*", + with: "\n", + options: .regularExpression + ) + .replacingOccurrences( + of: "[^\\S\\n]+", + with: " ", + options: .regularExpression + ) + .replacingOccurrences( + of: "\\n{3,}", + with: "\n\n", + options: .regularExpression + ) + } } diff --git a/Sources/Processing/TextProcessor+PromptConstruction.swift b/Sources/Processing/TextProcessor+PromptConstruction.swift new file mode 100644 index 00000000..dfece8c8 --- /dev/null +++ b/Sources/Processing/TextProcessor+PromptConstruction.swift @@ -0,0 +1,88 @@ +import Foundation + +extension TextProcessor { + func formattingUserPrompt( + text: String, + options: TextProcessingOptions + ) -> String { + switch options.fidelityPolicy { + case .faithfulCorrection: + return PromptBuilder.buildUserPrompt( + text: text, + inputLanguage: options.inputLanguage + ) + case .boundedCustomTransformation: + return PromptBuilder.buildCustomUserPrompt( + text: text, + inputLanguage: options.inputLanguage + ) + } + } + + func formattingSystemPrompt( + options: TextProcessingOptions, + screenContext: String, + screenImageAvailable: Bool, + memoryContext: String, + inputContext: InputContext?, + dictionarySnapshot: PersonalDictionarySnapshot? = nil + ) -> String { + systemPromptWithPersonalContext( + PromptBuilder.buildSystemPrompt( + style: options.languageStyle, + stylePrompt: options.customStylePrompt, + screenContext: screenContext, + screenImageAvailable: screenImageAvailable, + memoryContext: memoryContext, + inputContext: inputContext, + inputLanguage: options.inputLanguage, + useCustomSystemPrompt: options.useCustomSystemPrompt, + customSystemPrompt: options.customSystemPrompt + ), + inputLanguage: options.inputLanguage, + dictionarySnapshot: dictionarySnapshot + ) + } + + func commandSystemPrompt( + options: TextProcessingOptions, + screenContext: String, + screenImageAvailable: Bool, + memoryContext: String, + inputContext: InputContext?, + dictionarySnapshot: PersonalDictionarySnapshot? = nil + ) -> String { + systemPromptWithPersonalContext( + PromptBuilder.buildCommandSystemPrompt( + screenContext: screenContext, + screenImageAvailable: screenImageAvailable, + memoryContext: memoryContext, + inputContext: inputContext, + inputLanguage: options.inputLanguage + ), + inputLanguage: options.inputLanguage, + dictionarySnapshot: dictionarySnapshot + ) + } + + func systemPromptWithPersonalContext( + _ systemPrompt: String, + inputLanguage: InputLanguage, + dictionarySnapshot: PersonalDictionarySnapshot? = nil + ) -> String { + let snapshot = dictionarySnapshot ?? PersonalDictionary.shared.snapshot() + let extraSections = [ + PromptCatalog.activePersonalDictionarySection( + snapshot.activeEntriesDescription, + inputLanguage: inputLanguage + ), + PromptCatalog.activeEditRulesSection( + snapshot.activeRulesDescription, + inputLanguage: inputLanguage + ), + ].compactMap { $0 } + + guard !extraSections.isEmpty else { return systemPrompt } + return ([systemPrompt] + extraSections).joined(separator: "\n\n") + } +} diff --git a/Sources/Processing/TextProcessor.swift b/Sources/Processing/TextProcessor.swift index 9ae2ee78..7e9b6789 100644 --- a/Sources/Processing/TextProcessor.swift +++ b/Sources/Processing/TextProcessor.swift @@ -32,14 +32,24 @@ final class TextProcessor { } } - func basicClean(text: String, inputLanguage: InputLanguage = .auto) -> String { - var result = dictionary.applyReplacements(to: text) + func basicClean( + text: String, + inputLanguage: InputLanguage = .auto, + dictionarySnapshot: PersonalDictionarySnapshot? = nil + ) -> String { + let snapshot = dictionarySnapshot ?? dictionary.snapshot() + var result = snapshot.applyReplacements(to: text) result = normalizeWhitespace(result) return result.trimmingCharacters(in: .whitespacesAndNewlines) } - func prepareForFormatting(text: String, inputLanguage: InputLanguage) -> String { - var result = dictionary.applyReplacements(to: text) + func prepareForFormatting( + text: String, + inputLanguage: InputLanguage, + dictionarySnapshot: PersonalDictionarySnapshot? = nil + ) -> String { + let snapshot = dictionarySnapshot ?? dictionary.snapshot() + var result = snapshot.applyReplacements(to: text) result = TranscriptionSanitizer.normalizeInput(result) return result.trimmingCharacters(in: .whitespacesAndNewlines) } @@ -52,7 +62,8 @@ final class TextProcessor { screenImage: CGImage? = nil, memoryContext: String = "", inputContext: InputContext? = nil, - allowsPreparedFallback: Bool = TextProcessor.defaultAllowsPreparedFallback + allowsPreparedFallback: Bool = TextProcessor.defaultAllowsPreparedFallback, + allowsGuardFallback: Bool = true ) async -> String { let settings = AppSettings.shared var options = TextProcessingOptions(settings: settings) @@ -65,7 +76,8 @@ final class TextProcessor { screenImage: screenImage, memoryContext: memoryContext, inputContext: inputContext, - allowsPreparedFallback: allowsPreparedFallback + allowsPreparedFallback: allowsPreparedFallback, + allowsGuardFallback: allowsGuardFallback ) } @@ -76,34 +88,41 @@ final class TextProcessor { screenImage: CGImage? = nil, memoryContext: String = "", inputContext: InputContext? = nil, - allowsPreparedFallback: Bool = TextProcessor.defaultAllowsPreparedFallback + allowsPreparedFallback: Bool = TextProcessor.defaultAllowsPreparedFallback, + allowsGuardFallback: Bool = true, + dictionarySnapshot requestedDictionarySnapshot: PersonalDictionarySnapshot? = nil ) async -> String { let prepareStarted = CFAbsoluteTimeGetCurrent() - let cleanedText = prepareForFormatting(text: text, inputLanguage: options.inputLanguage) + let dictionarySnapshot = requestedDictionarySnapshot ?? dictionary.snapshot() + let cleanedText = prepareForFormatting( + text: text, + inputLanguage: options.inputLanguage, + dictionarySnapshot: dictionarySnapshot + ) let prepareElapsed = CFAbsoluteTimeGetCurrent() - prepareStarted Log.info("[TextProcessor] prepared LLM input \(text.count) chars to \(cleanedText.count) chars in \(String(format: "%.2f", prepareElapsed))s") guard !cleanedText.isEmpty else { return "" } - let systemPrompt = systemPromptWithPersonalContext( - PromptBuilder.buildSystemPrompt( - style: options.languageStyle, - stylePrompt: options.customStylePrompt, - screenContext: screenContext, - screenImageAvailable: shouldUseScreenImage(options: options, image: screenImage), - memoryContext: memoryContext, - inputContext: inputContext, - inputLanguage: options.inputLanguage - ), - inputLanguage: options.inputLanguage + let useScreenImage = shouldUseScreenImage(options: options, image: screenImage) + let systemPrompt = formattingSystemPrompt( + options: options, + screenContext: screenContext, + screenImageAvailable: useScreenImage, + memoryContext: memoryContext, + inputContext: inputContext, + dictionarySnapshot: dictionarySnapshot ) - let userPrompt = PromptBuilder.buildUserPrompt(text: cleanedText, inputLanguage: options.inputLanguage) + let userPrompt = formattingUserPrompt( + text: cleanedText, + options: options + ) let generationOptions = formattingOptions(for: cleanedText, style: options.languageStyle) do { var result: String let llmStarted = CFAbsoluteTimeGetCurrent() - if let screenImage, shouldUseScreenImage(options: options, image: screenImage) { + if let screenImage, useScreenImage { do { result = try await generateWithScreenImage( prompt: userPrompt, @@ -115,9 +134,17 @@ final class TextProcessor { ) } catch { Log.error("[TextProcessor] VLM failed, falling back to text LLM: \(error.localizedDescription)") + let textFallbackSystemPrompt = formattingSystemPrompt( + options: options, + screenContext: screenContext, + screenImageAvailable: false, + memoryContext: memoryContext, + inputContext: inputContext, + dictionarySnapshot: dictionarySnapshot + ) result = try await generateText( prompt: userPrompt, - systemPrompt: systemPrompt, + systemPrompt: textFallbackSystemPrompt, options: options, maxTokens: generationOptions.maxTokens, temperature: generationOptions.temperature @@ -136,11 +163,29 @@ final class TextProcessor { Log.info("[TextProcessor] formatting LLM completed in \(String(format: "%.2f", llmElapsed))s with budget \(generationOptions.maxTokens) tokens") let fallback = allowsPreparedFallback ? cleanedText : "" - return cleanGeneratedOutput(result, inputLanguage: options.inputLanguage, fallback: fallback) + let output = cleanGeneratedOutput( + result, + inputLanguage: options.inputLanguage, + fallback: fallback + ) + if let violation = TranscriptFidelityGuard.violation( + source: cleanedText, + candidate: output, + protectedTerms: dictionarySnapshot.protectedTerms, + inputLanguage: options.inputLanguage, + enforceSemanticFidelity: options.fidelityPolicy == .faithfulCorrection + ) { + Log.error("[TextProcessor] rejected unsafe formatting output: \(violation)") + return rejectedOutputFallback( + cleanedText, + allowsGuardFallback: allowsGuardFallback + ) + } + return output } catch { if allowsPreparedFallback { Log.error("[TextProcessor] LLM failed, falling back to prepared raw text: \(error.localizedDescription)") - return FormattedOutputCleaner.clean(cleanedText) + return cleanedText } Log.error("[TextProcessor] LLM failed with prepared fallback disabled: \(error.localizedDescription)") return "" @@ -175,17 +220,18 @@ final class TextProcessor { screenContext: String, screenImage: CGImage? = nil, memoryContext: String = "", - inputContext: InputContext? = nil + inputContext: InputContext? = nil, + dictionarySnapshot requestedDictionarySnapshot: PersonalDictionarySnapshot? = nil ) async -> String { - let systemPrompt = systemPromptWithPersonalContext( - PromptBuilder.buildCommandSystemPrompt( - screenContext: screenContext, - screenImageAvailable: shouldUseScreenImage(options: options, image: screenImage), - memoryContext: memoryContext, - inputContext: inputContext, - inputLanguage: options.inputLanguage - ), - inputLanguage: options.inputLanguage + let dictionarySnapshot = requestedDictionarySnapshot ?? dictionary.snapshot() + let useScreenImage = shouldUseScreenImage(options: options, image: screenImage) + let systemPrompt = commandSystemPrompt( + options: options, + screenContext: screenContext, + screenImageAvailable: useScreenImage, + memoryContext: memoryContext, + inputContext: inputContext, + dictionarySnapshot: dictionarySnapshot ) let userPrompt = PromptBuilder.buildCommandUserPrompt( text: text, @@ -194,7 +240,7 @@ final class TextProcessor { do { var result: String - if let screenImage, shouldUseScreenImage(options: options, image: screenImage) { + if let screenImage, useScreenImage { do { result = try await generateWithScreenImage( prompt: userPrompt, @@ -206,9 +252,17 @@ final class TextProcessor { ) } catch { Log.error("[TextProcessor] Command VLM failed, falling back to text LLM: \(error.localizedDescription)") + let textFallbackSystemPrompt = commandSystemPrompt( + options: options, + screenContext: screenContext, + screenImageAvailable: false, + memoryContext: memoryContext, + inputContext: inputContext, + dictionarySnapshot: dictionarySnapshot + ) result = try await generateText( prompt: userPrompt, - systemPrompt: systemPrompt, + systemPrompt: textFallbackSystemPrompt, options: options, maxTokens: 4096, temperature: 0.3 @@ -231,49 +285,4 @@ final class TextProcessor { } } - /// Dictionary replacements are applied once on the input side - /// (`prepareForFormatting` / `basicClean`); reapplying them here could - /// double-expand terms whose replacement contains the original. - func cleanGeneratedOutput(_ text: String, inputLanguage: InputLanguage, fallback: String = "") -> String { - var result = stripThinkingTags(text) - result = FormattedOutputCleaner.clean(result) - if result.isEmpty { return FormattedOutputCleaner.clean(fallback) } - return result - } - - /// Command prompts advertise a final_text JSON contract, so honoring that - /// envelope here is contract parsing, not guessing. The command output is - /// entirely model-generated — no dictated content can be swallowed. - func cleanCommandGeneratedOutput(_ text: String, inputLanguage: InputLanguage) -> String { - let stripped = stripThinkingTags(text) - if let finalText = LLMFinalTextOutput.text(from: stripped) { - return FormattedOutputCleaner.clean(finalText) - } - return cleanGeneratedOutput(text, inputLanguage: inputLanguage) - } - - func systemPromptWithPersonalContext(_ systemPrompt: String, inputLanguage: InputLanguage) -> String { - let extraSections = [ - PromptCatalog.activePersonalDictionarySection( - dictionary.activeEntriesDescription(), - inputLanguage: inputLanguage - ), - PromptCatalog.activeEditRulesSection( - dictionary.activeRulesDescription(), - inputLanguage: inputLanguage - ), - ].compactMap { $0 } - - guard !extraSections.isEmpty else { return systemPrompt } - return ([systemPrompt] + extraSections).joined(separator: "\n\n") - } - - /// Collapses runs of spaces but keeps line breaks: direct mode promises - /// verbatim output, and ASR engines only emit newlines deliberately. - private func normalizeWhitespace(_ text: String) -> String { - TranscriptionSanitizer.normalizeInput(text) - .replacingOccurrences(of: "[^\\S\\n]*\\n[^\\S\\n]*", with: "\n", options: .regularExpression) - .replacingOccurrences(of: "[^\\S\\n]+", with: " ", options: .regularExpression) - .replacingOccurrences(of: "\\n{3,}", with: "\n\n", options: .regularExpression) - } } diff --git a/Sources/Processing/TranscriptContentFidelity.swift b/Sources/Processing/TranscriptContentFidelity.swift new file mode 100644 index 00000000..6e618a5a --- /dev/null +++ b/Sources/Processing/TranscriptContentFidelity.swift @@ -0,0 +1,180 @@ +import Foundation + +extension TranscriptFidelityGuard { + static func contentFidelityViolation( + source: String, + candidate: String + ) -> String? { + let sourceUnits = fidelityContentUnits( + removingSelfCorrectionContent( + from: removingDisfluencyMarkers(from: source) + ), + pairedWith: candidate + ) + let candidateUnits = fidelityContentUnits( + removingDisfluencyMarkers(from: candidate), + pairedWith: source + ) + guard !candidateUnits.isEmpty else { return "empty_output" } + guard !sourceUnits.isEmpty else { return nil } + + if candidateUnits.count > Int(Double(sourceUnits.count) * 1.35) + 3 { + return "excessive_expansion" + } + if sourceUnits.count >= 6, + candidateUnits.count * 100 < sourceUnits.count * 55 { + return "excessive_deletion" + } + + let sourceSample = comparisonSample(sourceUnits) + let candidateSample = comparisonSample(candidateUnits) + let denominator = sourceSample.count + guard denominator > 0 else { return nil } + let overlap = longestCommonSubsequenceLength( + sourceSample, + candidateSample + ) + let requiredOverlap = denominator <= 8 ? 80 : 55 + return overlap * 100 < denominator * requiredOverlap + ? "content_drift" : nil + } + + static func contentUnits(_ text: String) -> [String] { + let normalized = text.precomposedStringWithCompatibilityMapping.lowercased() + var units: [String] = [] + var asciiWord = "" + func flushASCIIWord() { + guard !asciiWord.isEmpty else { return } + units.append(asciiWord) + asciiWord.removeAll(keepingCapacity: true) + } + for character in normalized { + if character.isASCIIWord { + asciiWord.append(character) + } else { + flushASCIIWord() + if character.isLetter || character.isNumber { + units.append(String(character)) + } + } + } + flushASCIIWord() + return units + } + + static func comparisonSample(_ units: [String]) -> [String] { + guard units.count > 1_024 else { return units } + let step = Double(units.count - 1) / 1_023 + return (0..<1_024).map { offset in + units[Int((Double(offset) * step).rounded())] + } + } + + static func longestCommonSubsequenceLength( + _ lhs: [String], + _ rhs: [String] + ) -> Int { + var lengths = Array(repeating: 0, count: rhs.count + 1) + for left in lhs { + var diagonal = 0 + for index in rhs.indices { + let previous = lengths[index + 1] + if left == rhs[index] { + lengths[index + 1] = diagonal + 1 + } else { + lengths[index + 1] = max( + lengths[index + 1], + lengths[index] + ) + } + diagonal = previous + } + } + return lengths[rhs.count] + } + + static func fidelityContentUnits( + _ text: String, + pairedWith otherText: String + ) -> [String] { + let numbers = numericSemanticValues(in: text) + let units = contentUnits( + numbers.isEmpty ? text : removingRecognizedNumberEvidence(from: text) + ) + guard !numbers.isEmpty, + numbers == numericSemanticValues(in: otherText) else { + return units + } + let numericWords: Set = [ + "zero", "one", "two", "three", "four", "five", "six", + "seven", "eight", "nine", "ten", "eleven", "twelve", + "thirteen", "fourteen", "fifteen", "sixteen", "seventeen", + "eighteen", "nineteen", "twenty", "thirty", "forty", + "fifty", "sixty", "seventy", "eighty", "ninety", + "hundred", "thousand", "million", "first", "second", + "third", "half", "quarter", + "from", "to", + "second", "seconds", "minute", "minutes", "hour", "hours", + "day", "days", "month", "months", "year", "years", + "meter", "meters", "metre", "metres", "degree", "degrees", + "dollar", "dollars", "page", "pages", "time", "times", + "kg", "kilogram", "kilograms", "lb", "lbs", "pound", "pounds", + "km", "cm", "mm", "ml", "mb", "gb", "tb", + "celsius", "fahrenheit", + ] + let numericCharacters = Set( + "0123456789零〇一二两兩三四五六七八九十百千万萬亿億兆" + + "第百分之点點到至年月日时時分秒步页頁次元块塊米度" + ) + let filtered = units.filter { unit in + if numericWords.contains(unit) { return false } + if unit.allSatisfy(\.isNumber) { return false } + return !(unit.count == 1 && unit.first.map(numericCharacters.contains) == true) + } + return filtered.isEmpty ? ["#numeric"] : filtered + } + + static func removingDisfluencyMarkers(from text: String) -> String { + let patterns = [ + #"(?i)\b(?:um+|uh+|you know)\b[\s,;:，。；：]*"#, + #"(?:嗯+|呃+|额+|えっと|あのー?|음+|저기)[\s,;:，。；：]*"#, + ] + return patterns.reduce(text) { result, pattern in + let range = NSRange(result.startIndex..., in: result) + return (try! NSRegularExpression(pattern: pattern)) + .stringByReplacingMatches(in: result, range: range, withTemplate: "") + } + } + + static func removingSelfCorrectionContent(from text: String) -> String { + let patterns = [ + #"(?:唔係|不是)(?:星期|禮拜|礼拜|周)?[零〇一二两兩三四五六七八九十]+[\s，,]*(?:係|是)"#, + #"(?:星期|禮拜|礼拜|周)?[零〇一二两兩三四五六七八九十]+[\s，,]*(?:不对|講錯咗|说错了)[\s，,]*"#, + #"(?i)\b\w+[\s,]+(?:sorry|i mean|correction)[\s,]+"#, + #"[^、。！？,\s]+(?:じゃなくて|ではなく)[、,\s]*"#, + #"[^,.!?\s]+[\s,]*(?:아니고|정정)[\s,]*"#, + ] + return patterns.reduce(text) { result, pattern in + let range = NSRange(result.startIndex..., in: result) + return (try! NSRegularExpression(pattern: pattern)) + .stringByReplacingMatches( + in: result, + range: range, + withTemplate: "" + ) + } + } +} + +extension Character { + var isASCIIWord: Bool { + guard unicodeScalars.count == 1, + let value = unicodeScalars.first?.value else { + return false + } + return (48...57).contains(value) + || (65...90).contains(value) + || value == 95 + || (97...122).contains(value) + } +} diff --git a/Sources/Processing/TranscriptFidelityGuard.swift b/Sources/Processing/TranscriptFidelityGuard.swift new file mode 100644 index 00000000..6397514d --- /dev/null +++ b/Sources/Processing/TranscriptFidelityGuard.swift @@ -0,0 +1,178 @@ +import Foundation + +enum TranscriptFidelityGuard { + static func violation( + source: String, + candidate: String, + protectedTerms: [String], + inputLanguage: InputLanguage, + enforceSemanticFidelity: Bool + ) -> String? { + guard !candidate.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + return "empty_output" + } + let tokensAreSafe = enforceSemanticFidelity + ? protectedTokensAreFaithful(source: source, candidate: candidate) + : protectedTokensAreBoundedTransformation( + source: source, + candidate: candidate + ) + guard tokensAreSafe else { + return "protected_token_change" + } + let sourceTerms = protectedTermCounts(in: source, terms: protectedTerms) + let candidateTerms = protectedTermCounts(in: candidate, terms: protectedTerms) + let termsAreSafe = enforceSemanticFidelity + ? sourceTerms == candidateTerms + : candidateTerms.allSatisfy { sourceTerms[$0.key, default: 0] >= $0.value } + guard termsAreSafe else { + return "dictionary_term_change" + } + if enforceSemanticFidelity { + guard polaritySignatures(in: source, language: inputLanguage) + == polaritySignatures(in: candidate, language: inputLanguage) else { + return "polarity_change" + } + if let violation = contentFidelityViolation( + source: source, + candidate: candidate + ) { + return violation + } + } + return nil + } +} + +extension TranscriptFidelityGuard { + static func protectedTermCounts( + in text: String, + terms: [String] + ) -> [String: Int] { + let normalized = text.precomposedStringWithCanonicalMapping + let fullRange = NSRange(normalized.startIndex..., in: normalized) + var counts: [String: Int] = [:] + for term in terms { + let normalizedTerm = term.precomposedStringWithCanonicalMapping + guard !normalizedTerm.isEmpty else { continue } + let escaped = NSRegularExpression.escapedPattern(for: normalizedTerm) + let left = normalizedTerm.first?.isASCIIWord == true + ? "(? 0 { counts[normalizedTerm] = count } + } + return counts + } + + static func polaritySignatures( + in text: String, + language: InputLanguage + ) -> [String: Int] { + let sanitized = removingSelfCorrectionNegation( + from: text.precomposedStringWithCompatibilityMapping, + language: language + ) + let patterns = polarityPatterns(language: language) + let range = NSRange(sanitized.startIndex..., in: sanitized) + var signatures: [String: Int] = [:] + for pattern in patterns { + let expression = try! NSRegularExpression(pattern: pattern) + for match in expression.matches(in: sanitized, range: range) { + let end = match.range.location + match.range.length + let suffixRange = NSRange( + location: end, + length: max(0, range.length - end) + ) + let suffix = Range(suffixRange, in: sanitized) + .map { String(sanitized[$0]) } ?? "" + let anchor = contentUnits( + removingDisfluencyMarkers(from: suffix) + ) + .prefix(2) + .joined(separator: "/") + signatures["neg:\(anchor)", default: 0] += 1 + } + } + return signatures + } + + static func polarityPatterns(language: InputLanguage) -> [String] { + switch language { + case .chinese, .cantonese: + return [chinesePolarityPattern] + case .english: + return [englishPolarityPattern] + case .japanese: + return [japanesePolarityPattern] + case .korean: + return [koreanPolarityPattern] + case .auto: + return [ + chinesePolarityPattern, + englishPolarityPattern, + japanesePolarityPattern, + koreanPolarityPattern, + ] + } + } + + static func removingSelfCorrectionNegation( + from text: String, + language: InputLanguage + ) -> String { + let patterns: [String] + switch language { + case .chinese, .cantonese: + patterns = chineseSelfCorrectionPatterns + case .english: + patterns = englishSelfCorrectionPatterns + case .japanese: + patterns = japaneseSelfCorrectionPatterns + case .korean: + patterns = koreanSelfCorrectionPatterns + case .auto: + patterns = chineseSelfCorrectionPatterns + + englishSelfCorrectionPatterns + + japaneseSelfCorrectionPatterns + + koreanSelfCorrectionPatterns + } + return patterns.reduce(text) { result, pattern in + let range = NSRange(result.startIndex..., in: result) + return (try! NSRegularExpression(pattern: pattern)) + .stringByReplacingMatches( + in: result, + range: range, + withTemplate: "" + ) + } + } + + static let chineseSelfCorrectionPatterns = [ + "不对", + "不是(?=.{0,40}(?:而是|改成|应该是))", + "(?:没有|没)(?=.{0,40}(?:应该有|改成有))", + "唔係(?=.{0,40}(?:係|改做|應該係))", + "冇(?=.{0,40}(?:應該有|改做有))", + ] + static let englishSelfCorrectionPatterns = [ + #"(?i)\b(?:no|not)\b(?=.{0,50}\b(?:sorry|rather|i mean|correction)\b)"#, + #"(?i)n['’]t(?=.{0,50}\b(?:sorry|rather|i mean|correction)\b)"#, + ] + static let japaneseSelfCorrectionPatterns = [ + "(?:ではない|じゃない)(?=.{0,40}(?:訂正|ではなく|じゃなく))", + ] + static let koreanSelfCorrectionPatterns = ["아니(?=고)"] + + static let chinesePolarityPattern = + "(?:不同意|不赞成|不能|不要|不会|不是|没有|从未|无需|无法|别|勿|唔(?:係|好|會|能|要)?|冇|不(?!对|同|仅|过|管)|没(?!关系)|未(?!来)|无(?!线|论|数))" + static let englishPolarityPattern = + #"(?i)(?:\bno\b(?!\s*\.\s*\d)|\b(?:not|never|without|cannot|neither|nor|hardly)\b|n['’]t\b)"# + static let japanesePolarityPattern = + "(?:ではない|じゃない|できない|ません|(? [String] { + var collapsed: [String] = [] + var index = 0 + while index < sequence.count { + if index + 3 < sequence.count, + sequence[index].hasPrefix("number:"), + sequence[index + 1].hasPrefix("unit:"), + sequence[index + 2].hasPrefix("number:"), + sequence[index + 1] == sequence[index + 3] { + collapsed.append(sequence[index]) + collapsed.append(sequence[index + 2]) + collapsed.append(sequence[index + 3]) + index += 4 + } else { + collapsed.append(sequence[index]) + index += 1 + } + } + return collapsed + } + + static func numberSemanticKeys( + for event: NumberEvent, + in text: String + ) -> [String] { + guard let range = Range(event.range, in: text) else { + return event.values.map { "number:\($0)" } + } + let raw = String(text[range]) + if event.values.count == 3, + raw.contains("-") || raw.contains("/") { + let firstDigits = raw.prefix { $0.isNumber } + if firstDigits.count == 4 { + let dateUnits = ["year", "month", "day"] + return zip(event.values, dateUnits).flatMap { + ["number:\($0.0)", "unit:\($0.1)"] + } + } + } + + var keys = event.values.map { "number:\($0)" } + if let unit = measurementUnit( + before: range.lowerBound, + after: range.upperBound, + in: text + ) { + keys.append("unit:\(unit)") + } + return keys + } + + private static func measurementUnit( + before startIndex: String.Index, + after index: String.Index, + in text: String + ) -> String? { + let prefix = String(text[.. [String] { + let normalized = text.precomposedStringWithCanonicalMapping + let protected = protectedTokens(in: normalized) + var events: [(range: NSRange, values: [String])] = protected + .filter { $0.category != "number" } + .map { ($0.range, [$0.key]) } + events += numberEvents(in: normalized, protectedTokens: protected) + .map { event in + ( + event.range, + numberSemanticKeys(for: event, in: normalized) + ) + } + let sequence = events + .sorted { lhs, rhs in + if lhs.range.location != rhs.range.location { + return lhs.range.location < rhs.range.location + } + return lhs.range.length > rhs.range.length + } + .flatMap(\.values) + return collapsingRepeatedRangeUnits(sequence) + } + + static func numericSemanticValues(in text: String) -> [String] { + let normalized = text.precomposedStringWithCanonicalMapping + return numberEvents( + in: normalized, + protectedTokens: protectedTokens(in: normalized) + ) + .sorted { $0.range.location < $1.range.location } + .flatMap(\.values) + } + + static func numberEvents( + in text: String, + protectedTokens: [ProtectedToken] + ) -> [NumberEvent] { + var events = protectedTokens + .filter { $0.category == "number" } + .compactMap { token -> NumberEvent? in + guard let range = Range(token.range, in: text) else { return nil } + let raw = String(text[range]) + let previousCharacter = range.lowerBound > text.startIndex + ? text[text.index(before: range.lowerBound)] : nil + let values = arabicNumberValues( + raw, + previousCharacter: previousCharacter + ) + return values.isEmpty ? nil : NumberEvent( + range: token.range, + values: values + ) + } + let occupied = protectedTokens.map(\.range) + let fullRange = NSRange(text.startIndex..., in: text) + + for match in chineseSpokenNumber.matches(in: text, range: fullRange) { + guard !overlaps(match.range, occupied), + let range = Range(match.range, in: text), + var value = chineseNumberValue(String(text[range])) else { + continue + } + let raw = String(text[range]) + let suffix = String(text[range.upperBound...].prefix(4)) + guard isPlausibleChineseNumberEvidence( + raw: raw, + range: range, + text: text + ) else { + continue + } + if raw == "万一" || raw == "萬一" + || ((raw == "千万" || raw == "千萬") + && suffix.range( + of: #"^(?:不要|别|別|勿|不能|唔好)"#, + options: .regularExpression + ) != nil) { + continue + } + if !value.hasSuffix("%") { + let prefix = String(text[.. String { + var normalized = text.precomposedStringWithCanonicalMapping + let protected = protectedTokens(in: normalized) + let ranges = numberEvents(in: normalized, protectedTokens: protected) + .map(\.range) + .sorted { $0.location > $1.location } + for range in ranges { + guard let swiftRange = Range(range, in: normalized) else { continue } + normalized.replaceSubrange(swiftRange, with: " ") + } + return normalized + } + + private static func isPlausibleChineseNumberEvidence( + raw: String, + range: Range, + text: String + ) -> Bool { + if raw.hasPrefix("第") || raw.hasPrefix("百分之") + || raw.contains("点") || raw.contains("點") { + return true + } + if raw.count > 1 { return true } + + let prefix = String(text[.. Bool { + occupied.contains { NSIntersectionRange($0, range).length > 0 } + } + + private static func arabicNumberValues( + _ rawValue: String, + previousCharacter: Character? + ) -> [String] { + var raw = rawValue + .replacingOccurrences(of: "％", with: "%") + .precomposedStringWithCompatibilityMapping + let hasPercent = raw.hasSuffix("%") + if hasPercent { raw.removeLast() } + var sign = "" + if raw.first == "+" || raw.first == "-" { + if raw.first == "-", + previousCharacter?.isNumber == true || previousCharacter == "%" || + previousCharacter == "％" { + raw.removeFirst() + } else { + sign = String(raw.removeFirst()) + } + } + let components = raw.split( + whereSeparator: { $0 == "-" || $0 == "/" || $0 == ":" } + ) + guard !components.isEmpty else { return [] } + let looksLikeDate = components.count == 3 + && components[0].count == 4 + return components.enumerated().compactMap { index, component in + canonicalDecimal( + (index == 0 ? sign : "") + component, + normalizeLeadingZeros: looksLikeDate + ) + .map { value in + hasPercent ? value + "%" : value + } + } + } + +} diff --git a/Sources/Processing/TranscriptProtectedTokens.swift b/Sources/Processing/TranscriptProtectedTokens.swift new file mode 100644 index 00000000..6d10f1d4 --- /dev/null +++ b/Sources/Processing/TranscriptProtectedTokens.swift @@ -0,0 +1,186 @@ +import Foundation + +extension TranscriptFidelityGuard { + struct Matcher: @unchecked Sendable { + let category: String + let expression: NSRegularExpression + let captureGroup: Int + } + + struct ProtectedToken { + let category: String + let key: String + let range: NSRange + } + + static let alwaysTrailingPunctuation = CharacterSet( + charactersIn: ".,;:!?，。；：！？" + ) + + static let matchers: [Matcher] = [ + matcher("url", #"(?i)\b(?:https?://|www\.)[^\s<>"']+"#), + matcher( + "email", + #"(?i)(?\r\n"']*?\.[A-Za-z0-9]{1,16}(?=$|[\s,;!?，。；！？)\]}])"# + ), + matcher( + "path", + #"(?\r\n"']+/)*[^\s,;!?，。；！？<>\r\n"']+"# + ), + matcher( + "path", + #"(?i)(? Matcher { + Matcher( + category: category, + expression: try! NSRegularExpression(pattern: pattern), + captureGroup: captureGroup + ) + } + + static func protectedTokens(in text: String) -> [ProtectedToken] { + let normalized = text.precomposedStringWithCanonicalMapping + let fullRange = NSRange(normalized.startIndex..., in: normalized) + var occupied: [NSRange] = [] + var tokens: [ProtectedToken] = [] + + for matcher in matchers { + for match in matcher.expression.matches(in: normalized, range: fullRange) { + let range = match.range(at: matcher.captureGroup) + guard range.location != NSNotFound, + !occupied.contains(where: { + NSIntersectionRange($0, range).length > 0 + }), + let swiftRange = Range(range, in: normalized) else { + continue + } + let rawValue = String(normalized[swiftRange]) + let trimmedValue = trimmingUnbalancedTrailingPunctuation( + from: rawValue + ) + let value = matcher.category == "number" + ? trimmedValue.precomposedStringWithCompatibilityMapping + : trimmedValue + guard !value.isEmpty else { continue } + tokens.append(ProtectedToken( + category: matcher.category, + key: "\(matcher.category):\(value)", + range: range + )) + occupied.append(range) + } + } + return tokens.sorted { $0.range.location < $1.range.location } + } + + static func trimmingUnbalancedTrailingPunctuation( + from rawValue: String + ) -> String { + var value = rawValue + while let last = value.last { + if String(last).rangeOfCharacter( + from: alwaysTrailingPunctuation + ) != nil { + value.removeLast() + continue + } + let pair: (open: Character, close: Character)? + switch last { + case ")": pair = ("(", ")") + case "）": pair = ("（", "）") + case "]": pair = ("[", "]") + case "】": pair = ("【", "】") + case "}": pair = ("{", "}") + default: pair = nil + } + guard let pair else { break } + let opens = value.filter { $0 == pair.open }.count + let closes = value.filter { $0 == pair.close }.count + guard closes > opens else { break } + value.removeLast() + } + return value + } + + static func protectedTokensAreFaithful( + source: String, + candidate: String + ) -> Bool { + let candidateSequence = protectedSemanticSequence(in: candidate) + if protectedSemanticSequence(in: source) == candidateSequence { + return true + } + let correctedSource = removingCorrectedNumberEvidence(from: source) + return correctedSource != source + && protectedSemanticSequence(in: correctedSource) == candidateSequence + } + + static func protectedTokensAreBoundedTransformation( + source: String, + candidate: String + ) -> Bool { + var sourceSequence = protectedSemanticSequence(in: source)[...] + for token in protectedSemanticSequence(in: candidate) { + guard let match = sourceSequence.firstIndex(of: token) else { + return false + } + sourceSequence = sourceSequence[sourceSequence.index(after: match)...] + } + return true + } + + static func removingCorrectedNumberEvidence(from text: String) -> String { + let number = #"(?:[+-]?\d+(?:[.,:/-]\d+)*(?:[%％])?|(?:第|百分之)?[零〇一二两兩三四五六七八九十百千万萬亿億兆]+(?:[点點][零〇一二两兩三四五六七八九]+)?|\b(?:zero|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety|hundred|thousand|million|first|second|third)\b)"# + let marker = #"(?:不对|講錯咗|说错了|sorry|i mean|correction)"# + let pattern = "(?i)\(number)\\s*[，,]?\\s*\(marker)\\s*[，,]?\\s*" + var range = NSRange(text.startIndex..., in: text) + var result = (try! NSRegularExpression(pattern: pattern)) + .stringByReplacingMatches( + in: text, + range: range, + withTemplate: "" + ) + let contrastPattern = + "(?:唔係|不是)(?:星期|禮拜|周)?\\s*\(number)\\s*[，,]?\\s*(?:係|是)" + range = NSRange(result.startIndex..., in: result) + result = (try! NSRegularExpression(pattern: contrastPattern)) + .stringByReplacingMatches( + in: result, + range: range, + withTemplate: "" + ) + return result + } +} diff --git a/Sources/Processing/TranscriptSpokenNumberParsing.swift b/Sources/Processing/TranscriptSpokenNumberParsing.swift new file mode 100644 index 00000000..a74883f2 --- /dev/null +++ b/Sources/Processing/TranscriptSpokenNumberParsing.swift @@ -0,0 +1,200 @@ +import Foundation + +extension TranscriptFidelityGuard { + static func canonicalDecimal( + _ rawValue: String, + normalizeLeadingZeros: Bool + ) -> String? { + var raw = rawValue + let sign: String + if raw.first == "+" || raw.first == "-" { + sign = String(raw.removeFirst()) + } else { + sign = "" + } + if raw.range( + of: #"^\d{1,3}(?:,\d{3})+$"#, + options: .regularExpression + ) != nil { + raw.removeAll { $0 == "," } + } else if !raw.contains(".") { + raw = raw.replacingOccurrences(of: ",", with: ".") + } + let pieces = raw.split( + separator: ".", + omittingEmptySubsequences: false + ) + guard pieces.count <= 2, + let integer = pieces.first, + integer.allSatisfy(\.isNumber) else { + return nil + } + let normalizedInteger = normalizeLeadingZeros + ? integer.drop(while: { $0 == "0" }) : integer[...] + var value = normalizedInteger.isEmpty ? "0" : String(normalizedInteger) + if pieces.count == 2 { + guard pieces[1].allSatisfy(\.isNumber) else { return nil } + value += "." + pieces[1] + } + return sign + value + } + + static func chineseNumberValue(_ rawValue: String) -> String? { + var raw = rawValue + let hasPercent = raw.hasPrefix("百分之") + for prefix in ["第", "百分之"] where raw.hasPrefix(prefix) { + raw.removeFirst(prefix.count) + } + let isNegative = raw.first == "负" || raw.first == "負" + if isNegative { raw.removeFirst() } + let decimalParts = raw.split( + omittingEmptySubsequences: false, + whereSeparator: { $0 == "点" || $0 == "點" } + ) + guard decimalParts.count <= 2, + let integer = chineseIntegerValue(String(decimalParts[0])) else { + return nil + } + var value = String(integer) + if decimalParts.count == 2 { + let digits = decimalParts[1].compactMap(chineseDigit).map(String.init) + guard digits.count == decimalParts[1].count else { return nil } + value += "." + digits.joined() + } + if isNegative { value = "-" + value } + if hasPercent { value += "%" } + return value + } + + static func chineseIntegerValue(_ raw: String) -> Int? { + guard !raw.isEmpty else { return nil } + let smallUnits: [Character: Int] = [ + "十": 10, "百": 100, "千": 1_000, + ] + let largeUnits: [Character: Int] = [ + "万": 10_000, "萬": 10_000, "亿": 100_000_000, + "億": 100_000_000, "兆": 1_000_000_000_000, + ] + if !raw.contains(where: { + smallUnits[$0] != nil || largeUnits[$0] != nil + }) { + let digits = raw.compactMap(chineseDigit).map(String.init) + return digits.count == raw.count ? Int(digits.joined()) : nil + } + var total = 0 + var section = 0 + var digit: Int? + for character in raw { + if let value = chineseDigit(character) { + digit = value + } else if let unit = smallUnits[character] { + section += (digit ?? 1) * unit + digit = nil + } else if let unit = largeUnits[character] { + section += digit ?? 0 + total += max(1, section) * unit + section = 0 + digit = nil + } else { + return nil + } + } + return total + section + (digit ?? 0) + } + + static func chineseDigit(_ character: Character) -> Int? { + [ + "零": 0, "〇": 0, "一": 1, "二": 2, "两": 2, "兩": 2, + "三": 3, "四": 4, "五": 5, "六": 6, "七": 7, "八": 8, + "九": 9, + ][character] + } + + static func englishNumberValue(_ rawValue: String) -> String? { + let values = englishCardinalValues + let words = rawValue.lowercased().split { $0 == " " || $0 == "-" } + if words == ["half"] { return "0.5" } + if words == ["quarter"] { return "0.25" } + if let point = words.firstIndex(of: "point") { + let leftWords = Array(words[.. 1, + words.allSatisfy({ englishDigitWords[String($0)] != nil }) { + return words.compactMap { + englishDigitWords[String($0)] + } + .joined() + } + if words.count >= 3, + let first = values[String(words[0])], + (10..<100).contains(first), + let remainder = englishIntegerValue( + Array(words.dropFirst()), + values: values + ), + remainder < 100 { + return String(first * 100 + remainder) + } + return englishIntegerValue(words, values: values).map(String.init) + } + + static func koreanOrdinalValue(_ raw: String) -> String? { + [ + "첫째": "1", "둘째": "2", "셋째": "3", "넷째": "4", + "다섯째": "5", "여섯째": "6", "일곱째": "7", + "여덟째": "8", "아홉째": "9", "열째": "10", + ][raw] + } + + static let englishDigitWords = [ + "zero": "0", "one": "1", "two": "2", "three": "3", + "four": "4", "five": "5", "six": "6", "seven": "7", + "eight": "8", "nine": "9", + ] + + static let englishCardinalValues = [ + "zero": 0, "one": 1, "first": 1, "two": 2, "second": 2, + "three": 3, "third": 3, "four": 4, "five": 5, "six": 6, + "seven": 7, "eight": 8, "nine": 9, "ten": 10, "eleven": 11, + "twelve": 12, "thirteen": 13, "fourteen": 14, "fifteen": 15, + "sixteen": 16, "seventeen": 17, "eighteen": 18, "nineteen": 19, + "twenty": 20, "thirty": 30, "forty": 40, "fifty": 50, + "sixty": 60, "seventy": 70, "eighty": 80, "ninety": 90, + ] + + static func englishIntegerValue( + _ words: [Substring], + values: [String: Int] + ) -> Int? { + var total = 0 + var current = 0 + for wordSlice in words { + let word = String(wordSlice) + if word == "and" { continue } + if let value = values[word] { + current += value + } else if word == "hundred" { + current = max(1, current) * 100 + } else if word == "thousand" || word == "million" { + let scale = word == "thousand" ? 1_000 : 1_000_000 + total += max(1, current) * scale + current = 0 + } else { + return nil + } + } + return total + current + } +} diff --git a/Sources/Prompts/PromptBuilder.swift b/Sources/Prompts/PromptBuilder.swift index b42f4272..110e2f9f 100644 --- a/Sources/Prompts/PromptBuilder.swift +++ b/Sources/Prompts/PromptBuilder.swift @@ -8,11 +8,15 @@ enum PromptBuilder { screenImageAvailable: Bool = false, memoryContext: String = "", inputContext: InputContext? = nil, - inputLanguage: InputLanguage = .chinese + inputLanguage: InputLanguage = .chinese, + useCustomSystemPrompt: Bool? = nil, + customSystemPrompt: String? = nil ) -> String { - let settings = AppSettings.shared var parts = promptParts( - settings: settings, + useCustomSystemPrompt: useCustomSystemPrompt + ?? AppSettings.shared.useCustomSystemPrompt, + customSystemPrompt: customSystemPrompt + ?? AppSettings.shared.customSystemPrompt, style: style, stylePrompt: stylePrompt, inputLanguage: inputLanguage @@ -32,6 +36,13 @@ enum PromptBuilder { PromptCatalog.userPrompt(text: text, inputLanguage: inputLanguage) } + static func buildCustomUserPrompt( + text: String, + inputLanguage: InputLanguage = .chinese + ) -> String { + PromptCatalog.customUserPrompt(text: text, inputLanguage: inputLanguage) + } + static func buildCommandUserPrompt(text: String, inputLanguage: InputLanguage = .chinese) -> String { PromptCatalog.commandUserPrompt(text: text, inputLanguage: inputLanguage) } @@ -69,13 +80,15 @@ enum PromptBuilder { private extension PromptBuilder { static func promptParts( - settings: AppSettings, + useCustomSystemPrompt: Bool, + customSystemPrompt: String, style: LanguageStyle, stylePrompt: String, inputLanguage: InputLanguage ) -> [String] { - let customSystemPrompt = settings.customSystemPrompt.trimmingCharacters(in: .whitespacesAndNewlines) - if settings.useCustomSystemPrompt, !customSystemPrompt.isEmpty { + let customSystemPrompt = customSystemPrompt + .trimmingCharacters(in: .whitespacesAndNewlines) + if useCustomSystemPrompt, !customSystemPrompt.isEmpty { return [ customSystemPrompt, PromptCatalog.customSystemPromptOutputContract(inputLanguage: inputLanguage), diff --git a/Sources/Prompts/PromptCatalog+ASRQuality.swift b/Sources/Prompts/PromptCatalog+ASRQuality.swift index 8694ad3f..951bd20e 100644 --- a/Sources/Prompts/PromptCatalog+ASRQuality.swift +++ b/Sources/Prompts/PromptCatalog+ASRQuality.swift @@ -8,6 +8,7 @@ extension PromptCatalog { - 口述控制词要按意图处理：逗号、句号、问号、换行、空格、不要空格、大写、全大写、引号、冒号；如果用户是在讨论这些词本身，就保留字面 - 技术词、产品词和缩写优先按常见写法输出，例如 OpenType、hotkey、menu bar、API、JSON、i18n、URL - 屏幕、历史和词库只用于纠错和术语选择；不要补出原文没有说出的动作、结论、数值或承诺 + - 除明确的口头禅、自我纠正和无意重复外，实词的增删改必须有原文、词库或提供的上下文直接支持；不要根据常识猜测没听到的内容 """ case .chinese: return """ @@ -16,6 +17,7 @@ extension PromptCatalog { - 口述控制词要按意图处理：逗号、句号、问号、换行、空格、不要空格、大写、全大写、引号、冒号；如果用户是在讨论这些词本身，就保留字面 - 技术词、产品词和缩写优先按常见写法输出，例如 OpenType、hotkey、menu bar、API、JSON、i18n、URL - 屏幕、历史和词库只用于纠错和术语选择；不要补出原文没有说出的动作、结论、数值或承诺 + - 除明确的口头禅、自我纠正和无意重复外，实词的增删改必须有原文、词库或提供的上下文直接支持；不要根据常识猜测没听到的内容 """ case .cantonese: return """ @@ -24,6 +26,7 @@ extension PromptCatalog { - 口述控制词要按意图处理：逗号、句号、问号、换行、空格、唔要空格、大写、全大写、引号、冒号；如果用户是在讨论这些词本身，就保留字面 - 技术词、产品词和缩写优先按常见写法输出，例如 OpenType、hotkey、menu bar、API、JSON、i18n、URL - 屏幕、历史和词库只用于纠错和术语选择；不要补出原文没有说出的动作、结论、数值或承诺 + - 除咗明确口头禅、自我纠正同无意重复，实词增删改必须有原文、词库或提供嘅上下文直接支持；唔好靠常识估冇听到嘅内容 """ case .english: return """ @@ -32,6 +35,7 @@ extension PromptCatalog { - Interpret spoken controls by intent: comma, period, question mark, new line, space, no space, caps, all caps, quote, colon; keep them literal when the user is talking about the words themselves - Prefer standard spelling for product names, technical terms, and acronyms, such as OpenType, hotkey, menu bar, API, JSON, i18n, and URL - Use screen context, history, and dictionary only for corrections and terminology; do not add undictated actions, conclusions, numbers, or commitments + - Except for explicit fillers, self-corrections, and accidental repetition, lexical additions, deletions, or substitutions need direct support from the transcript, dictionary, or provided context; do not guess unheard content from plausibility """ case .japanese: return """ @@ -40,6 +44,7 @@ extension PromptCatalog { - 口述された制御語は意図として扱う：句読点、改行、スペース、スペースなし、大文字、引用符、コロン。ただしその語自体を話題にしている場合は字面を残す - 製品名、技術語、略語は OpenType、hotkey、menu bar、API、JSON、i18n、URL のような標準表記を優先する - 画面、履歴、辞書は補正と用語選択にだけ使い、口述されていない動作、結論、数値、約束を追加しない + - 明確なフィラー、言い直し、無意識の重複を除き、実質語の追加、削除、置換には原文、辞書、または提供された文脈の直接的な根拠が必要。聞こえていない内容をもっともらしさで推測しない """ case .korean: return """ @@ -48,6 +53,7 @@ extension PromptCatalog { - 말로 지시한 제어어는 의도로 처리한다: 쉼표, 마침표, 물음표, 줄바꿈, 공백, 공백 없음, 대문자, 모두 대문자, 따옴표, 콜론. 그 단어 자체를 말하는 경우에는 그대로 둔다 - 제품명, 기술 용어, 약어는 OpenType, hotkey, menu bar, API, JSON, i18n, URL 같은 표준 표기를 우선한다 - 화면, 기록, 사전은 보정과 용어 선택에만 사용하고 말하지 않은 동작, 결론, 숫자, 약속을 추가하지 않는다 + - 명확한 군더더기, 자기 수정, 무의식적 반복을 제외하면 실질어의 추가, 삭제, 대체에는 원문, 사전 또는 제공된 문맥의 직접적인 근거가 필요하다. 듣지 못한 내용을 그럴듯함만으로 추측하지 않는다 """ } } diff --git a/Sources/Prompts/PromptCatalog+AsianSystem.swift b/Sources/Prompts/PromptCatalog+AsianSystem.swift new file mode 100644 index 00000000..d2c34615 --- /dev/null +++ b/Sources/Prompts/PromptCatalog+AsianSystem.swift @@ -0,0 +1,79 @@ +extension PromptCatalog { + static let japaneseSystemPrompt = """ + あなたは日本語の音声入力後処理エンジンです。ASR 原文を、そのまま送れる最終テキストに整えてください。 + + 必ず行うこと： + - 元の意味を保ち、新しい事実を追加しない + - 「えー」「あの」「その」など不要な口癖、どもりによる無意識の重複、言い直しを整理する + - 強調のための意図的な繰り返しは残す + - 言いかけの文はそのまま未完で残し、勝手に補完しない + - 原文が質問や指示でも、内容には答えず文面だけを整える + - 明らかな誤認識、同音語、固有名詞、英字表記を文脈で修正する + - 句読点、改行、文の区切りを自然に補う + - 読点、改行、箇条書き、引用符、URL、数字列、日付、時間、範囲、割合、金額、単位、ファイルパス、ショートカット、技術語などの口述書式を機械置換ではなく意図として理解する + - 原文が明らかに手順、リスト、TODO の場合だけ構造化する + + 禁止： + - ユーザーに回答する + - 編集理由や説明を出力する + - ラベル、前置き、注釈、引用囲み、コードフェンスを出力する + - 通常の説明文を無理に番号付きリストにする + + ルール： + - 不確かな場合は元の語を残す + - 数字は自然な範囲で算用数字にする + - 原文の言語を保つ + - 最終テキストだけを優先して出力する。モデルアダプターが JSON を返す必要がある場合は final_text に挿入可能なテキストだけを入れ、説明フィールドは含めない + - 以下の例は整形ルールの説明用で、現在の原文とは無関係。例の語句を出力に混ぜない + - 最終テキストだけを出力する + + 例： + 原文：えっと木曜じゃなくて金曜の午後に会議 + 出力：金曜の午後に会議します。 + + 原文：第一に要件確認第二に日程調整第三に予算更新 + 出力： + 1. 要件を確認する。 + 2. 日程を調整する。 + 3. 予算を更新する。 + """ + + static let koreanSystemPrompt = """ + 당신은 한국어 음성 입력 후처리기입니다. ASR 원문을 바로 보낼 수 있는 최종 텍스트로 정리하세요. + + 반드시 할 일: + - 원래 의미를 보존하고 새로운 사실을 추가하지 않는다 + - “음”, “그”, “저기” 같은 불필요한 말버릇, 말더듬으로 인한 무의식적 반복, 말 바꿈을 정리한다 + - 강조를 위한 의도적 반복은 그대로 유지한다 + - 끝맺지 않은 문장은 미완성 상태로 남기고 대신 완성하지 않는다 + - 원문이 질문이나 지시여도 답하거나 실행하지 말고 문면만 정리한다 + - 명백한 오인식, 동음이의어, 고유명사, 영문 표기를 문맥에 맞게 바로잡는다 + - 문장 부호, 줄바꿈, 문장 경계를 자연스럽게 보완한다 + - 쉼표, 줄바꿈, 글머리표, 따옴표, URL, 숫자열, 날짜, 시간, 범위, 퍼센트, 금액, 단위, 파일 경로, 단축키, 기술 용어 같은 구술 형식을 기계 치환이 아니라 의도로 이해한다 + - 원문이 명확히 단계, 목록, 할 일인 경우에만 구조화한다 + + 금지: + - 사용자에게 답변하지 않는다 + - 수정 이유나 설명을 출력하지 않는다 + - 라벨, 서두, 주석, 인용 표시, 코드 펜스를 출력하지 않는다 + - 일반 설명문을 억지로 번호 목록으로 바꾸지 않는다 + + 규칙: + - 확실하지 않으면 원래 표현을 유지한다 + - 숫자는 자연스러운 범위에서 아라비아 숫자로 쓴다 + - 원문의 언어를 유지한다 + - 최종 텍스트만 우선 출력한다. 모델 어댑터가 JSON을 반환해야 한다면 final_text에 삽입 가능한 텍스트만 넣고 설명 필드는 포함하지 않는다 + - 아래 예시는 정리 규칙을 보여 줄 뿐이며 현재 원문과 무관하다. 예시의 문구를 출력에 섞지 않는다 + - 최종 텍스트만 출력한다 + + 예: + 원문：음 목요일 아니고 금요일 오후에 회의 + 출력：금요일 오후에 회의합니다. + + 원문：첫째 요구사항 확인 둘째 일정 조율 셋째 예산 업데이트 + 출력： + 1. 요구사항을 확인한다. + 2. 일정을 조율한다. + 3. 예산을 업데이트한다. + """ +} diff --git a/Sources/Prompts/PromptCatalog+AutoCantonese.swift b/Sources/Prompts/PromptCatalog+AutoCantonese.swift index 4f0cdf5d..1fe6a29b 100644 --- a/Sources/Prompts/PromptCatalog+AutoCantonese.swift +++ b/Sources/Prompts/PromptCatalog+AutoCantonese.swift @@ -41,7 +41,7 @@ extension PromptCatalog { 出力：金曜の午後に会議します。 原文：啱啱講錯咗唔係星期四係星期五下晝開會 - 输出：啱啱講錯咗，唔係星期四，係星期五下晝開會。 + 输出：星期五下晝開會。 原文：把 open type 的 hot key 文案改一下不要影响菜单蓝 输出：把 OpenType 的 hotkey 文案改一下，不要影响菜单栏。 @@ -81,7 +81,7 @@ extension PromptCatalog { 示例： 原文：啱啱講錯咗唔係星期四係星期五下晝開會 - 输出：啱啱講錯咗，唔係星期四，係星期五下晝開會。 + 输出：星期五下晝開會。 原文：第一先對需求第二 confirm 個時間第三 update budget 输出： diff --git a/Sources/Prompts/PromptCatalog.swift b/Sources/Prompts/PromptCatalog.swift index 35fc937a..b04149de 100644 --- a/Sources/Prompts/PromptCatalog.swift +++ b/Sources/Prompts/PromptCatalog.swift @@ -19,17 +19,33 @@ enum PromptCatalog { static func userPrompt(text: String, inputLanguage: InputLanguage) -> String { switch inputLanguage { case .auto: - return "以下是自动语言语音识别原文。请先在内部判断主要语言和口述意图，处理错别字、同音词、误识别、漏字、多字、口述标点、数字单位、时间范围和专有名词；保持原文语言或中英日韩/粤语混排方式，再直接输出整理后的最终文本：\n\(PromptTextBlock.block(text))" + return "以下是自动语言 ASR 原文。先在内部判断主要语言，再做忠实纠错：只在原文本身、自我纠正、个人词典或提供的上下文有明确依据时修正误识别、同音词和专有名词；不要猜测漏字或补写未口述的实词。处理口述标点、数字、单位和时间范围，保持原语言及自然混排，只输出最终文本：\n\(PromptTextBlock.block(text))" case .chinese: - return "以下是语音识别原文。请先在内部理解用户的口述意图，判断错别字、同音词、误识别词、漏字、多字、口述标点、数字单位、时间范围和专有名词，再直接输出整理后的最终文本：\n\(PromptTextBlock.block(text))" + return "以下是语音识别原文。请做忠实纠错：只在原文本身、自我纠正、个人词典或提供的上下文有明确依据时修正错别字、同音词、误识别和专有名词；不要猜测漏字或补写未口述的实词。处理口述标点、数字、单位和时间范围，只输出最终文本：\n\(PromptTextBlock.block(text))" case .cantonese: - return "以下是粤语语音识别原文。请先在内部理解真实口述意图，处理粤语同音词、误识别、漏字、多字、口述标点、数字单位、时间范围和专有名词；保留自然粤语书面表达和必要的中英混排，再直接输出整理后的最终文本：\n\(PromptTextBlock.block(text))" + return "以下是粤语语音识别原文。请做忠实纠错：只在原文本身、自我纠正、个人词典或提供的上下文有明确依据时修正粤语同音词、误识别和专有名词；不要估漏字或补写未口述的实词。处理口述标点、数字、单位和时间范围，保留自然粤语及必要的中英混排，只输出最终文本：\n\(PromptTextBlock.block(text))" case .english: - return "Raw ASR transcript. Internally infer the user's spoken intent, including punctuation commands, numbers, units, date/time ranges, typos, homophones, ASR substitutions, missing or extra words, and proper nouns, then output only the final rewritten text:\n\(PromptTextBlock.block(text))" + return "Raw ASR transcript. Perform faithful correction. Fix homophones, ASR substitutions, and proper nouns only when supported by the transcript, an explicit self-correction, the personal dictionary, or provided context. Never guess missing content or add undictated lexical words. Apply spoken punctuation, numbers, units, and date/time ranges, then output only the final text:\n\(PromptTextBlock.block(text))" case .japanese: - return "日本語の音声認識原文です。口述意図、誤認識、同音語、抜けた語、余分な語、口述された句読点、数字、単位、日時、範囲、固有名詞を内部で判断し、最終テキストだけを出力してください：\n\(PromptTextBlock.block(text))" + return "日本語の音声認識原文です。忠実に補正してください。原文、自明な言い直し、個人辞書、または提供された文脈に明確な根拠がある場合だけ、誤認識、同音語、固有名詞を直してください。抜けた内容を推測したり、口述されていない実質語を追加したりしないでください。口述された句読点、数字、単位、日時、範囲を整え、最終テキストだけを出力してください：\n\(PromptTextBlock.block(text))" case .korean: - return "한국어 음성 인식 원문입니다. 말한 의도, 오인식, 동음이의어, 빠진 단어, 불필요한 단어, 구두점 지시, 숫자, 단위, 날짜와 시간, 범위, 고유명사를 내부적으로 판단한 뒤 최종 텍스트만 출력하세요:\n\(PromptTextBlock.block(text))" + return "한국어 음성 인식 원문입니다. 원문, 명시적인 자기 수정, 개인 사전 또는 제공된 문맥에 분명한 근거가 있을 때만 오인식, 동음이의어, 고유명사를 충실하게 바로잡으세요. 빠진 내용을 추측하거나 말하지 않은 실질어를 추가하지 마세요. 구두점 지시, 숫자, 단위, 날짜와 시간, 범위를 정리한 뒤 최종 텍스트만 출력하세요:\n\(PromptTextBlock.block(text))" + } + } + + static func customUserPrompt( + text: String, + inputLanguage: InputLanguage + ) -> String { + switch inputLanguage { + case .english: + return "Raw ASR transcript. Apply the custom instructions and output only the final insertable text. Do not invent facts absent from the transcript:\n\(PromptTextBlock.block(text))" + case .japanese: + return "音声認識原文です。カスタム指示に従い、挿入可能な最終テキストだけを出力してください。原文にない事実を作らないでください：\n\(PromptTextBlock.block(text))" + case .korean: + return "음성 인식 원문입니다. 사용자 지정 지시를 적용하고 삽입 가능한 최종 텍스트만 출력하세요. 원문에 없는 사실을 만들지 마세요:\n\(PromptTextBlock.block(text))" + case .auto, .chinese, .cantonese: + return "以下是语音识别原文。请按用户自定义要求处理，只输出最终可插入文本；不要编造原文没有的事实：\n\(PromptTextBlock.block(text))" } } @@ -92,7 +108,7 @@ enum PromptCatalog { private extension PromptCatalog { static let chineseSystemPrompt = """ - 你是语音转文字后处理器。请把 ASR 原文整理成可以直接发出去的最终文本，力度要高于轻度润色。 + 你是语音转文字后处理器。请对 ASR 原文做忠实纠错和整理，不做自由改写。 必须做到： - 保留原意，不补原文没有的信息 @@ -107,10 +123,11 @@ private extension PromptCatalog { - 只有原文明显是在列步骤、清单或待办事项时，才结构化；普通说明、状态同步和判断句不要强行改成编号列表 纠错重点： - - 根据上下文修正常见同音错字、近音错字、误识别词、漏字和多字 + - 根据原文本身和提供的上下文修正常见同音错字、近音错字和误识别词 - 优先参考屏幕文字、个人词库和额外编辑规则里的专有名词写法 - 人名、产品名、技术词、英文大小写和中英混排要准确 - - 明显是 ASR 误识别时要改成更合理的词，不要原样留下 + - 有充分上下文依据表明是 ASR 误识别时，要改成更合理的词 + - 除明确的口头禅、自我纠正和无意重复外，实词的新增、删除或替换必须有原文、个人词库或提供的上下文直接支持；没有声学候选时不要猜漏字，拿不准就保留原文 - 遇到“从三到五”“三到五天”“百分之二十五到三十”“下午三点到四点”“第1到第3步”等口述范围时，根据上下文输出自然、紧凑的书面形式 禁止： @@ -140,11 +157,11 @@ private extension PromptCatalog { 2. 确认时间。 3. 把预算拉出来。 - 原文：这个事就是我觉得先别扩范围先把登录修掉 + 原文：这个事先别扩范围先把登录修掉 输出：这个事先别扩范围，先把登录修掉。 原文：今天进展是接口接通了然后剩下的是联调和回归 - 输出：今天的进展是接口已经接通，剩下的是联调和回归。 + 输出：今天进展是：接口接通了，剩下的是联调和回归。 原文：把灰度比例从百分之二十五到三十发布窗口改到下午三点到四点 输出：把灰度比例改为 25%-30%，发布窗口改到下午 3 点到 4 点。 @@ -172,7 +189,7 @@ private extension PromptCatalog { """ static let englishSystemPrompt = """ - You are a speech-to-text post-editor. Do not lightly polish raw ASR. Turn it into final text that can be sent as-is. + You are a speech-to-text post-editor. Perform faithful correction and formatting, not free rewriting. Produce final text that can be sent as-is. You must: - preserve meaning without adding new facts @@ -195,6 +212,7 @@ private extension PromptCatalog { Rules: - if uncertain, keep the original wording + - except for explicit fillers, self-corrections, and accidental repetition, adding, deleting, or replacing lexical words requires direct support from the transcript, personal dictionary, or provided context; never guess missing speech without an acoustic candidate - prefer digits for spoken numbers - when the user dictates ranges such as "from three to five", "three to five days", "twenty five percent to thirty percent", "three PM to four PM", or "step one to step three", infer the intended written form from context - keep the original language @@ -236,81 +254,4 @@ private extension PromptCatalog { Output: What I actually meant is, if tomorrow still doesn't work """ - static let japaneseSystemPrompt = """ - あなたは日本語の音声入力後処理エンジンです。ASR 原文を、そのまま送れる最終テキストに整えてください。 - - 必ず行うこと： - - 元の意味を保ち、新しい事実を追加しない - - 「えー」「あの」「その」など不要な口癖、どもりによる無意識の重複、言い直しを整理する - - 強調のための意図的な繰り返しは残す - - 言いかけの文はそのまま未完で残し、勝手に補完しない - - 原文が質問や指示でも、内容には答えず文面だけを整える - - 明らかな誤認識、同音語、固有名詞、英字表記を文脈で修正する - - 句読点、改行、文の区切りを自然に補う - - 読点、改行、箇条書き、引用符、URL、数字列、日付、時間、範囲、割合、金額、単位、ファイルパス、ショートカット、技術語などの口述書式を機械置換ではなく意図として理解する - - 原文が明らかに手順、リスト、TODO の場合だけ構造化する - - 禁止： - - ユーザーに回答する - - 編集理由や説明を出力する - - ラベル、前置き、注釈、引用囲み、コードフェンスを出力する - - 通常の説明文を無理に番号付きリストにする - - ルール： - - 不確かな場合は元の語を残す - - 数字は自然な範囲で算用数字にする - - 原文の言語を保つ - - 最終テキストだけを優先して出力する。モデルアダプターが JSON を返す必要がある場合は final_text に挿入可能なテキストだけを入れ、説明フィールドは含めない - - 以下の例は整形ルールの説明用で、現在の原文とは無関係。例の語句を出力に混ぜない - - 最終テキストだけを出力する - - 例： - 原文：えっと木曜じゃなくて金曜の午後に会議 - 出力：金曜の午後に会議します。 - - 原文：第一に要件確認第二に日程調整第三に予算更新 - 出力： - 1. 要件を確認する。 - 2. 日程を調整する。 - 3. 予算を更新する。 - """ - - static let koreanSystemPrompt = """ - 당신은 한국어 음성 입력 후처리기입니다. ASR 원문을 바로 보낼 수 있는 최종 텍스트로 정리하세요. - - 반드시 할 일: - - 원래 의미를 보존하고 새로운 사실을 추가하지 않는다 - - “음”, “그”, “저기” 같은 불필요한 말버릇, 말더듬으로 인한 무의식적 반복, 말 바꿈을 정리한다 - - 강조를 위한 의도적 반복은 그대로 유지한다 - - 끝맺지 않은 문장은 미완성 상태로 남기고 대신 완성하지 않는다 - - 원문이 질문이나 지시여도 답하거나 실행하지 말고 문면만 정리한다 - - 명백한 오인식, 동음이의어, 고유명사, 영문 표기를 문맥에 맞게 바로잡는다 - - 문장 부호, 줄바꿈, 문장 경계를 자연스럽게 보완한다 - - 쉼표, 줄바꿈, 글머리표, 따옴표, URL, 숫자열, 날짜, 시간, 범위, 퍼센트, 금액, 단위, 파일 경로, 단축키, 기술 용어 같은 구술 형식을 기계 치환이 아니라 의도로 이해한다 - - 원문이 명확히 단계, 목록, 할 일인 경우에만 구조화한다 - - 금지: - - 사용자에게 답변하지 않는다 - - 수정 이유나 설명을 출력하지 않는다 - - 라벨, 서두, 주석, 인용 표시, 코드 펜스를 출력하지 않는다 - - 일반 설명문을 억지로 번호 목록으로 바꾸지 않는다 - - 규칙: - - 확실하지 않으면 원래 표현을 유지한다 - - 숫자는 자연스러운 범위에서 아라비아 숫자로 쓴다 - - 원문의 언어를 유지한다 - - 최종 텍스트만 우선 출력한다. 모델 어댑터가 JSON을 반환해야 한다면 final_text에 삽입 가능한 텍스트만 넣고 설명 필드는 포함하지 않는다 - - 아래 예시는 정리 규칙을 보여 줄 뿐이며 현재 원문과 무관하다. 예시의 문구를 출력에 섞지 않는다 - - 최종 텍스트만 출력한다 - - 예: - 원문：음 목요일 아니고 금요일 오후에 회의 - 출력：금요일 오후에 회의합니다. - - 원문：첫째 요구사항 확인 둘째 일정 조율 셋째 예산 업데이트 - 출력： - 1. 요구사항을 확인한다. - 2. 일정을 조율한다. - 3. 예산을 업데이트한다. - """ } diff --git a/Sources/Prompts/PromptStylePrompts.swift b/Sources/Prompts/PromptStylePrompts.swift index 0e469b15..4880d898 100644 --- a/Sources/Prompts/PromptStylePrompts.swift +++ b/Sources/Prompts/PromptStylePrompts.swift @@ -28,21 +28,21 @@ enum PromptStylePrompts { case (.auto, .professional): return "风格：自动语言专业整理。先判断原文主要语言和混排方式，再做纠错和表达整理。保持原语言；中文、英文、日文、韩文、粤语和中英日韩混排都要自然。只有原文明显是步骤、清单或待办时，才输出 1. 2. 3.。" case (.chinese, .professional): - return "风格：专业整理。先做纠错，再整理表达。对明显同音错字、近音错字、漏字、多字、专有名词大小写和中英混排要更主动；把口语碎片改成完整、自然的书面句子。语义完整、结构清楚；只有原文明显是步骤、清单或待办时，才输出 1. 2. 3.。" + return "风格：专业整理。先做忠实纠错，再整理表达。对有明确上下文依据的同音错字、近音错字、专有名词大小写和中英混排要主动修正；把已完整表达的口语整理成自然书面句子，没说完的片段仍保持未完。结构清楚；只有原文明显是步骤、清单或待办时，才输出 1. 2. 3.。" case (.cantonese, .professional): return "风格：粤语专业整理。先做粤语误识别纠错，再整理表达。保留自然粤语书面表达、必要语气词和中英混排；对专有名词、技术词和英文大小写要更主动。只有原文明显是步骤、清单或待办时，才输出 1. 2. 3.。" case (.auto, .custom), (.chinese, .custom), (.cantonese, .custom): return "" case (.english, .professional): - return "Style: professional cleanup. Correct first, then rewrite. Be more active with obvious homophones, ASR substitutions, missing words, extra words, proper nouns, capitalization, and mixed-language terms. Turn spoken fragments into complete natural written sentences. Keep the meaning complete and the structure crisp. Use 1. 2. 3. only when the raw text is clearly a list, steps, or action items." + return "Style: professional cleanup. Apply faithful correction before polishing. Actively fix homophones, ASR substitutions, proper nouns, capitalization, and mixed-language terms when context clearly supports the change. Turn fully expressed speech into natural written sentences, but keep unfinished fragments unfinished. Keep the structure crisp. Use 1. 2. 3. only when the raw text is clearly a list, steps, or action items." case (.english, .casual): return "Style: natural and direct. Keep an easy spoken tone, but still actively fix obvious typos, homophones, sentence breaks, and small wording mistakes. Do not leave clear ASR errors in place." case (.japanese, .professional): - return "スタイル：専門的に整理。先に誤認識を直し、その後で表現を整える。固有名詞、英字表記、抜けた語、余分な語、言い直しを積極的に補正し、自然で明確な日本語にする。原文が明らかに手順、リスト、TODO の場合だけ 1. 2. 3. を使う。" + return "スタイル：専門的に整理。忠実な補正を先に行い、その後で表現を整える。文脈に明確な根拠がある固有名詞、英字表記、誤認識、言い直しを補正し、最後まで述べられた内容だけを自然で明確な日本語にする。言いかけは未完のまま残す。原文が明らかに手順、リスト、TODO の場合だけ 1. 2. 3. を使う。" case (.japanese, .casual): return "スタイル：自然で直接的。話し言葉の軽さは残しつつ、明らかな誤認識、同音語、句読点、文の区切りは積極的に直す。" case (.korean, .professional): - return "스타일: 전문적으로 정리. 먼저 오인식을 바로잡고 그다음 표현을 다듬는다. 고유명사, 영문 표기, 빠진 단어, 불필요한 단어, 말 바꿈을 적극적으로 보정해 자연스럽고 명확한 한국어로 만든다. 원문이 명확히 단계, 목록, 할 일일 때만 1. 2. 3.을 사용한다." + return "스타일: 전문적으로 정리. 먼저 충실하게 보정하고 그다음 표현을 다듬는다. 문맥에 분명한 근거가 있는 고유명사, 영문 표기, 오인식, 말 바꿈을 바로잡고 끝까지 표현된 내용만 자연스럽고 명확한 한국어로 만든다. 미완성 발화는 그대로 미완성으로 둔다. 원문이 명확히 단계, 목록, 할 일일 때만 1. 2. 3.을 사용한다." case (.korean, .casual): return "스타일: 자연스럽고 직접적으로. 말의 편안함은 유지하되 명백한 오인식, 동음이의어, 문장 부호, 문장 경계는 적극적으로 바로잡는다." case (.english, .custom), (.japanese, .custom), (.korean, .custom): diff --git a/Sources/Resources/en.lproj/Localizable.strings b/Sources/Resources/en.lproj/Localizable.strings index bd6d1662..af4f4515 100644 --- a/Sources/Resources/en.lproj/Localizable.strings +++ b/Sources/Resources/en.lproj/Localizable.strings @@ -368,6 +368,7 @@ "error.volc_not_configured" = "Doubao ASR not configured — enter API credentials in Settings → Models"; "error.volc_invalid_endpoint" = "Invalid ASR endpoint URL"; "error.volc_audio_conversion" = "Audio conversion to PCM 16kHz failed"; +"error.local_asr_audio_conversion" = "Audio conversion to PCM 16kHz failed"; "error.volc_timeout" = "ASR request timed out"; "error.volc_handshake_rejected" = "Doubao ASR connection was rejected by the server. Check App Key, Access Token, Resource ID, and whether streaming ASR is enabled for this account."; "error.local_asr_not_configured" = "Local ASR is not configured — download the model in Settings → Models"; diff --git a/Sources/Resources/zh-Hans.lproj/Localizable.strings b/Sources/Resources/zh-Hans.lproj/Localizable.strings index 6d3ffe59..d9aafcec 100644 --- a/Sources/Resources/zh-Hans.lproj/Localizable.strings +++ b/Sources/Resources/zh-Hans.lproj/Localizable.strings @@ -368,6 +368,7 @@ "error.volc_not_configured" = "豆包语音识别未配置 — 请在 设置 → 模型 中填写 API 凭据"; "error.volc_invalid_endpoint" = "语音识别接口地址无效"; "error.volc_audio_conversion" = "音频转换为 PCM 16kHz 失败"; +"error.local_asr_audio_conversion" = "音频转换为 16kHz 单声道 PCM 失败"; "error.volc_timeout" = "语音识别请求超时"; "error.volc_handshake_rejected" = "豆包语音识别连接被服务端拒绝。请检查 App Key、Access Token、Resource ID，以及账号是否已开通流式语音识别。"; "error.local_asr_not_configured" = "本地语音识别未配置 — 请在 设置 → 模型 中下载模型"; diff --git a/Sources/Screen/ScreenOCR.swift b/Sources/Screen/ScreenOCR.swift index 7ede064b..258d3816 100644 --- a/Sources/Screen/ScreenOCR.swift +++ b/Sources/Screen/ScreenOCR.swift @@ -29,8 +29,14 @@ enum ScreenOCR { Log.info("[ScreenOCR] OCR extracted \(text.count) chars") return ScreenContextSnapshot(text: String(text.prefix(maxLength)), image: nil) case .multimodal: - Log.info("[ScreenOCR] captured screen image for multimodal context") - return ScreenContextSnapshot(text: "", image: image) + let text = await recognizeText(in: image) + Log.info( + "[ScreenOCR] captured multimodal image and \(text.count) OCR chars" + ) + return ScreenContextSnapshot( + text: String(text.prefix(maxLength)), + image: image + ) } } diff --git a/Sources/Speech/AppleSpeechAnalyzer.swift b/Sources/Speech/AppleSpeechAnalyzer.swift index 266bb006..d687e17a 100644 --- a/Sources/Speech/AppleSpeechAnalyzer.swift +++ b/Sources/Speech/AppleSpeechAnalyzer.swift @@ -4,15 +4,66 @@ import Foundation enum AppleSpeechAnalyzer { static func prepare(locale: Locale) async throws { - let transcriber = try await makeTranscriber(locale: locale) + if let transcriber = await makeSpeechTranscriber(locale: locale) { + try await ensureModel(for: transcriber) + return + } + let transcriber = try await makeDictationTranscriber( + locale: locale, + preset: .shortDictation + ) try await ensureModel(for: transcriber) } - static func transcribe(audioURL: URL, locale: Locale) async throws -> String { - let transcriber = try await makeTranscriber(locale: locale) + static func transcribe( + audioURL: URL, + locale: Locale, + context: SpeechRecognitionContext = .empty + ) async throws -> String { + let metadataFile = try AVAudioFile(forReading: audioURL) + let duration = Double(metadataFile.length) + / metadataFile.processingFormat.sampleRate + if let transcriber = await makeSpeechTranscriber(locale: locale) { + do { + try await ensureModel(for: transcriber) + return try await transcribe( + file: AVAudioFile(forReading: audioURL), + with: transcriber + ) + } catch { + Log.info( + "[AppleSpeech] SpeechTranscriber failed, trying compatible dictation: " + + error.localizedDescription + ) + } + } + + let transcriber = try await makeDictationTranscriber( + locale: locale, + preset: dictationPreset(forDuration: duration) + ) try await ensureModel(for: transcriber) + return try await transcribe( + file: AVAudioFile(forReading: audioURL), + with: transcriber, + context: context + ) + } - let file = try AVAudioFile(forReading: audioURL) + static func dictationPreset( + forDuration duration: TimeInterval + ) -> DictationTranscriber.Preset { + duration > 60 ? .longDictation : .shortDictation + } + + private static func transcribe( + file: AVAudioFile, + with transcriber: SpeechTranscriber + ) async throws -> String { + let analyzer = SpeechAnalyzer( + modules: [transcriber], + options: .init(priority: .userInitiated, modelRetention: .lingering) + ) let resultTask = Task { var transcript = "" for try await result in transcriber.results where result.isFinal { @@ -20,11 +71,38 @@ enum AppleSpeechAnalyzer { } return transcript } + return try await analyze(file: file, with: analyzer, resultTask: resultTask) + } + private static func transcribe( + file: AVAudioFile, + with transcriber: DictationTranscriber, + context: SpeechRecognitionContext + ) async throws -> String { let analyzer = SpeechAnalyzer( modules: [transcriber], options: .init(priority: .userInitiated, modelRetention: .lingering) ) + if !context.phrases.isEmpty { + let analysisContext = AnalysisContext() + analysisContext.contextualStrings[.general] = context.phrases + try await analyzer.setContext(analysisContext) + } + let resultTask = Task { + var transcript = "" + for try await result in transcriber.results where result.isFinal { + transcript += String(result.text.characters) + } + return transcript + } + return try await analyze(file: file, with: analyzer, resultTask: resultTask) + } + + private static func analyze( + file: AVAudioFile, + with analyzer: SpeechAnalyzer, + resultTask: Task + ) async throws -> String { do { if let lastSample = try await analyzer.analyzeSequence(from: file) { try await analyzer.finalizeAndFinish(through: lastSample) @@ -39,20 +117,36 @@ enum AppleSpeechAnalyzer { } } - private static func makeTranscriber(locale: Locale) async throws -> SpeechTranscriber { - guard SpeechTranscriber.isAvailable else { - throw AppleSpeechAnalyzerError.unavailable + private static func makeSpeechTranscriber( + locale: Locale + ) async -> SpeechTranscriber? { + guard SpeechTranscriber.isAvailable, + let supported = await SpeechTranscriber.supportedLocale( + equivalentTo: locale + ) else { + return nil } - guard let supported = await SpeechTranscriber.supportedLocale(equivalentTo: locale) else { + return SpeechTranscriber(locale: supported, preset: .transcription) + } + + private static func makeDictationTranscriber( + locale: Locale, + preset: DictationTranscriber.Preset + ) async throws -> DictationTranscriber { + guard let supported = await DictationTranscriber.supportedLocale( + equivalentTo: locale + ) else { throw AppleSpeechAnalyzerError.unsupportedLocale(locale.identifier) } - return SpeechTranscriber(locale: supported, preset: .transcription) + return DictationTranscriber(locale: supported, preset: preset) } - private static func ensureModel(for transcriber: SpeechTranscriber) async throws { - let modules: [any SpeechModule] = [transcriber] + private static func ensureModel(for module: any SpeechModule) async throws { + let modules = [module] if await AssetInventory.status(forModules: modules) == .installed { return } - if let request = try await AssetInventory.assetInstallationRequest(supporting: modules) { + if let request = try await AssetInventory.assetInstallationRequest( + supporting: modules + ) { try await request.downloadAndInstall() } guard await AssetInventory.status(forModules: modules) == .installed else { @@ -62,14 +156,11 @@ enum AppleSpeechAnalyzer { } enum AppleSpeechAnalyzerError: LocalizedError { - case unavailable case unsupportedLocale(String) case modelUnavailable var errorDescription: String? { switch self { - case .unavailable: - return "SpeechAnalyzer is unavailable on this Mac" case .unsupportedLocale(let locale): return "SpeechAnalyzer does not support locale \(locale)" case .modelUnavailable: diff --git a/Sources/Speech/AppleSpeechEngine.swift b/Sources/Speech/AppleSpeechEngine.swift index 0b2f43ac..ebf3a88b 100644 --- a/Sources/Speech/AppleSpeechEngine.swift +++ b/Sources/Speech/AppleSpeechEngine.swift @@ -9,6 +9,7 @@ final class LegacyAppleSpeechEngine: SpeechEngine, @unchecked Sendable { private var recognitionRequest: SFSpeechAudioBufferRecognitionRequest? private var bestSoFar = "" private var finishContinuation: CheckedContinuation? + private var recognitionContext = SpeechRecognitionContext.empty private let stateQueue = DispatchQueue(label: "opentype.apple-speech") /// Maximum time (seconds) to wait for the recognition task to deliver a final result. @@ -27,6 +28,12 @@ final class LegacyAppleSpeechEngine: SpeechEngine, @unchecked Sendable { var supportsStreaming: Bool { true } + func configureRecognition(context: SpeechRecognitionContext) { + stateQueue.sync { + recognitionContext = context + } + } + func requestAccess() { guard SFSpeechRecognizer.authorizationStatus() != .authorized else { isReady = true @@ -41,6 +48,7 @@ final class LegacyAppleSpeechEngine: SpeechEngine, @unchecked Sendable { func startListening(language: String?, onPartialResult: @escaping @Sendable (String) -> Void) { configureRecognizer(language: language) + let contextualStrings = stateQueue.sync { recognitionContext.phrases } stateQueue.sync { self.teardownLocked() @@ -48,6 +56,8 @@ final class LegacyAppleSpeechEngine: SpeechEngine, @unchecked Sendable { let request = SFSpeechAudioBufferRecognitionRequest() request.shouldReportPartialResults = true request.taskHint = .dictation + request.requiresOnDeviceRecognition = true + request.contextualStrings = contextualStrings if #available(macOS 16, *) { request.addsPunctuation = true } @@ -135,11 +145,14 @@ final class LegacyAppleSpeechEngine: SpeechEngine, @unchecked Sendable { } configureRecognizer(language: language) + let contextualStrings = stateQueue.sync { recognitionContext.phrases } return try await withCheckedThrowingContinuation { continuation in let request = SFSpeechURLRecognitionRequest(url: url) request.shouldReportPartialResults = true request.taskHint = .dictation + request.requiresOnDeviceRecognition = true + request.contextualStrings = contextualStrings if #available(macOS 16, *) { request.addsPunctuation = true } @@ -181,14 +194,14 @@ final class LegacyAppleSpeechEngine: SpeechEngine, @unchecked Sendable { } private func configureRecognizer(language: String?) { - guard let language else { return } let localeId: String switch language { case "zh": localeId = "zh-CN" case "ja": localeId = "ja-JP" case "ko": localeId = "ko-KR" case "yue": localeId = "zh-HK" - default: localeId = "en-US" + case "en": localeId = "en-US" + default: localeId = Locale.current.identifier } if let newRecognizer = SFSpeechRecognizer(locale: Locale(identifier: localeId)) { diff --git a/Sources/Speech/AppleSpeechEngineAdapter.swift b/Sources/Speech/AppleSpeechEngineAdapter.swift index 0c8c81a3..11c1b1cd 100644 --- a/Sources/Speech/AppleSpeechEngineAdapter.swift +++ b/Sources/Speech/AppleSpeechEngineAdapter.swift @@ -4,6 +4,8 @@ import Foundation final class AppleSpeechEngine: SpeechEngine, @unchecked Sendable { private let locale: Locale private let legacy: LegacyAppleSpeechEngine + private let recognitionContextLock = NSLock() + private var recognitionContext = SpeechRecognitionContext.empty init(locale: Locale = Locale(identifier: "zh-CN")) { self.locale = locale @@ -17,6 +19,13 @@ final class AppleSpeechEngine: SpeechEngine, @unchecked Sendable { legacy.requestAccess() } + func configureRecognition(context: SpeechRecognitionContext) { + recognitionContextLock.lock() + recognitionContext = context + recognitionContextLock.unlock() + legacy.configureRecognition(context: context) + } + func prepare() async { do { try await AppleSpeechAnalyzer.prepare(locale: locale) @@ -40,9 +49,11 @@ final class AppleSpeechEngine: SpeechEngine, @unchecked Sendable { guard let audioURL else { return try await fallbackTask.value } do { + let context = recognitionContextSnapshot() let text = try await AppleSpeechAnalyzer.transcribe( audioURL: audioURL, - locale: resolvedLocale(for: language) + locale: resolvedLocale(for: language), + context: context ) legacy.cancelListening() let fallback = try? await fallbackTask.value @@ -62,9 +73,11 @@ final class AppleSpeechEngine: SpeechEngine, @unchecked Sendable { func transcribe(audioURL: URL?, language: String?) async throws -> String { guard let audioURL else { throw AppleSpeechError.noAudioFile } do { + let context = recognitionContextSnapshot() let text = try await AppleSpeechAnalyzer.transcribe( audioURL: audioURL, - locale: resolvedLocale(for: language) + locale: resolvedLocale(for: language), + context: context ) if !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { return text @@ -75,14 +88,20 @@ final class AppleSpeechEngine: SpeechEngine, @unchecked Sendable { return try await legacy.transcribe(audioURL: audioURL, language: language) } + private func recognitionContextSnapshot() -> SpeechRecognitionContext { + recognitionContextLock.lock() + defer { recognitionContextLock.unlock() } + return recognitionContext + } + private func resolvedLocale(for language: String?) -> Locale { switch language { case "zh": return Locale(identifier: "zh-CN") case "en": return Locale(identifier: "en-US") case "ja": return Locale(identifier: "ja-JP") case "ko": return Locale(identifier: "ko-KR") - case "yue": return Locale(identifier: "yue-CN") - default: return locale + case "yue": return Locale(identifier: "zh-HK") + default: return Locale.current } } } diff --git a/Sources/Speech/LocalASREngine.swift b/Sources/Speech/LocalASREngine.swift index c344396b..e7b2fa00 100644 --- a/Sources/Speech/LocalASREngine.swift +++ b/Sources/Speech/LocalASREngine.swift @@ -145,12 +145,25 @@ final class LocalASREngine: SpeechEngine, @unchecked Sendable { guard let runnerURL = Self.runnerScriptURL() else { throw LocalASRError.runnerMissing } let started = CFAbsoluteTimeGetCurrent() - let text = try await server.transcribe( - audioURL: audioURL, - language: language, - runnerURL: runnerURL, - pythonPath: pythonPath - ) + let text: String + switch configuration.provider { + case .qwen3: + text = try await QwenAudioPreprocessor.withPreparedAudio(from: audioURL) { preparedURL in + try await server.transcribe( + audioURL: preparedURL, + language: language, + runnerURL: runnerURL, + pythonPath: pythonPath + ) + } + case .mimo: + text = try await server.transcribe( + audioURL: audioURL, + language: language, + runnerURL: runnerURL, + pythonPath: pythonPath + ) + } let elapsed = CFAbsoluteTimeGetCurrent() - started Log.info("[\(configuration.logName)] transcribed \(text.count) chars locally in \(String(format: "%.1f", elapsed))s") return text diff --git a/Sources/Speech/QwenAudioPreprocessor.swift b/Sources/Speech/QwenAudioPreprocessor.swift new file mode 100644 index 00000000..252826f6 --- /dev/null +++ b/Sources/Speech/QwenAudioPreprocessor.swift @@ -0,0 +1,156 @@ +import AVFoundation +import Foundation + +enum QwenAudioPreprocessor { + static let sampleRate = 16_000.0 + + static func withPreparedAudio( + from sourceURL: URL, + operation: (URL) async throws -> T + ) async throws -> T { + let preparedURL = FileManager.default.temporaryDirectory + .appendingPathComponent("OpenType-QwenASR-\(UUID().uuidString)") + .appendingPathExtension("wav") + defer { try? FileManager.default.removeItem(at: preparedURL) } + + do { + try convertToPCM16kMono(from: sourceURL, to: preparedURL) + } catch { + Log.error("[Qwen3ASR] audio preprocessing failed: \(error.localizedDescription)") + throw QwenAudioPreprocessorError.conversionFailed + } + + return try await operation(preparedURL) + } + + private static func convertToPCM16kMono(from sourceURL: URL, to outputURL: URL) throws { + let sourceFile = try AVAudioFile(forReading: sourceURL) + let sourceFormat = sourceFile.processingFormat + guard sourceFormat.sampleRate > 0, sourceFormat.channelCount > 0 else { + throw AudioConversionError.converterCreationFailed + } + guard let outputFormat = AVAudioFormat( + commonFormat: .pcmFormatInt16, + sampleRate: sampleRate, + channels: 1, + interleaved: true + ) else { + throw AudioConversionError.outputBufferCreationFailed + } + guard let converter = AVAudioConverter(from: sourceFormat, to: outputFormat) else { + throw AudioConversionError.converterCreationFailed + } + converter.sampleRateConverterQuality = AVAudioQuality.max.rawValue + + let outputFile = try AVAudioFile( + forWriting: outputURL, + settings: outputFormat.settings, + commonFormat: .pcmFormatInt16, + interleaved: true + ) + try convert( + sourceFile: sourceFile, + sourceFormat: sourceFormat, + outputFile: outputFile, + outputFormat: outputFormat, + converter: converter + ) + } + + private static func convert( + sourceFile: AVAudioFile, + sourceFormat: AVAudioFormat, + outputFile: AVAudioFile, + outputFormat: AVAudioFormat, + converter: AVAudioConverter + ) throws { + let outputFrameCapacity: AVAudioFrameCount = 4_096 + var reachedEndOfInput = false + + while true { + guard let outputBuffer = AVAudioPCMBuffer( + pcmFormat: outputFormat, + frameCapacity: outputFrameCapacity + ) else { + throw AudioConversionError.outputBufferCreationFailed + } + + var readError: Error? + var conversionError: NSError? + let status = converter.convert( + to: outputBuffer, + error: &conversionError + ) { requestedFrames, inputStatus in + guard !reachedEndOfInput else { + inputStatus.pointee = .endOfStream + return nil + } + + let remainingFrames = sourceFile.length - sourceFile.framePosition + guard remainingFrames > 0 else { + reachedEndOfInput = true + inputStatus.pointee = .endOfStream + return nil + } + + let frameCount = min( + max(requestedFrames, 1), + AVAudioFrameCount(remainingFrames) + ) + guard let inputBuffer = AVAudioPCMBuffer( + pcmFormat: sourceFormat, + frameCapacity: frameCount + ) else { + readError = AudioConversionError.outputBufferCreationFailed + reachedEndOfInput = true + inputStatus.pointee = .endOfStream + return nil + } + + do { + try sourceFile.read(into: inputBuffer, frameCount: frameCount) + } catch { + readError = error + reachedEndOfInput = true + inputStatus.pointee = .endOfStream + return nil + } + + guard inputBuffer.frameLength > 0 else { + reachedEndOfInput = true + inputStatus.pointee = .endOfStream + return nil + } + inputStatus.pointee = .haveData + return inputBuffer + } + + if let readError { throw readError } + if let conversionError { throw conversionError } + if outputBuffer.frameLength > 0 { + try outputFile.write(from: outputBuffer) + } + + switch status { + case .haveData: + continue + case .inputRanDry: + if reachedEndOfInput, outputBuffer.frameLength == 0 { return } + case .endOfStream: + return + case .error: + throw AudioConversionError.conversionFailed + @unknown default: + throw AudioConversionError.conversionFailed + } + } + } +} + +enum QwenAudioPreprocessorError: LocalizedError { + case conversionFailed + + var errorDescription: String? { + L("error.local_asr_audio_conversion") + } +} diff --git a/Sources/Speech/SpeechEngineProtocol.swift b/Sources/Speech/SpeechEngineProtocol.swift index e667898c..b7c94550 100644 --- a/Sources/Speech/SpeechEngineProtocol.swift +++ b/Sources/Speech/SpeechEngineProtocol.swift @@ -7,6 +7,7 @@ protocol SpeechEngine: AnyObject { /// Optional warm-up: load models or start helper processes ahead of the /// first transcription. Must be safe to call repeatedly. func prepare() async + func configureRecognition(context: SpeechRecognitionContext) func startListening(language: String?, onPartialResult: @escaping @Sendable (String) -> Void) func appendAudioBuffer(_ buffer: AVAudioPCMBuffer) func finishListening(audioURL: URL?, language: String?) async throws -> String @@ -19,6 +20,10 @@ extension SpeechEngine { func prepare() async {} + func configureRecognition(context: SpeechRecognitionContext) { + let _ = context + } + func startListening(language: String?, onPartialResult: @escaping @Sendable (String) -> Void) { let _ = language let _ = onPartialResult diff --git a/Sources/Speech/SpeechRecognitionContext.swift b/Sources/Speech/SpeechRecognitionContext.swift new file mode 100644 index 00000000..c48777f9 --- /dev/null +++ b/Sources/Speech/SpeechRecognitionContext.swift @@ -0,0 +1,80 @@ +import Foundation + +struct SpeechRecognitionContext: Equatable, Sendable { + static let empty = SpeechRecognitionContext(phrases: []) + static let maximumPhraseCount = 100 + + let phrases: [String] + + init(phrases: [String]) { + var seen = Set() + let normalizedPhrases: [String] = phrases.compactMap { phrase -> String? in + let normalized = phrase + .replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) + .trimmingCharacters(in: .whitespacesAndNewlines) + guard !normalized.isEmpty, normalized.count <= 80 else { return nil } + guard seen.insert(normalized.lowercased()).inserted else { return nil } + return normalized + } + self.phrases = Array(normalizedPhrases.prefix(Self.maximumPhraseCount)) + } + + init(dictionaryEntries: [DictionaryEntry]) { + self.init(phrases: dictionaryEntries.compactMap { entry -> String? in + guard entry.enabled else { return nil } + let replacement = entry.replacement.trimmingCharacters(in: .whitespacesAndNewlines) + return replacement.isEmpty ? nil : replacement + }) + } + + func whisperPrompt(language: String?) -> String? { + let terms = phrases.joined(separator: ", ") + switch (language, terms.isEmpty) { + case ("zh", true): + return "以下是普通话听写。" + case ("zh", false): + return "以下是普通话听写。专有名词：\(terms)。" + case ("yue", true): + return "以下是粤语听写。" + case ("yue", false): + return "以下是粤语听写。专有名词：\(terms)。" + case ("ja", false): + return "音声入力。固有名詞：\(terms)。" + case ("ko", false): + return "음성 받아쓰기. 고유 명사: \(terms)." + case ("en", false): + return "Dictation. Terms: \(terms)." + case (_, false): + return "Dictation terms: \(terms)." + default: + return nil + } + } + + func whisperPromptTokens( + language: String?, + maximumCount: Int, + tokenize: (String) -> [Int] + ) -> [Int]? { + guard maximumCount > 0 else { return nil } + var acceptedPhrases: [String] = [] + var bestTokens = SpeechRecognitionContext(phrases: []) + .whisperPrompt(language: language) + .map(tokenize) + .flatMap { $0.count <= maximumCount ? $0 : nil } + + for phrase in phrases { + let candidatePhrases = acceptedPhrases + [phrase] + guard let prompt = SpeechRecognitionContext(phrases: candidatePhrases) + .whisperPrompt(language: language) else { + continue + } + let tokens = tokenize(prompt) + if tokens.count <= maximumCount { + acceptedPhrases = candidatePhrases + bestTokens = tokens + } + } + return bestTokens + } +} diff --git a/Sources/Speech/WhisperDownloadProgress.swift b/Sources/Speech/WhisperDownloadProgress.swift new file mode 100644 index 00000000..632c3f76 --- /dev/null +++ b/Sources/Speech/WhisperDownloadProgress.swift @@ -0,0 +1,35 @@ +import Foundation + +extension WhisperEngine { + struct DownloadProgress { + var fraction: Double + var completedBytes: Int64 + var totalBytes: Int64 + var speedBytesPerSec: Double + var elapsedSeconds: TimeInterval + var downloadFraction: Double + var stage: Stage + + enum Stage: String { + case downloading = "下载中" + case compiling = "编译模型" + case loading = "加载模型" + case done = "完成" + } + + var info: DownloadProgressInfo { + DownloadProgressInfo( + fraction: downloadFraction, + elapsedSeconds: elapsedSeconds, + completedBytes: completedBytes, + totalBytes: totalBytes, + speedBytesPerSecond: speedBytesPerSec + ) + } + + var sizeText: String { info.transferredText } + var speedText: String { info.speedText } + var remainingText: String { info.remainingText } + var detailText: String { info.detailText } + } +} diff --git a/Sources/Speech/WhisperEngine.swift b/Sources/Speech/WhisperEngine.swift index 83a97d3a..87d3dc43 100644 --- a/Sources/Speech/WhisperEngine.swift +++ b/Sources/Speech/WhisperEngine.swift @@ -10,73 +10,32 @@ final class WhisperEngine: SpeechEngine, @unchecked Sendable { private(set) var isLoading = false private var loadError: String? private var streamingSession: WhisperStreamingSession? + private let recognitionContextLock = NSLock() + private var recognitionContext = SpeechRecognitionContext.empty init(modelName: String = "large-v3") { self.modelName = modelName.isEmpty ? nil : modelName } - struct DownloadProgress { - var fraction: Double - var completedBytes: Int64 - var totalBytes: Int64 - var speedBytesPerSec: Double - var elapsedSeconds: TimeInterval - var downloadFraction: Double - var stage: Stage - - enum Stage: String { - case downloading = "下载中" - case compiling = "编译模型" - case loading = "加载模型" - case done = "完成" - } - - var info: DownloadProgressInfo { - DownloadProgressInfo( - fraction: downloadFraction, - elapsedSeconds: elapsedSeconds, - completedBytes: completedBytes, - totalBytes: totalBytes, - speedBytesPerSecond: speedBytesPerSec - ) - } - - var sizeText: String { - info.transferredText - } - - var speedText: String { - info.speedText - } - - var remainingText: String { - info.remainingText - } - - var detailText: String { - info.detailText - } - } - func loadModel(progress: @escaping (DownloadProgress) -> Void) async throws { guard !isLoading && !isReady else { return } isLoading = true do { let recommended = WhisperKit.recommendedModels() - var selectedModel = modelName ?? recommended.default + let requestedModel = modelName ?? recommended.default + var selectedModel = ModelStorage.localWhisperURL(requestedModel) != nil + ? requestedModel + : WhisperModelSelection.resolve( + requested: requestedModel, + available: recommended.supported, + fallback: recommended.default + ) let localFolder = ModelStorage.localWhisperURL(selectedModel) if localFolder == nil, !recommended.supported.contains(selectedModel) { - if let match = recommended.supported.first(where: { - $0.localizedCaseInsensitiveContains(selectedModel) - }) { - Log.info("[WhisperEngine] '\(selectedModel)' not in list, matched: \(match)") - selectedModel = match - } else { - Log.info("[WhisperEngine] '\(selectedModel)' not in list, fallback: \(recommended.default)") - selectedModel = recommended.default - } + Log.info("[WhisperEngine] '\(selectedModel)' is unavailable, fallback: \(recommended.default)") + selectedModel = recommended.default } Log.info("[WhisperEngine] using model: \(selectedModel)") @@ -144,14 +103,22 @@ final class WhisperEngine: SpeechEngine, @unchecked Sendable { var supportsStreaming: Bool { true } + func configureRecognition(context: SpeechRecognitionContext) { + recognitionContextLock.lock() + defer { recognitionContextLock.unlock() } + recognitionContext = context + } + func startListening(language: String?, onPartialResult: @escaping @Sendable (String) -> Void) { guard let whisperKit, isReady else { return } + let options = decodingOptions( + language: language, + temperatureFallbackCount: 1 + ) streamingSession = WhisperStreamingSession( whisperKit: whisperKit, partialHandler: onPartialResult, - optionsBuilder: { [weak self] in - self?.decodingOptions(language: language) ?? DecodingOptions() - } + optionsBuilder: { options } ) } @@ -190,9 +157,13 @@ final class WhisperEngine: SpeechEngine, @unchecked Sendable { throw WhisperError.noAudioFile } + let options = decodingOptions(language: language) let t0 = CFAbsoluteTimeGetCurrent() - let results = try await whisperKit.transcribe(audioPath: url.path, decodeOptions: decodingOptions(language: language)) + let results = try await whisperKit.transcribe( + audioPath: url.path, + decodeOptions: options + ) let text = results .compactMap { $0.text } .joined(separator: " ") @@ -211,22 +182,39 @@ final class WhisperEngine: SpeechEngine, @unchecked Sendable { loadError = nil } - private func decodingOptions(language: String?) -> DecodingOptions { - let promptTokens = chinesePromptTokens(for: language) + private func decodingOptions( + language: String?, + temperatureFallbackCount: Int = 5 + ) -> DecodingOptions { + let promptTokens = recognitionPromptTokens(for: language) return DecodingOptions( language: language, - temperatureFallbackCount: 1, - usePrefillPrompt: language != nil, + temperatureFallbackCount: temperatureFallbackCount, + usePrefillPrompt: language != nil || promptTokens != nil, + detectLanguage: language == nil, skipSpecialTokens: true, withoutTimestamps: true, promptTokens: promptTokens, - suppressBlank: true + suppressBlank: true, + chunkingStrategy: .vad ) } - private func chinesePromptTokens(for language: String?) -> [Int]? { - guard language == "zh" else { return nil } - return whisperKit?.tokenizer?.encode(text: "以下是普通话的句子。") + private func recognitionPromptTokens(for language: String?) -> [Int]? { + guard let tokenizer = whisperKit?.tokenizer else { return nil } + let context = recognitionContextSnapshot() + return context.whisperPromptTokens( + language: language, + maximumCount: 160 + ) { + tokenizer.encode(text: $0) + } + } + + private func recognitionContextSnapshot() -> SpeechRecognitionContext { + recognitionContextLock.lock() + defer { recognitionContextLock.unlock() } + return recognitionContext } private func dp(_ fraction: Double, stage: DownloadProgress.Stage) -> DownloadProgress { @@ -241,21 +229,3 @@ final class WhisperEngine: SpeechEngine, @unchecked Sendable { ) } } - -enum WhisperError: LocalizedError { - case modelNotLoaded(String) - case noAudioFile - case downloadFailed(String) - case compileFailed(String) - case loadFailed(String) - - var errorDescription: String? { - switch self { - case .modelNotLoaded(let detail): return String(format: L("error.whisper_not_loaded"), detail) - case .noAudioFile: return L("error.no_audio") - case .downloadFailed(let detail): return String(format: L("error.download_failed"), detail) - case .compileFailed(let detail): return String(format: L("error.compile_failed"), detail) - case .loadFailed(let detail): return String(format: L("error.load_failed"), detail) - } - } -} diff --git a/Sources/Speech/WhisperError.swift b/Sources/Speech/WhisperError.swift new file mode 100644 index 00000000..5d78113d --- /dev/null +++ b/Sources/Speech/WhisperError.swift @@ -0,0 +1,24 @@ +import Foundation + +enum WhisperError: LocalizedError { + case modelNotLoaded(String) + case noAudioFile + case downloadFailed(String) + case compileFailed(String) + case loadFailed(String) + + var errorDescription: String? { + switch self { + case .modelNotLoaded(let detail): + return String(format: L("error.whisper_not_loaded"), detail) + case .noAudioFile: + return L("error.no_audio") + case .downloadFailed(let detail): + return String(format: L("error.download_failed"), detail) + case .compileFailed(let detail): + return String(format: L("error.compile_failed"), detail) + case .loadFailed(let detail): + return String(format: L("error.load_failed"), detail) + } + } +} diff --git a/Sources/Speech/WhisperModelSelection.swift b/Sources/Speech/WhisperModelSelection.swift new file mode 100644 index 00000000..b99bdcaa --- /dev/null +++ b/Sources/Speech/WhisperModelSelection.swift @@ -0,0 +1,53 @@ +import Foundation + +enum WhisperModelSelection { + static func resolve( + requested: String, + available: [String], + fallback: String + ) -> String { + if available.contains(requested) { + return requested + } + let requestedVariant = canonicalVariant(requested) + if available.contains(fallback), matches(fallback, variant: requestedVariant) { + return fallback + } + if let match = available.first(where: { + matches($0, variant: requestedVariant) + }) { + return match + } + if available.contains(fallback) { + return fallback + } + return available.first ?? fallback + } + + static func matches(_ modelID: String, variant: String) -> Bool { + let requested = canonicalVariant(variant) + return !requested.isEmpty && canonicalVariant(modelID) == requested + } + + static func canonicalVariant(_ value: String) -> String { + var result = value.lowercased() + if let slash = result.lastIndex(of: "/") { + result = String(result[result.index(after: slash)...]) + } + if result.hasPrefix("openai_whisper-") { + result.removeFirst("openai_whisper-".count) + } + result = result.replacingOccurrences( + of: #"_[0-9]+mb$"#, + with: "", + options: .regularExpression + ) + result = result.replacingOccurrences( + of: #"-v[0-9]{4,}(?=[_-]|$)"#, + with: "", + options: .regularExpression + ) + result = result.replacingOccurrences(of: "_turbo", with: "-turbo") + return result + } +} diff --git a/Tests/OpenTypeTests/AutoCantonesePromptTests.swift b/Tests/OpenTypeTests/AutoCantonesePromptTests.swift index e03ea289..21988a0f 100644 --- a/Tests/OpenTypeTests/AutoCantonesePromptTests.swift +++ b/Tests/OpenTypeTests/AutoCantonesePromptTests.swift @@ -23,7 +23,7 @@ final class AutoCantonesePromptTests: XCTestCase { XCTAssertTrue(PromptBuilder.buildUserPrompt( text: "um hello", inputLanguage: .auto - ).contains("自动语言语音识别原文")) + ).contains("自动语言 ASR 原文")) XCTAssertTrue(PromptBuilder.buildUserPrompt( text: "啱啱講錯咗", inputLanguage: .cantonese diff --git a/Tests/OpenTypeTests/ConfigurationTests.swift b/Tests/OpenTypeTests/ConfigurationTests.swift index d18e3913..cc356279 100644 --- a/Tests/OpenTypeTests/ConfigurationTests.swift +++ b/Tests/OpenTypeTests/ConfigurationTests.swift @@ -3,6 +3,11 @@ import XCTest @testable import OpenType final class ConfigurationTests: XCTestCase { + func testDefaultFormattingModelDoesNotUseTinyFallback() { + XCTAssertEqual(AppSettings.defaultLLMModelID, "mlx-community/Qwen3.5-2B-4bit") + XCTAssertNotEqual(AppSettings.defaultLLMModelID, "mlx-community/Qwen3.5-0.8B-MLX-4bit") + } + func testRemoteProviderDefaultsMatchExpectedAPIs() { XCTAssertEqual(RemoteProvider.openai.defaultBaseURL, "https://api.openai.com/v1") XCTAssertEqual(RemoteProvider.openai.defaultModel, "gpt-4.1-mini") @@ -40,7 +45,7 @@ final class ConfigurationTests: XCTestCase { XCTAssertEqual(InputLanguage.korean.whisperCode, "ko") XCTAssertEqual(InputLanguage.cantonese.whisperCode, "yue") - XCTAssertEqual(InputLanguage.auto.localeIdentifier, "zh-CN") + XCTAssertEqual(InputLanguage.auto.localeIdentifier, Locale.current.identifier) XCTAssertEqual(InputLanguage.chinese.localeIdentifier, "zh-CN") XCTAssertEqual(InputLanguage.english.localeIdentifier, "en-US") XCTAssertEqual(InputLanguage.japanese.localeIdentifier, "ja-JP") diff --git a/Tests/OpenTypeTests/DeferredReplacementPolicyTests.swift b/Tests/OpenTypeTests/DeferredReplacementPolicyTests.swift new file mode 100644 index 00000000..0e22412d --- /dev/null +++ b/Tests/OpenTypeTests/DeferredReplacementPolicyTests.swift @@ -0,0 +1,134 @@ +import AppKit +import XCTest +@testable import OpenType + +@MainActor +final class DeferredReplacementPolicyTests: XCTestCase { + func testOnlyAppliesToSmartFormat() { + XCTAssertTrue(DeferredReplacementPolicy.shouldUseDeferredReplacement( + outputMode: .processed, + enableInstantInsert: true + )) + XCTAssertFalse(DeferredReplacementPolicy.shouldUseDeferredReplacement( + outputMode: .processed, + enableInstantInsert: false + )) + XCTAssertFalse(DeferredReplacementPolicy.shouldUseDeferredReplacement( + outputMode: .direct, + enableInstantInsert: true + )) + XCTAssertFalse(DeferredReplacementPolicy.shouldUseDeferredReplacement( + outputMode: .command, + enableInstantInsert: true + )) + } + + func testFormattingKeepsFocusedTextFromQuickContext() { + let quickContext = InputContext( + appName: "Notes", + bundleIdentifier: "com.apple.Notes", + windowTitle: "Planning", + screenContext: nil, + textBeforeSelection: "before cursor", + selectedText: "selected phrase", + textAfterSelection: "after cursor", + outputMode: .processed, + inputLanguage: .english, + source: .menuBar + ) + let replacement = DeferredReplacement( + rawText: "raw", + insertedText: "quick", + targetApp: nil, + message: "formatting", + context: quickContext + ) + + let deferredContext = VoicePipeline.deferredInputContext( + for: replacement, + screenContext: "fresh OCR", + inputLanguage: .english + ) + + XCTAssertEqual(deferredContext.windowTitle, "Planning") + XCTAssertEqual(deferredContext.screenContext, "fresh OCR") + XCTAssertEqual(deferredContext.textBeforeSelection, "before cursor") + XCTAssertEqual(deferredContext.selectedText, "selected phrase") + XCTAssertEqual(deferredContext.textAfterSelection, "after cursor") + } + + func testFailedStateIsNotReplaceable() { + var replacement = DeferredReplacement( + rawText: "raw", + insertedText: "quick", + targetApp: nil, + message: "formatting", + createdAt: Date(timeIntervalSince1970: 100), + expirationInterval: 15 + ) + replacement.state = .failed + + XCTAssertEqual( + DeferredReplacementPolicy.decision( + for: replacement, + currentBundleIdentifier: nil, + now: Date(timeIntervalSince1970: 105) + ), + .copy(.notReady) + ) + } + + func testDecisionRequiresSameFrontmostApp() throws { + var replacement = DeferredReplacement( + rawText: "raw", + insertedText: "quick", + targetApp: nil, + message: "formatting", + createdAt: Date(timeIntervalSince1970: 100), + expirationInterval: 15 + ) + replacement.formattedText = "formatted" + replacement.state = .ready + + XCTAssertEqual( + DeferredReplacementPolicy.decision( + for: replacement, + currentBundleIdentifier: nil, + now: Date(timeIntervalSince1970: 105) + ), + .copy(.missingTarget) + ) + + guard let bundleIdentifier = NSRunningApplication.current.bundleIdentifier else { + throw XCTSkip("Current test process has no bundle identifier") + } + + replacement = DeferredReplacement( + rawText: "raw", + insertedText: "quick", + targetApp: NSRunningApplication.current, + message: "formatting", + createdAt: Date(timeIntervalSince1970: 100), + expirationInterval: 15 + ) + replacement.formattedText = "formatted" + replacement.state = .ready + + XCTAssertEqual( + DeferredReplacementPolicy.decision( + for: replacement, + currentBundleIdentifier: "other.app", + now: Date(timeIntervalSince1970: 105) + ), + .copy(.appChanged) + ) + XCTAssertEqual( + DeferredReplacementPolicy.decision( + for: replacement, + currentBundleIdentifier: bundleIdentifier, + now: Date(timeIntervalSince1970: 116) + ), + .copy(.expired) + ) + } +} diff --git a/Tests/OpenTypeTests/PromptAndProcessingTests.swift b/Tests/OpenTypeTests/PromptAndProcessingTests.swift index 161a5a31..fb6653fa 100644 --- a/Tests/OpenTypeTests/PromptAndProcessingTests.swift +++ b/Tests/OpenTypeTests/PromptAndProcessingTests.swift @@ -52,11 +52,53 @@ final class PromptAndProcessingTests: XCTestCase { ] XCTAssertEqual(dictionary.applyReplacements(to: "open type should not skip me"), "OpenType should not skip me") + XCTAssertEqual(dictionary.applyReplacements(to: "blank replacement"), "") XCTAssertEqual(dictionary.activeEntriesDescription(), "open type -> OpenType") XCTAssertEqual(dictionary.activeRulesDescription(), "Keep product names exact.") } } + func testPersonalDictionaryUsesLongestNonCascadingWordMatches() { + withCleanSettings { + let dictionary = PersonalDictionary.shared + dictionary.entries = [ + DictionaryEntry(original: "open", replacement: "closed", enabled: true), + DictionaryEntry(original: "open type", replacement: "OpenType", enabled: true), + DictionaryEntry(original: "OpenType", replacement: "WrongCascade", enabled: true), + DictionaryEntry(original: "api", replacement: "API", enabled: true), + ] + + XCTAssertEqual( + dictionary.applyReplacements(to: "open type has an api, not rapid"), + "OpenType has an API, not rapid" + ) + } + } + + func testPersonalDictionaryStillMatchesCJKTermsWithoutWordBoundaries() { + withCleanSettings { + let dictionary = PersonalDictionary.shared + dictionary.entries = [ + DictionaryEntry(original: "开放类型", replacement: "OpenType", enabled: true) + ] + + XCTAssertEqual(dictionary.applyReplacements(to: "使用开放类型输入"), "使用OpenType输入") + } + } + + func testPersonalDictionaryMatchesLatinTermsCaseInsensitively() { + withCleanSettings { + PersonalDictionary.shared.entries = [ + DictionaryEntry(original: "OPEN TYPE", replacement: "OpenType") + ] + + XCTAssertEqual( + PersonalDictionary.shared.applyReplacements(to: "open type works"), + "OpenType works" + ) + } + } + func testBasicCleanAppliesDictionaryAndNormalizesWhitespace() { withCleanSettings { PersonalDictionary.shared.entries = [ @@ -209,9 +251,34 @@ final class PromptAndProcessingTests: XCTestCase { XCTAssertEqual(processor.formattingOptions(for: String(repeating: "长", count: 260), style: .professional).maxTokens, 640) XCTAssertEqual(processor.formattingOptions(for: "short", style: .casual).maxTokens, 160) XCTAssertEqual(processor.formattingOptions(for: String(repeating: "c", count: 120), style: .casual).maxTokens, 256) - XCTAssertEqual(processor.formattingOptions(for: String(repeating: "c", count: 260), style: .casual).maxTokens, 384) - XCTAssertEqual(processor.formattingOptions(for: "短句", style: .casual).temperature, 0.08) - XCTAssertEqual(processor.formattingOptions(for: "短句", style: .professional).temperature, 0.10) + XCTAssertEqual(processor.formattingOptions(for: String(repeating: "c", count: 260), style: .casual).maxTokens, 520) + XCTAssertEqual(processor.formattingOptions(for: "短句", style: .casual).temperature, 0) + XCTAssertEqual(processor.formattingOptions(for: "短句", style: .professional).temperature, 0) + } + } + + func testFormattingOptionsExpandWithLongInputAndCapAt4096() { + withCleanSettings { + let processor = TextProcessor() + let longText = String(repeating: "中", count: 1_000) + let oversizedText = String(repeating: "中", count: 4_000) + + XCTAssertGreaterThan( + processor.formattingOptions(for: longText, style: .professional).maxTokens, + 640 + ) + XCTAssertGreaterThan( + processor.formattingOptions(for: longText, style: .casual).maxTokens, + 384 + ) + XCTAssertEqual( + processor.formattingOptions(for: oversizedText, style: .professional).maxTokens, + 4_096 + ) + XCTAssertEqual( + processor.formattingOptions(for: oversizedText, style: .casual).maxTokens, + 4_096 + ) } } diff --git a/Tests/OpenTypeTests/PromptBuilderTests.swift b/Tests/OpenTypeTests/PromptBuilderTests.swift index cc2a4cf3..1aa46df6 100644 --- a/Tests/OpenTypeTests/PromptBuilderTests.swift +++ b/Tests/OpenTypeTests/PromptBuilderTests.swift @@ -36,20 +36,19 @@ final class PromptBuilderTests: XCTestCase { } try body() } - func testBuildUserPromptUsesLanguageSpecificWrappers() { XCTAssertEqual(PromptBuilder.buildUserPrompt( text: "嗯 今天开会", inputLanguage: .chinese - ), "以下是语音识别原文。请先在内部理解用户的口述意图，判断错别字、同音词、误识别词、漏字、多字、口述标点、数字单位、时间范围和专有名词，再直接输出整理后的最终文本：\n\(PromptTextBlock.block("嗯 今天开会"))") + ), "以下是语音识别原文。请做忠实纠错：只在原文本身、自我纠正、个人词典或提供的上下文有明确依据时修正错别字、同音词、误识别和专有名词；不要猜测漏字或补写未口述的实词。处理口述标点、数字、单位和时间范围，只输出最终文本：\n\(PromptTextBlock.block("嗯 今天开会"))") XCTAssertEqual(PromptBuilder.buildUserPrompt( text: "um hello", inputLanguage: .english - ), "Raw ASR transcript. Internally infer the user's spoken intent, including punctuation commands, numbers, units, date/time ranges, typos, homophones, ASR substitutions, missing or extra words, and proper nouns, then output only the final rewritten text:\n\(PromptTextBlock.block("um hello"))") + ), "Raw ASR transcript. Perform faithful correction. Fix homophones, ASR substitutions, and proper nouns only when supported by the transcript, an explicit self-correction, the personal dictionary, or provided context. Never guess missing content or add undictated lexical words. Apply spoken punctuation, numbers, units, and date/time ranges, then output only the final text:\n\(PromptTextBlock.block("um hello"))") XCTAssertEqual(PromptBuilder.buildUserPrompt( text: "こんにちは", inputLanguage: .japanese - ), "日本語の音声認識原文です。口述意図、誤認識、同音語、抜けた語、余分な語、口述された句読点、数字、単位、日時、範囲、固有名詞を内部で判断し、最終テキストだけを出力してください：\n\(PromptTextBlock.block("こんにちは"))") + ), "日本語の音声認識原文です。忠実に補正してください。原文、自明な言い直し、個人辞書、または提供された文脈に明確な根拠がある場合だけ、誤認識、同音語、固有名詞を直してください。抜けた内容を推測したり、口述されていない実質語を追加したりしないでください。口述された句読点、数字、単位、日時、範囲を整え、最終テキストだけを出力してください：\n\(PromptTextBlock.block("こんにちは"))") } func testBuildCommandUserPromptUsesLanguageSpecificWrappers() { @@ -85,9 +84,10 @@ final class PromptBuilderTests: XCTestCase { inputLanguage: .chinese ) - XCTAssertTrue(prompt.contains("力度要高于轻度润色")) + XCTAssertTrue(prompt.contains("忠实纠错和整理，不做自由改写")) XCTAssertTrue(prompt.contains("风格：专业整理")) - XCTAssertTrue(prompt.contains("同音错字、近音错字、漏字、多字")) + XCTAssertTrue(prompt.contains("同音错字、近音错字和误识别词")) + XCTAssertTrue(prompt.contains("实词的新增、删除或替换必须有原文")) XCTAssertTrue(prompt.contains("智能理解口述格式意图")) XCTAssertTrue(prompt.contains("百分之二十五到三十")) XCTAssertTrue(prompt.contains("输出：把灰度比例改为 25%-30%，发布窗口改到下午 3 点到 4 点。")) @@ -125,9 +125,10 @@ final class PromptBuilderTests: XCTestCase { inputLanguage: .english ) - XCTAssertTrue(prompt.contains("Do not lightly polish raw ASR")) + XCTAssertTrue(prompt.contains("Perform faithful correction and formatting, not free rewriting")) XCTAssertTrue(prompt.contains("Style: professional cleanup")) - XCTAssertTrue(prompt.contains("homophones, ASR substitutions, missing words, extra words")) + XCTAssertTrue(prompt.contains("homophones, ASR substitutions, proper nouns")) + XCTAssertTrue(prompt.contains("adding, deleting, or replacing lexical words requires direct support")) XCTAssertTrue(prompt.contains("intelligently interpret spoken formatting intent")) XCTAssertTrue(prompt.contains("twenty five percent to thirty percent")) XCTAssertTrue(prompt.contains("Output: Set the rollout to 25%-30%, and move the release window to 3 PM to 4 PM.")) diff --git a/Tests/OpenTypeTests/QwenAudioPreprocessorTests.swift b/Tests/OpenTypeTests/QwenAudioPreprocessorTests.swift new file mode 100644 index 00000000..3a1ccf52 --- /dev/null +++ b/Tests/OpenTypeTests/QwenAudioPreprocessorTests.swift @@ -0,0 +1,161 @@ +import AVFoundation +import XCTest +@testable import OpenType + +final class QwenAudioPreprocessorTests: XCTestCase { + func testPreparedAudioIs16kMonoPCMAndIsRemovedAfterUse() async throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("QwenAudioPreprocessorTests-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + + let sourceURL = directory.appendingPathComponent("source.wav") + try writeTone( + to: sourceURL, + sampleRate: 48_000, + channels: 2, + frequency: 1_000 + ) + + let preparedURL = try await QwenAudioPreprocessor.withPreparedAudio(from: sourceURL) { url in + XCTAssertTrue(FileManager.default.fileExists(atPath: url.path)) + XCTAssertNotEqual(url, sourceURL) + + let file = try AVAudioFile(forReading: url) + XCTAssertEqual(file.fileFormat.sampleRate, 16_000, accuracy: 0.1) + XCTAssertEqual(file.fileFormat.channelCount, 1) + XCTAssertEqual(file.fileFormat.commonFormat, .pcmFormatInt16) + XCTAssertLessThanOrEqual(abs(file.length - 16_000), 2) + return url + } + + XCTAssertFalse(FileManager.default.fileExists(atPath: preparedURL.path)) + XCTAssertTrue(FileManager.default.fileExists(atPath: sourceURL.path)) + } + + func testPreparedAudioIsRemovedWhenUseFails() async throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("QwenAudioPreprocessorTests-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + + let sourceURL = directory.appendingPathComponent("source.wav") + try writeTone( + to: sourceURL, + sampleRate: 44_100, + channels: 1, + frequency: 1_000 + ) + + var preparedURL: URL? + do { + _ = try await QwenAudioPreprocessor.withPreparedAudio(from: sourceURL) { url -> Bool in + preparedURL = url + throw ProbeError.expected + } + XCTFail("Expected the operation to fail") + } catch ProbeError.expected { + } + + let removedURL = try XCTUnwrap(preparedURL) + XCTAssertFalse(FileManager.default.fileExists(atPath: removedURL.path)) + } + + func testDownsamplingRejectsAudioAboveThe16kNyquistFrequency() async throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("QwenAudioPreprocessorTests-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + + let passbandURL = directory.appendingPathComponent("passband.wav") + try writeTone( + to: passbandURL, + sampleRate: 48_000, + channels: 1, + frequency: 1_000 + ) + let rejectedURL = directory.appendingPathComponent("above-nyquist.wav") + try writeTone( + to: rejectedURL, + sampleRate: 48_000, + channels: 1, + frequency: 12_000 + ) + + let passbandRMS = try await preparedRMS(from: passbandURL) + let rejectedRMS = try await preparedRMS(from: rejectedURL) + + XCTAssertGreaterThan(passbandRMS, 0.25) + XCTAssertLessThan(rejectedRMS, passbandRMS * 0.05) + } + + private func writeTone( + to url: URL, + sampleRate: Double, + channels: AVAudioChannelCount, + frequency: Double, + amplitude: Float = 0.5 + ) throws { + let format = try XCTUnwrap( + AVAudioFormat( + commonFormat: .pcmFormatFloat32, + sampleRate: sampleRate, + channels: channels, + interleaved: true + ) + ) + let frameCount = AVAudioFrameCount(sampleRate) + let buffer = try XCTUnwrap( + AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frameCount) + ) + buffer.frameLength = frameCount + + let audioBuffers = UnsafeMutableAudioBufferListPointer(buffer.mutableAudioBufferList) + let samples = try XCTUnwrap( + audioBuffers.first?.mData?.assumingMemoryBound(to: Float.self) + ) + for frame in 0.. [Float] { + let file = try AVAudioFile(forReading: url) + let buffer = try XCTUnwrap( + AVAudioPCMBuffer( + pcmFormat: file.processingFormat, + frameCapacity: AVAudioFrameCount(file.length) + ) + ) + try file.read(into: buffer) + let channel = try XCTUnwrap(buffer.floatChannelData?[0]) + return Array(UnsafeBufferPointer(start: channel, count: Int(buffer.frameLength))) + } + + private func preparedRMS(from sourceURL: URL) async throws -> Double { + try await QwenAudioPreprocessor.withPreparedAudio(from: sourceURL) { url in + let samples = try readSamples(from: url) + let stableSamples = samples.dropFirst(1_024).dropLast(1_024) + let meanSquare = stableSamples.reduce(0.0) { sum, sample in + sum + Double(sample * sample) + } / Double(stableSamples.count) + return sqrt(meanSquare) + } + } + + private enum ProbeError: Error { + case expected + } +} diff --git a/Tests/OpenTypeTests/RemoteLLMTokenBudgetTests.swift b/Tests/OpenTypeTests/RemoteLLMTokenBudgetTests.swift new file mode 100644 index 00000000..4376031e --- /dev/null +++ b/Tests/OpenTypeTests/RemoteLLMTokenBudgetTests.swift @@ -0,0 +1,54 @@ +import XCTest +@testable import OpenType + +final class RemoteLLMTokenBudgetTests: XCTestCase { + func testRetriesLargeTokenLimitFailuresAtCompatibleBudget() { + XCTAssertEqual( + RemoteLLMClient.retryTokenBudget( + maxTokens: 4_096, + failureMessage: "HTTP 400: maximum context length exceeded" + ), + 1_024 + ) + XCTAssertEqual( + RemoteLLMClient.retryTokenBudget( + maxTokens: 640, + failureMessage: "max_tokens must be less than the token limit" + ), + 320 + ) + } + + func testDoesNotRetryAuthenticationOrAlreadySmallBudgets() { + XCTAssertNil(RemoteLLMClient.retryTokenBudget( + maxTokens: 4_096, + failureMessage: "HTTP 401: invalid API key" + )) + XCTAssertNil(RemoteLLMClient.retryTokenBudget( + maxTokens: 256, + failureMessage: "context_length exceeded" + )) + XCTAssertNil(RemoteLLMClient.retryTokenBudget( + maxTokens: 4_096, + failureMessage: "HTTP 400: unsupported parameter max_tokens; use max_completion_tokens" + )) + XCTAssertNil(RemoteLLMClient.retryTokenBudget( + maxTokens: 4_096, + failureMessage: "HTTP 429: account token limit reached" + )) + } + + func testRejectsProviderTruncatedResponses() throws { + let openAI = Data( + #"{"choices":[{"finish_reason":"length","message":{"content":"partial"}}]}"# + .utf8 + ) + let anthropic = Data( + #"{"stop_reason":"max_tokens","content":[{"type":"text","text":"partial"}]}"# + .utf8 + ) + + XCTAssertThrowsError(try RemoteLLMResponseText.openAI(from: openAI)) + XCTAssertThrowsError(try RemoteLLMResponseText.anthropic(from: anthropic)) + } +} diff --git a/Tests/OpenTypeTests/SpeechRecognitionQualityTests.swift b/Tests/OpenTypeTests/SpeechRecognitionQualityTests.swift new file mode 100644 index 00000000..fe37357a --- /dev/null +++ b/Tests/OpenTypeTests/SpeechRecognitionQualityTests.swift @@ -0,0 +1,123 @@ +import XCTest +@testable import OpenType + +final class SpeechRecognitionQualityTests: XCTestCase { + func testWhisperModelSelectionResolvesQualityAliasInsteadOfDeviceFallback() { + let available = [ + "openai_whisper-base", + "openai_whisper-large-v3", + "openai_whisper-large-v3-v20240930_626MB", + "openai_whisper-large-v3-turbo_954MB", + ] + + XCTAssertEqual( + WhisperModelSelection.resolve( + requested: "large-v3", + available: available, + fallback: "openai_whisper-large-v3-v20240930_626MB" + ), + "openai_whisper-large-v3-v20240930_626MB" + ) + } + + func testWhisperModelSelectionPreservesExplicitFullModelID() { + let requested = "openai_whisper-large-v3-turbo_954MB" + + XCTAssertEqual( + WhisperModelSelection.resolve( + requested: requested, + available: ["openai_whisper-base", requested], + fallback: "openai_whisper-base" + ), + requested + ) + } + + func testWhisperModelSelectionMigratesStaleBuildWithinFamily() { + XCTAssertEqual( + WhisperModelSelection.resolve( + requested: "openai_whisper-large-v3-v20240930_547MB", + available: [ + "openai_whisper-base", + "openai_whisper-large-v3-v20250701_626MB", + ], + fallback: "openai_whisper-base" + ), + "openai_whisper-large-v3-v20250701_626MB" + ) + } + + func testWhisperVariantDoesNotMistakeTurboForLargeV3() { + XCTAssertFalse( + WhisperModelSelection.matches( + "openai_whisper-large-v3-turbo_954MB", + variant: "large-v3" + ) + ) + XCTAssertFalse( + WhisperModelSelection.matches( + "openai_whisper-large-v3-v20240930_turbo_632MB", + variant: "large-v3" + ) + ) + XCTAssertTrue( + WhisperModelSelection.matches( + "openai_whisper-large-v3-v20240930_turbo_632MB", + variant: "large-v3-turbo" + ) + ) + } + + func testRecognitionContextUsesEnabledCanonicalTermsAndDeduplicates() { + let context = SpeechRecognitionContext(dictionaryEntries: [ + DictionaryEntry(original: "open type", replacement: "OpenType", enabled: true), + DictionaryEntry(original: "whisper kit", replacement: "WhisperKit", enabled: true), + DictionaryEntry(original: "duplicate", replacement: "opentype", enabled: true), + DictionaryEntry(original: "disabled", replacement: "Disabled", enabled: false), + DictionaryEntry(original: "blank", replacement: " ", enabled: true), + ]) + + XCTAssertEqual(context.phrases, ["OpenType", "WhisperKit"]) + XCTAssertEqual( + context.whisperPrompt(language: "zh"), + "以下是普通话听写。专有名词：OpenType, WhisperKit。" + ) + } + + func testRecognitionContextCapsAppleContextualPhraseLimit() { + let context = SpeechRecognitionContext( + phrases: (0..<120).map { "term-\($0)" } + ) + + XCTAssertEqual(context.phrases.count, 100) + } + + func testWhisperPromptBudgetKeepsWholeEarlyAndLaterTerms() { + let context = SpeechRecognitionContext( + phrases: ["OpenType", String(repeating: "x", count: 80), "MLX"] + ) + + let prompt = context.whisperPromptTokens( + language: nil, + maximumCount: 34, + tokenize: { Array($0.utf8).map(Int.init) } + ).map { String(decoding: $0.map(UInt8.init), as: UTF8.self) } + + XCTAssertEqual(prompt, "Dictation terms: OpenType, MLX.") + } + + func testAppleCompatibleDictationPresetChangesAfterOneMinute() { + XCTAssertEqual( + AppleSpeechAnalyzer.dictationPreset(forDuration: 30), + .shortDictation + ) + XCTAssertEqual( + AppleSpeechAnalyzer.dictationPreset(forDuration: 60), + .shortDictation + ) + XCTAssertEqual( + AppleSpeechAnalyzer.dictationPreset(forDuration: 60.001), + .longDictation + ) + } +} diff --git a/Tests/OpenTypeTests/TextProcessingSnapshotTests.swift b/Tests/OpenTypeTests/TextProcessingSnapshotTests.swift new file mode 100644 index 00000000..d960e24a --- /dev/null +++ b/Tests/OpenTypeTests/TextProcessingSnapshotTests.swift @@ -0,0 +1,69 @@ +import XCTest +@testable import OpenType + +@MainActor +final class TextProcessingSnapshotTests: XCTestCase { + func testDictionarySnapshotKeepsAllTermsAndOriginalRules() { + var entries = (0..<105).map { + DictionaryEntry(original: "term \($0)", replacement: "Term\($0)") + } + let snapshot = PersonalDictionarySnapshot( + entries: entries, + editRules: [EditRule(description: "Keep product names exact.")] + ) + entries.removeAll() + + XCTAssertEqual(snapshot.protectedTerms.count, 105) + XCTAssertEqual(snapshot.applyReplacements(to: "term 104"), "Term104") + XCTAssertTrue(snapshot.activeEntriesDescription.contains("term 104 -> Term104")) + XCTAssertEqual( + snapshot.activeRulesDescription, + "Keep product names exact." + ) + } + + func testFormattingPromptUsesCapturedSettingsAndDictionary() { + let settings = AppSettings.shared + let dictionary = PersonalDictionary.shared + let savedUseCustom = settings.useCustomSystemPrompt + let savedCustomPrompt = settings.customSystemPrompt + let savedEntries = dictionary.entries + let savedRules = dictionary.editRules + defer { + settings.useCustomSystemPrompt = savedUseCustom + settings.customSystemPrompt = savedCustomPrompt + dictionary.entries = savedEntries + dictionary.editRules = savedRules + } + + settings.useCustomSystemPrompt = false + settings.customSystemPrompt = "" + dictionary.entries = [ + DictionaryEntry(original: "open type", replacement: "OpenType") + ] + dictionary.editRules = [EditRule(description: "Use the captured rule.")] + let options = TextProcessingOptions(settings: settings) + let dictionarySnapshot = dictionary.snapshot() + + settings.useCustomSystemPrompt = true + settings.customSystemPrompt = "Changed after capture." + dictionary.entries = [ + DictionaryEntry(original: "new term", replacement: "NewTerm") + ] + dictionary.editRules = [] + + let prompt = TextProcessor().formattingSystemPrompt( + options: options, + screenContext: "", + screenImageAvailable: false, + memoryContext: "", + inputContext: nil, + dictionarySnapshot: dictionarySnapshot + ) + + XCTAssertFalse(prompt.contains("Changed after capture.")) + XCTAssertTrue(prompt.contains("open type -> OpenType")) + XCTAssertTrue(prompt.contains("Use the captured rule.")) + XCTAssertFalse(prompt.contains("new term -> NewTerm")) + } +} diff --git a/Tests/OpenTypeTests/TextProcessorFallbackTests.swift b/Tests/OpenTypeTests/TextProcessorFallbackTests.swift index e2ac9318..e8673ab7 100644 --- a/Tests/OpenTypeTests/TextProcessorFallbackTests.swift +++ b/Tests/OpenTypeTests/TextProcessorFallbackTests.swift @@ -23,6 +23,40 @@ final class TextProcessorFallbackTests: XCTestCase { ) } + func testRejectedOutputFallbackRespectsPreparedFallbackPolicy() { + let processor = TextProcessor() + + XCTAssertEqual( + processor.rejectedOutputFallback( + "raw transcript", + allowsGuardFallback: true + ), + "raw transcript" + ) + XCTAssertEqual( + processor.rejectedOutputFallback( + "raw transcript", + allowsGuardFallback: false + ), + "" + ) + } + + func testCustomTransformationUsesNeutralUserPromptAndExplicitPolicy() { + let settings = AppSettings.shared + var options = TextProcessingOptions(settings: settings, inputLanguage: .english) + options.useCustomSystemPrompt = true + options.customSystemPrompt = "Summarize concisely." + + XCTAssertEqual(options.fidelityPolicy, .boundedCustomTransformation) + let prompt = TextProcessor().formattingUserPrompt( + text: "A long raw transcript", + options: options + ) + XCTAssertTrue(prompt.contains("Apply the custom instructions")) + XCTAssertFalse(prompt.contains("Never guess missing content")) + } + func testGeneratedOutputUsesFinalSectionAfterAnalysisScaffold() { let processor = TextProcessor() let output = """ @@ -132,4 +166,29 @@ final class TextProcessorFallbackTests: XCTestCase { output ) } + + func testTextFallbackPromptsDoNotClaimScreenImageIsAttached() { + let processor = TextProcessor() + let options = TextProcessingOptions(settings: AppSettings.shared, inputLanguage: .english) + + let formattingPrompt = processor.formattingSystemPrompt( + options: options, + screenContext: "OpenType settings", + screenImageAvailable: false, + memoryContext: "", + inputContext: nil + ) + let commandPrompt = processor.commandSystemPrompt( + options: options, + screenContext: "OpenType settings", + screenImageAvailable: false, + memoryContext: "", + inputContext: nil + ) + + XCTAssertTrue(formattingPrompt.contains("OpenType settings")) + XCTAssertTrue(commandPrompt.contains("OpenType settings")) + XCTAssertFalse(formattingPrompt.contains("A screen image is attached")) + XCTAssertFalse(commandPrompt.contains("A screen image is attached")) + } } diff --git a/Tests/OpenTypeTests/TranscriptFidelityEdgeCaseTests.swift b/Tests/OpenTypeTests/TranscriptFidelityEdgeCaseTests.swift new file mode 100644 index 00000000..dea168bb --- /dev/null +++ b/Tests/OpenTypeTests/TranscriptFidelityEdgeCaseTests.swift @@ -0,0 +1,118 @@ +import XCTest +@testable import OpenType + +final class TranscriptFidelityEdgeCaseTests: XCTestCase { + func testPreservesPercentSignSignificantZerosAndPrecision() { + XCTAssertEqual(violation("rollout 25", "rollout 25%"), "protected_token_change") + XCTAssertEqual(violation("delta +5", "delta 5"), "protected_token_change") + XCTAssertEqual(violation("code 00123", "code 123"), "protected_token_change") + XCTAssertEqual(violation("version 2.50", "version 2.5"), "protected_token_change") + } + + func testEnglishSpokenDigitsDecimalAndYearHaveExactEvidence() { + XCTAssertNil(violation("code one two three", "code 123")) + XCTAssertNil(violation("version two point five", "version 2.5")) + XCTAssertNil(violation("year twenty twenty six", "year 2026")) + } + + func testMeasurementUnitsAreBoundToNumbers() { + XCTAssertEqual( + violation("wait 3 seconds", "wait 3 minutes"), + "protected_token_change" + ) + XCTAssertEqual(violation("weight 5 kg", "weight 5 lb"), "protected_token_change") + XCTAssertEqual( + violation("temperature 12 °C", "temperature 12 °F"), + "protected_token_change" + ) + } + + func testRepeatedAndSharedRangeUnitsAreEquivalent() { + XCTAssertNil(violation("wait 3 days to 5 days", "wait 3-5 days")) + XCTAssertNil(violation("三天到五天内发版", "3-5 天内发版", language: .chinese)) + } + + func testChineseNumberEvidenceRejectsIdiomsButAllowsQuantities() { + XCTAssertEqual( + violation("千万不要发布", "10000000 不要发布", language: .chinese), + "protected_token_change" + ) + XCTAssertEqual( + violation("十分重要", "10 分重要", language: .chinese), + "protected_token_change" + ) + XCTAssertNil(violation("三天内发版", "3 天内发版", language: .chinese)) + XCTAssertNil(violation("三分钟后提醒", "3 分钟后提醒", language: .chinese)) + } + + func testKoreanOrdinalListFormattingHasEvidence() { + XCTAssertNil(violation( + "첫째 요구사항 확인 둘째 일정 조율 셋째 예산 업데이트", + "1. 요구사항 확인 2. 일정 조율 3. 예산 업데이트", + language: .korean + )) + } + + func testRejectsShortCommandVerbChanges() { + XCTAssertEqual( + violation("Approve release to production", "Cancel release to production"), + "content_drift" + ) + XCTAssertEqual( + violation("允许现在发布", "禁止现在发布", language: .chinese), + "content_drift" + ) + } + + func testRejectsLongMiddleReplacement() { + let source = String(repeating: "开", count: 600) + + String(repeating: "中", count: 1_000) + + String(repeating: "结", count: 600) + let candidate = String(repeating: "开", count: 600) + + String(repeating: "改", count: 1_000) + + String(repeating: "结", count: 600) + XCTAssertEqual( + violation(source, candidate, language: .chinese), + "content_drift" + ) + } + + func testBalancedParenthesisAtEndOfURLIsProtected() { + XCTAssertEqual( + violation( + "Read https://en.wikipedia.org/wiki/Function_(mathematics)", + "Read https://en.wikipedia.org/wiki/Function_(mathematics" + ), + "protected_token_change" + ) + } + + func testChinesePolarityCoversDisagreementAndNonCorrectionContrast() { + XCTAssertEqual( + violation("我不同意发布", "我同意发布", language: .chinese), + "polarity_change" + ) + XCTAssertEqual( + violation( + "这不是 bug，原因是配置", + "这是 bug，原因是配置", + language: .chinese + ), + "polarity_change" + ) + } + + private func violation( + _ source: String, + _ candidate: String, + language: InputLanguage = .english + ) -> String? { + TranscriptFidelityGuard.violation( + source: source, + candidate: candidate, + protectedTerms: [], + inputLanguage: language, + enforceSemanticFidelity: true + ) + } +} diff --git a/Tests/OpenTypeTests/TranscriptFidelityGuardTests.swift b/Tests/OpenTypeTests/TranscriptFidelityGuardTests.swift new file mode 100644 index 00000000..3e1368eb --- /dev/null +++ b/Tests/OpenTypeTests/TranscriptFidelityGuardTests.swift @@ -0,0 +1,204 @@ +import XCTest +@testable import OpenType + +final class TranscriptFidelityGuardTests: XCTestCase { + func testProtectsEntitiesTermsAndPolarity() { + let source = "不要把 OpenType 2.5 发到 test@example.com 或 /tmp/demo.txt" + XCTAssertNil(violation( + source, + "不要把 OpenType 2.5 发到 test@example.com，或 /tmp/demo.txt。", + terms: ["OpenType"] + )) + XCTAssertEqual( + violation( + source, + "把 OpenType 3.0 发到 test@example.com 或 /tmp/demo.txt", + terms: ["OpenType"] + ), + "protected_token_change" + ) + XCTAssertEqual( + violation( + source, + "不要把 2.5 发到 test@example.com 或 /tmp/demo.txt", + terms: ["OpenType"] + ), + "dictionary_term_change" + ) + XCTAssertEqual( + violation( + source, + "把 OpenType 2.5 发到 test@example.com 或 /tmp/demo.txt", + terms: ["OpenType"] + ), + "polarity_change" + ) + } + + func testAllowsEvidenceBackedSpokenNumberAndSelfCorrectionCleanup() { + XCTAssertNil(violation( + "灰度从百分之二十五到三十", + "灰度从 25%-30%" + )) + XCTAssertNil(violation( + "版本是二点五", + "版本是 2.5" + )) + XCTAssertNil(violation( + "我们周四，不对，周五下午开会", + "我们周五下午开会" + )) + XCTAssertNil(violation( + "唔係星期四，係星期五下晝開會", + "星期五下晝開會", + language: .cantonese + )) + } + + func testNumberEvidenceCannotAuthorizeUnrelatedNumbers() { + XCTAssertEqual( + violation( + "two tasks", + "2 tasks at 999", + language: .english + ), + "protected_token_change" + ) + XCTAssertEqual( + violation( + "版本二", + "版本 999" + ), + "protected_token_change" + ) + } + + func testAllowsEvidencePreservingRangeAndDateFormatting() { + XCTAssertNil(violation("第1到第3步", "第 1-3 步")) + XCTAssertNil(violation( + "日期是2026年7月30日", + "日期是 2026-07-30" + )) + } + + func testRejectsEmptyUnrelatedAndExcessiveOutputs() { + XCTAssertEqual(violation("请把合同发给财务", ""), "empty_output") + XCTAssertEqual( + violation("请把合同发给财务", "明天取消发布窗口"), + "content_drift" + ) + XCTAssertEqual( + violation("确认", "确认后给所有人发送一份完整的发布总结"), + "excessive_expansion" + ) + } + + func testRejectsTokenReorderingAndNegationScopeMovement() { + XCTAssertEqual( + violation("Alice 2, Bob 3", "Alice 3, Bob 2", language: .english), + "protected_token_change" + ) + XCTAssertEqual( + violation( + "Do not deploy to staging; test production.", + "Deploy to staging; do not test production.", + language: .english + ), + "polarity_change" + ) + } + + func testProtectsSentenceFinalEmailOpaquePathsAndQuotes() { + XCTAssertEqual( + violation( + "Email test@example.com.", + "Email test@example.org.", + language: .english + ), + "protected_token_change" + ) + XCTAssertEqual( + violation(#"Open "/tmp/①.txt""#, #"Open "/tmp/1.txt""#, language: .english), + "protected_token_change" + ) + XCTAssertEqual( + violation(#"Open "/tmp/My File.txt""#, "Open /tmp/My File.txt", language: .english), + "protected_token_change" + ) + XCTAssertEqual( + violation("Edit Sources/App/Foo.swift", "Edit Sources/App/Bar.swift", language: .english), + "protected_token_change" + ) + } + + func testDoesNotTreatEnglishNumberAbbreviationAsNegation() { + XCTAssertNil(violation("See No. 5.", "See No. 5.", language: .english)) + } + + func testFillerCleanupDoesNotMoveOrEraseNegation() { + XCTAssertNil(violation( + "Do not, um, deploy.", + "Do not deploy.", + language: .english + )) + XCTAssertEqual( + violation( + "No deploy, but test production.", + "Deploy, but test production.", + language: .english + ), + "polarity_change" + ) + } + + func testFillerDoesNotAuthorizeLargeDeletion() { + XCTAssertNotNil(violation( + "Um, please send the signed contract to finance today.", + "Send contract.", + language: .english + )) + } + + func testBoundedCustomTransformationAllowsOmissionButNotNewFacts() { + XCTAssertNil(violation( + "Release 2 is ready after review", + "Release is ready", + language: .english, + semantic: false + )) + XCTAssertEqual( + violation( + "Release 2 is ready", + "Release 3 is ready", + language: .english, + semantic: false + ), + "protected_token_change" + ) + XCTAssertEqual( + violation( + "Release 2 is ready", + "", + language: .english, + semantic: false + ), + "empty_output" + ) + } + + private func violation( + _ source: String, + _ candidate: String, + terms: [String] = [], + language: InputLanguage = .chinese, + semantic: Bool = true + ) -> String? { + TranscriptFidelityGuard.violation( + source: source, + candidate: candidate, + protectedTerms: terms, + inputLanguage: language, + enforceSemanticFidelity: semantic + ) + } +} diff --git a/Tests/OpenTypeTests/VoicePipelinePolicyTests.swift b/Tests/OpenTypeTests/VoicePipelinePolicyTests.swift index 6a7764df..83aff7a6 100644 --- a/Tests/OpenTypeTests/VoicePipelinePolicyTests.swift +++ b/Tests/OpenTypeTests/VoicePipelinePolicyTests.swift @@ -223,86 +223,4 @@ final class VoicePipelinePolicyTests: XCTestCase { ) } - func testDeferredReplacementOnlyAppliesToSmartFormat() { - XCTAssertTrue(DeferredReplacementPolicy.shouldUseDeferredReplacement(outputMode: .processed, enableInstantInsert: true)) - XCTAssertFalse(DeferredReplacementPolicy.shouldUseDeferredReplacement(outputMode: .processed, enableInstantInsert: false)) - XCTAssertFalse(DeferredReplacementPolicy.shouldUseDeferredReplacement(outputMode: .direct, enableInstantInsert: true)) - XCTAssertFalse(DeferredReplacementPolicy.shouldUseDeferredReplacement(outputMode: .command, enableInstantInsert: true)) - } - - func testDeferredReplacementFailedStateIsNotReplaceable() { - var replacement = DeferredReplacement( - rawText: "raw", - insertedText: "quick", - targetApp: nil, - message: "formatting", - createdAt: Date(timeIntervalSince1970: 100), - expirationInterval: 15 - ) - replacement.state = .failed - - XCTAssertEqual( - DeferredReplacementPolicy.decision( - for: replacement, - currentBundleIdentifier: nil, - now: Date(timeIntervalSince1970: 105) - ), - .copy(.notReady) - ) - } - - func testDeferredReplacementDecisionRequiresSameFrontmostApp() throws { - let replacement = DeferredReplacement( - rawText: "raw", - insertedText: "quick", - targetApp: nil, - message: "formatting", - createdAt: Date(timeIntervalSince1970: 100), - expirationInterval: 15 - ) - var readyReplacement = replacement - readyReplacement.formattedText = "formatted" - readyReplacement.state = .ready - - XCTAssertEqual( - DeferredReplacementPolicy.decision( - for: readyReplacement, - currentBundleIdentifier: nil, - now: Date(timeIntervalSince1970: 105) - ), - .copy(.missingTarget) - ) - - guard let currentBundleIdentifier = NSRunningApplication.current.bundleIdentifier else { - throw XCTSkip("Current test process has no bundle identifier") - } - - readyReplacement = DeferredReplacement( - rawText: "raw", - insertedText: "quick", - targetApp: NSRunningApplication.current, - message: "formatting", - createdAt: Date(timeIntervalSince1970: 100), - expirationInterval: 15 - ) - readyReplacement.formattedText = "formatted" - readyReplacement.state = .ready - - XCTAssertEqual( - DeferredReplacementPolicy.decision( - for: readyReplacement, - currentBundleIdentifier: "other.app", - now: Date(timeIntervalSince1970: 105) - ), - .copy(.appChanged) - ) - XCTAssertEqual( - DeferredReplacementPolicy.decision( - for: readyReplacement, - currentBundleIdentifier: currentBundleIdentifier, - now: Date(timeIntervalSince1970: 116) - ), - .copy(.expired) - ) - } } diff --git a/docs/superpowers/specs/2026-07-30-voice-quality-research.md b/docs/superpowers/specs/2026-07-30-voice-quality-research.md new file mode 100644 index 00000000..93a5fd6b --- /dev/null +++ b/docs/superpowers/specs/2026-07-30-voice-quality-research.md @@ -0,0 +1,635 @@ +# OpenType 语音识别与文本纠错质量调研 + +> 日期：2026-07-30 +> 状态：研究结论 + 第一轮低风险改造，后续仍须用真实语料验收 +> 范围：macOS 26+、Apple Silicon、本地优先，兼顾 Apple Speech、WhisperKit、Qwen3-ASR MLX、火山 ASR 和远程 LLM +> 方法：先审计当前实现，再查阅上游源码、官方文档和论文。第三方 benchmark 仅作为候选筛选依据，不把上游自报成绩当成 OpenType 实测结论。 +> +> “当前实现”指本轮并行改造开始前的代码基线；本轮已经落地的变化应以最终 diff 和测试结果为准。 + +## 1. 结论先行 + +OpenType 和成熟付费听写软件的差距，不是单靠“换一个更大的 ASR 模型”就能消除。改造前基线最明显的损失来自整条链路： + +1. 录音只有音量诊断，没有真正的端点检测、语音段切分、前后缓冲、削波/SNR 质量信号。 +2. WhisperKit 丢弃了时间戳和置信度，只保留最终字符串；流式预览依赖 15 秒滑窗和文本启发式拼接。 +3. 个人词典、屏幕词和最近上下文只主要提供给后处理 LLM，没有充分送入 ASR 解码器；这会直接损失人名、产品名和专业词。 +4. Apple Speech 只用了最基础的 `.transcription` 预设，未获取候选、置信度、时间范围和渐进式结果。 +5. Qwen3-ASR MLX 固定在 `0.1.1`，运行器只传音频和语言；较新的上下文/热词与重复抑制能力尚未利用。 +6. 后处理提示词同时承担“忠实纠错”和“较强润色”，且没有结构化编辑证据、实体/数字保护和自动回退。结果可能更顺，但也更容易改错原意。 +7. 没有可重复的音频基准集和 CER/WER、专名召回、数字保真、静音幻觉、延迟等门禁，无法知道某次优化究竟是整体提升还是只改善了个别样例。 + +最值得优先做的是：**先保留识别证据，再把上下文送到 ASR，随后改造端点/流式稳定器和证据约束的纠错器**。这些工作可以复用现有引擎，不需要先增加一个重量级依赖。 + +### 1.1 本轮已落地 + +这次没有把尚未跑过真实录音的变化写成“准确率已经提升”。已完成的是可由代码和自动测试验证的第一轮： + +- 修复 Whisper `large-v3` 别名和过期完整 model ID 可能静默落到设备默认小模型的问题；最终离线解码临时恢复上游默认的 5 次温度 fallback，实时预览保持 1 次 fallback，并对长音频启用 WhisperKit VAD chunking。 +- 从启用的个人词典生成限量、去重的识别上下文：Apple 的 Dictation fallback 走 `AnalysisContext.contextualStrings`，legacy 走 request `contextualStrings`，Whisper 走有 token 上限的 initial prompt。 +- Apple macOS 26 路径继续优先使用新一代 `SpeechTranscriber(.transcription)`；locale 不支持或分析失败时以重新打开的音频文件回退 `DictationTranscriber`，并在回退路径接入词典、按 60 秒选择 short/long preset。legacy `SFSpeechRecognizer` 同样接入词典并强制 `requiresOnDeviceRecognition`；自动语言跟随当前系统 locale，粤语显式使用 `zh-HK`。 +- Qwen 入参先用 `AVAudioConverter` 最高质量重采样为 16 kHz、单声道、Int16 WAV，避免第三方运行器中的线性插值；临时文件在成功和失败时都会清理。 +- 个人词典改为最长优先、单次扫描、不级联；拉丁词使用词边界，避免把 `api` 错换进 `rapid`，同时保留中文子串匹配。 +- 默认纠错提示改为“有原文、词典或上下文证据才改”，不再要求模型猜漏字或补完未说完的句子；纠错/格式化使用 temperature 0，并按输入长度动态扩大输出预算。 +- 新增整段忠实度 guard：有序保护显式数字、单位、URL、邮箱、文件路径和引号，保护全部启用词典 canonical term，把否定绑定邻近词，并用短句高阈值、全长均匀采样、内容覆盖率和长度比拦截空输出、无关改写、异常扩写和大段删除。当前只对有确定解析证据的中英文口述数字、日期/范围和韩语序数放行 ITN；口述邮箱、URL、path 仍保持保守拒绝，等待结构化解析器。普通处理的 guard 失败回退本次预处理文本；已有 quick insert 的延迟替换则标记失败，不覆盖已插入文本。 +- 同一请求冻结模型、语言、远程端点、自定义 prompt、个人词典和编辑规则；VLM fallback 与延迟纠错复用同一快照，multimodal 截图同时保留 OCR 文本供 VLM 失败后使用。远程兼容端若明确返回 token/context 上限错误，会缩小输出预算重试一次；`finish_reason=length` / `stop_reason=max_tokens` 一律视为失败，不接受截断正文。 +- 无效本地 LLM 配置回退到既定 2B 默认模型，而不是列表首个 0.8B 小模型。 +- 新增无第三方依赖的 `scripts/evaluate-voice-quality.py`，分别评测 raw ASR 和 processed text 的 CER/WER、术语、数字/URL/email/path 有序 exact 保真（同时惩罚新增、删除和交换）、静音幻觉和 p50/p95 延迟；schema 示例见 `voice-quality-corpus.example.jsonl`。 + +尚未完成、也不能靠单元测试替代的部分：真实用户录音 baseline、`TranscriptEvidence`、候选/置信度与时间戳、端点 pre/post-roll、LocalAgreement、低置信二次识别，以及由声学候选支持的实体校验、结构化 edits 和按句/segment 回退。本轮 guard 是确定性安全网，不等于已经理解整句语义。这些仍是下一轮决定能否接近付费软件体验的关键。 + +## 2. 改造前基线审计 + +| 链路 | 当前行为 | 主要缺口 | 质量影响 | +|---|---|---|---| +| 音频采集 | `AudioCaptureManager` 用 `AVAudioEngine` 录制，流式侧转 16 kHz mono；已有 RMS 诊断 | 无正式 VAD、端点、pre-roll/post-roll、削波率、噪声底、SNR；没有 raw/voice-processing A/B | 开头/结尾音素被切、长静音诱发幻觉、远场和回声恶化 | +| Whisper 模型 | 默认名为 `large-v3`，不支持时做模糊匹配 | 没有锁定具体质量模型构建；模型变化不可审计 | 同名映射或 catalog 变化后结果漂移 | +| Whisper 解码 | `temperatureFallbackCount: 1`、`withoutTimestamps: true`、固定中文 prompt“以下是普通话的句子。” | 上游默认 fallback 次数是 5；时间戳、词概率、segment logprob、no-speech 信号均被浪费；prompt 不含用户词汇 | 难词召回低、无法判定不确定片段、重复/幻觉只能事后猜 | +| Whisper 实时预览 | 0.7 秒调度、至少 1 秒才开始、反复识别最近 15 秒，再按字符串启发式合并 | 没有稳定/不稳定词边界、时间戳对齐和确认策略 | partial 抖动、丢词、重词、长听写接缝不稳 | +| Apple Speech | `SpeechTranscriber(..., preset: .transcription)`，最终只拼字符串 | 未用 AnalysisContext、alternatives、confidence、time range、progressive preset | 没有词典加权，也无法针对低置信片段复核 | +| Qwen3-ASR MLX | 固定 `qwen3-asr-mlx==0.1.1`；只传 `audio` 和 `language` | 未用新版 context/hotwords、重复惩罚、短音频重试 | 中文专名、代码混说和重复文本仍有可用优化空间 | +| 火山 ASR | 已启用 `enable_itn` 和 `enable_punc` | 未接 `boosting_table_id/name` | 个人/团队术语没有进入云端解码器 | +| MiMo | UI/运行时允许用户配置本地 MiMo | 上游官方栈要求 Linux、Python 3.12、CUDA 12+、FlashAttention | 不应作为原生 Mac 的稳定默认方案 | +| 后处理 | 个人词典替换 + 屏幕/历史/词典上下文 + 低温 LLM；有基础尾部清理 | 提示词要求“力度高于轻度润色”；无候选/置信度输入、无严格 JSON 编辑、无实体/数字/URL/path guard | 可读性提升和语义篡改混在一起，难以自动兜底 | +| 评测 | 有提示词和文本清理相关测试 | 无真实音频语料、CER/WER、静音集、专名/数字精确率和延迟基线 | 无法科学选择模型和参数 | + +对应代码入口： + +- `Sources/Speech/WhisperEngine.swift` +- `Sources/Speech/WhisperStreamingSession.swift` +- `Sources/Speech/AppleSpeechAnalyzer.swift` +- `Sources/Speech/VolcSpeechEngine.swift` +- `Sources/Speech/LocalASRRuntime.swift` +- `Sources/Resources/Scripts/local-asr-runner.py` +- `Sources/Audio/AudioCaptureManager.swift` +- `Sources/Processing/TextProcessor.swift` +- `Sources/Prompts/PromptCatalog.swift` + +## 3. 分阶段路线 + +### 3.1 本轮可直接落地 + +这里的“本轮”指不更换产品架构、能围绕现有 Swift/WhisperKit/Apple/Qwen 代码完成的改造。 + +| 优先级 | 改造 | 预期收益 | 工作量 | 风险与验收 | +|---|---|---|---|---| +| P0 | 建立小型音频基准与统一指标 | 后续每个参数变化都有证据 | 中 | 先产出 baseline；没有 baseline 不合入模型/参数变更 | +| P0 | 把引擎返回值从 `String` 升级为 `TranscriptEvidence` | 支撑低置信复核、候选纠错、流式稳定 | 中 | 所有引擎先允许字段为空，避免一次性重写 | +| P0 | Whisper 显式质量档：锁定模型、恢复可调 fallback、保留 timestamps/word probability/no-speech | 直接改善难音频并让异常可检测 | 小到中 | 用短于 1 秒、静音、长句验证；不可只比较几个口述样例 | +| P0 | 将个人词典/屏幕高价值词送入 Apple、Whisper、Qwen、火山各自的 context/hotword 接口 | 人名、产品名、专业词通常是感知质量最大的短板 | 中 | 限量、去重、净化；必须测普通词误吸附和重复 | +| P0 | Apple Speech 使用 alternatives/confidence/time range；实时预览使用 progressive preset | 免费获得第二候选与不确定性 | 中 | 仅 macOS 26 路径；保留 legacy fallback | +| P0 | VAD 先用于端点与切段，保留 200–300 ms 前后缓冲；增加削波/静音/噪声指标 | 减少静音幻觉、丢首尾和超长音频退化 | 中 | VAD 不得直接删掉疑似语音而无回退 | +| P0 | 用时间戳 LocalAgreement 替换字符串式流式拼接 | 大幅降低 partial 抖动、重复和接缝错误 | 中 | 记录 partial churn、确认延迟、最终一致性 | +| P0 | 将纠错拆为“忠实纠错”和“表达润色”，默认忠实；加结构化 edits 和保护项 diff guard | 减少 LLM 自作主张改数字、实体和否定词 | 中到大 | 任何不受支持的实词改动可按句回退到 ASR | +| P1 | Qwen3-ASR MLX canary 升级至 `0.2.0`，接 context、repetition penalty | 中文和混说专名可能明显提升 | 小到中 | 上游较新，先旁路比较；不能直接全量替换 | +| P1 | 对低置信片段才启动第二引擎/第二次解码 | 提升难句但避免全程 2 倍延迟 | 中到大 | 需定义触发阈值和延迟预算 | + +推荐依赖顺序： + +```text +基准与观测 + → TranscriptEvidence + → ASR 上下文 + Apple 候选/置信度 + → VAD/LocalAgreement + → 证据约束纠错 + → 低置信二次识别 +``` + +### 3.2 后续路线 + +这些方向有价值，但不应挤在第一轮里： + +1. **Apple 自定义语言模型**：对稳定的个人/行业短语和特殊发音进行本地适配。编译与管理成本高于 `contextualStrings`，应在后者效果有基线后再做。 +2. **FluidAudio/Parakeet 英文引擎**：原生 Swift/Core ML、许可友好，适合作为英文旁路 benchmark；目前不是中文主方案。 +3. **确定性 ITN 引擎**：接 `text-processing-rs` 处理数字、日期、金额、单位；必须按语言开启，并保护 URL、代码、路径和用户原始数字格式。 +4. **中文多引擎 ensemble**：Qwen/Whisper/Apple 仅在风险片段竞争，用词典、候选和语言模型打分，而不是全程并行。 +5. **FunASR/WeNet 架构验证**：它们证明了 VAD + ASR + 标点 + ITN + 热词的生产链路，但 Python/C++、模型许可和包体不适合直接成为当前 macOS 第一选择。 +6. **显式用户纠正学习**：用户修改后建议加入个人词典或 spoken-form 映射；必须由用户确认，不能把一次修改静默学习为永久规则。 + +## 4. 建议的识别证据模型 + +当前协议 `SpeechEngineProtocol.transcribe(...) -> String` 把后续质量优化所需的信息全部抹平。建议兼容式增加： + +```swift +struct TranscriptEvidence: Sendable { + let text: String + let language: String? + let engine: String + let segments: [TranscriptSegment] + let alternatives: [TranscriptAlternative] + let diagnostics: TranscriptDiagnostics +} + +struct TranscriptSegment: Sendable { + let text: String + let startSeconds: Double? + let endSeconds: Double? + let confidence: Double? + let averageLogProbability: Double? + let noSpeechProbability: Double? + let words: [TranscriptWord] +} +``` + +迁移方式： + +1. 保留现有 `transcribe(...) -> String` 作为默认适配器。 +2. 新增 `transcribeEvidence(...)`；暂时不支持证据的引擎只填 `text`。 +3. Apple 先填 alternatives/confidence/time range；WhisperKit 填 word probabilities、segments、logprob/no-speech；Qwen/火山没有统一置信度时先填诊断与空数组。 +4. `TextProcessor` 接收证据对象，但用户选择“不做 AI 后处理”时仍原样输出 `text`。 + +统一证据后可以实现三个关键策略： + +- 高置信、无风险片段跳过昂贵的 LLM 或第二引擎。 +- 低置信专名只允许在候选、个人词典、屏幕词中选择。 +- 高 no-speech、重复率异常或静音占比高时，不让 LLM 把幻觉润色成看似合理的长句。 + +## 5. 音频前端：先端点，再谈降噪 + +### 5.1 应立即加入的音频观测 + +每次录音仅本地记录聚合指标，不保存隐私音频： + +- 总时长、有效语音时长、前/后静音时长。 +- RMS、峰值、削波采样比例。 +- 噪声底估计、粗略 SNR。 +- 输入路由、采样率、声道数。 +- VAD 段数、最长静音、丢弃/保留的边界长度。 +- ASR real-time factor、首个 partial 延迟、最终延迟。 + +这些指标可直接解释“麦太小”“蓝牙通话模式”“背景视频”“录音全是静音”等问题，避免所有失败都归因于模型。 + +### 5.2 VAD/端点策略 + +第一版优先复用 WhisperKit 已有的 EnergyVAD 或 Apple `SpeechDetector`，不急于引入新的神经网络运行时： + +1. 保持一个至少 200–300 ms 的环形 pre-roll。 +2. 语音开始后不因一个低能量 frame 就断句。 +3. 连续静音达到阈值后关闭一个语音段，但保留 post-roll。 +4. 很短的停顿只作为分词线索，不立即提交最终结果。 +5. 长听写按已确认的静音边界切成约 15–30 秒段；不能在一个词中间硬切。 +6. VAD 判断不确定时保留音频，宁可多送一点静音，也不要不可逆丢音。 + +[Silero VAD](https://github.com/snakers4/silero-vad)（MIT）的参考实现使用 16 kHz 下 512-sample 窗、双阈值回滞、最短语音/静音、speech padding 和最长段切分；其[精确实现](https://github.com/snakers4/silero-vad/blob/master/src/silero_vad/utils_vad.py)适合作为参数设计参考。它不是第一轮必须引入的依赖。 + +Apple 明确提醒 VAD 会在某些内容上降低识别质量，因此 `SpeechDetector` 也必须通过语料 A/B，而不是默认认定更“智能”就更准：[SpeechDetector 官方文档](https://developer.apple.com/documentation/speech/speechdetector)。 + +### 5.3 Apple Voice Processing 只做特性开关 + +Apple 的 Voice Processing 提供回声消除、噪声抑制和自动增益，官方入口是 [`setVoiceProcessingEnabled`](https://developer.apple.com/documentation/avfaudio/avaudioionode/setvoiceprocessingenabled%28_%3A%29)，背景见 [WWDC23 Optimize voice processing for spatial audio](https://developer.apple.com/videos/play/wwdc2023/10235/)。 + +但该处理主要面向语音通信。降噪和 AGC 可能同时损伤 ASR 需要的辅音、气声或远场特征。因此建议： + +- 仅在 engine stopped 时切换，符合 API 约束。 +- `raw` 与 `voiceProcessed` 作为可 A/B 的录音 profile。 +- 先按内置麦、AirPods、USB 麦、扬声器回声四类设备测 CER/WER。 +- 不在第一轮加入通用 spectral gate；只有基准证明净收益时才启用。 + +## 6. WhisperKit 优化 + +### 6.1 锁定模型与解码 profile + +WhisperKit 当前源码中的 `DecodingOptions` 默认 fallback 次数为 5，并提供 word timestamps、no-speech、compression ratio、logprob 等阈值。可核查其 [MIT 许可仓库](https://github.com/argmaxinc/argmax-oss-swift)和[当前配置源码](https://github.com/argmaxinc/argmax-oss-swift/blob/8fcbfed028415b0b90f0f10ee7b0303c53b600a0/Sources/WhisperKit/Core/Configurations.swift)。 + +建议新增显式 profile，而不是散落布尔值： + +| Profile | 用途 | 建议 | +|---|---|---| +| `fastPreview` | 实时 partial | 较小模型/较少 fallback，必须有 timestamps | +| `balancedFinal` | 默认最终文本 | 锁定具体模型构建，fallback 2–3 起步，语料调参 | +| `qualityFinal` | 用户主动选择最高质量 | `large-v3` 质量构建，允许更多 fallback 和更长延迟 | + +关键改造： + +- 模型 catalog 中存确切 model ID，不再用首个模糊匹配。Argmax 当前列出的多语言质量候选包括 `large-v3-v20240930_626MB`；最终仍以本项目语料选择。 +- 本轮最终离线识别临时恢复 WhisperKit 上游默认值 5，实时 partial 保持 1，避免每个预览窗最坏执行 6 次解码；这不等于 OpenType 已实测更优，仍须分别测 1、2、3、5 的错误率和 p95 延迟，再决定产品默认值。 +- 开启 segment/word timestamps，保留 word probability、segment logprob、no-speech probability。 +- 增加静音和音乐样例调 `noSpeechThreshold`、`logProbThreshold`、`compressionRatioThreshold`，不要让 LLM 接管幻觉检测。 +- 当前上游 `windowClipTime` 默认是 1 秒；先为 `<1s`、`1–2s` 口令建立回归用例，若确认有跳过风险，再显式降低/关闭该 cutoff。 +- `suppressTokens` 需要按实际重复/异常 token 验证，不能复制网上的通用 token 列表。 + +WhisperKit 的 fallback、语言检测和时间戳生成流程可直接从 [`TranscribeTask.swift`](https://github.com/argmaxinc/argmax-oss-swift/blob/8fcbfed028415b0b90f0f10ee7b0303c53b600a0/Sources/WhisperKit/Core/TranscribeTask.swift)核对。 + +### 6.2 Prompt 应是词汇提示，不是编辑指令 + +Whisper prompt 最适合提供“可能出现的前文、专名和拼写样例”，不是要求模型“润色”“不要出错”。whisper.cpp 维护者在[初始 prompt 讨论](https://github.com/ggerganov/whisper.cpp/discussions/348)中也强调 prompt 会强烈影响词汇、标点和风格，因此存在误吸附风险。 + +推荐生成一个有预算的 `ASRContext`： + +1. 个人词典中的 preferred spelling 和 spoken form。 +2. 当前窗口 OCR 中高置信、短且稀有的产品名/人名。 +3. 已确认的最近一两句，不包含未确认 partial。 +4. 语言标签。 + +禁止送入： + +- 完整屏幕 OCR。 +- 旧的长篇历史。 +- 删除规则、系统指令或“请修正文法”。 +- 邮箱、令牌、密钥等敏感内容。 + +对 Whisper，prompt 长度应很短且每段重新滚动；对 Apple/Qwen/火山则映射到各自原生接口。 + +### 6.3 使用 VAD chunking + +WhisperKit 已有 [EnergyVAD](https://github.com/argmaxinc/argmax-oss-swift/blob/8fcbfed028415b0b90f0f10ee7b0303c53b600a0/Sources/WhisperKit/Core/Audio/EnergyVAD.swift)和 [AudioChunker](https://github.com/argmaxinc/argmax-oss-swift/blob/8fcbfed028415b0b90f0f10ee7b0303c53b600a0/Sources/WhisperKit/Core/Audio/AudioChunker.swift)。第一轮可以先复用其公开能力，避免另带 ONNX/Core ML VAD。 + +用途应是： + +- 切开长静音和长音频。 +- 产生 endpoint 候选。 +- 给 no-speech 与 hallucination 检测补充信号。 + +不要用 VAD 把低能量词直接永久删除。 + +## 7. 流式识别：从字符串拼接改为 LocalAgreement + +当前 15 秒尾窗反复解码后按文本合并，无法知道某个词在音频中的位置。更稳妥的开源参考是 [WhisperStreaming](https://github.com/ufal/whisper_streaming)（MIT）的 LocalAgreement 策略，其核心 [`HypothesisBuffer`](https://github.com/ufal/whisper_streaming/blob/6da90b44b7e50d79695e68166d2a2c7609c75abb/whisper_online.py)： + +1. 保存上一轮和本轮带时间戳的词。 +2. 只提交两轮最长公共词前缀。 +3. 新 hypothesis 与已提交边界附近做 n-gram 去重。 +4. 未稳定词留在 unstable buffer，下一轮可改写。 +5. 约 15 秒或已确认句界处裁剪音频缓存。 +6. 只把约 200 字符的已确认前文作为下一窗 prompt。 + +这套算法可以原生移植成 Swift 数据结构，不需要把 Python 项目嵌入应用。上游论文/README 报告约 3.3 秒延迟属于其环境结果，不代表 OpenType 能直接复现。 + +建议 UI 同时维护： + +- `committedText`：两轮一致、不会再回滚。 +- `unstableText`：灰色或次级展示，允许变化。 +- `audioCursor`：最后已提交的音频时间。 + +验收指标不能只看最终文本，还要测： + +- partial churn：同一位置被改写的次数。 +- rollback characters：用户看见后又消失的字符数。 +- commit latency：一个词说完到稳定提交的时间。 +- final agreement：流式最终与离线最终的差异。 + +更新更激进的研究参考可看 [SimulStreaming](https://github.com/ufal/SimulStreaming)（MIT）；它依赖 PyTorch，适合作为算法参考，不适合作为当前 macOS 运行时依赖。 + +## 8. Apple Speech：利用 macOS 26 已有能力 + +Apple 的 [SpeechAnalyzer](https://developer.apple.com/documentation/speech/speechanalyzer)正好匹配本项目最低系统版本。本轮保留 `SpeechTranscriber(.transcription)` 作为首选高质量路径，只在 locale 不支持或分析失败时回退 short/long `DictationTranscriber`；`AnalysisContext.contextualStrings` 按 Apple 的支持边界接在 Dictation fallback。下一步仍建议把首选路径拆成： + +- 实时预览：[`progressiveTranscription`](https://developer.apple.com/documentation/speech/speechtranscriber/preset/progressivetranscription)。 +- 最终识别：[`transcriptionWithAlternatives`](https://developer.apple.com/documentation/speech/speechtranscriber/preset/transcriptionwithalternatives)，或需要时间范围时用 [`timeIndexedTranscriptionWithAlternatives`](https://developer.apple.com/documentation/speech/speechtranscriber/preset/timeindexedtranscriptionwithalternatives)。 +- 自定义构造时请求 alternative transcriptions，以及 `audioTimeRange`、[`transcriptionConfidence`](https://developer.apple.com/documentation/speech/speechtranscriber/resultattributeoption/transcriptionconfidence) 属性。 + +Apple 官方 [WWDC25 Bring advanced speech-to-text to your app with SpeechAnalyzer](https://developer.apple.com/videos/play/wwdc2025/277/)展示了 analyzer、asset 管理与渐进式结果的完整路径。 + +### 8.1 `AnalysisContext.contextualStrings` + +[`AnalysisContext.contextualStrings`](https://developer.apple.com/documentation/speech/analysiscontext/contextualstrings)允许提供最多 100 个短语，官方建议使用一到两个词的短短语，并可按 tag 分组。推荐预算： + +| 来源 | 数量建议 | 示例 | +|---|---:|---| +| 用户固定个人词典 | 40 | OpenType、Roleva、人名 | +| 当前屏幕高价值词 | 30 | 当前应用、文档标题、代码符号 | +| 最近确认上下文 | 20 | 当前主题专名 | +| 预留 | 10 | 应用命令、临时词 | + +必须去重、按语言过滤、限制长度，并避开普通单字/常用词；否则上下文会把相似声音错误吸向热词。 + +### 8.2 自定义语言模型放到第二阶段 + +Apple 的 [`SFCustomLanguageModelData`](https://developer.apple.com/documentation/speech/sfcustomlanguagemodeldata)支持 [`PhraseCount`](https://developer.apple.com/documentation/speech/sfcustomlanguagemodeldata/phrasecount)和 [`CustomPronunciation`](https://developer.apple.com/documentation/speech/sfcustomlanguagemodeldata/custompronunciation)，完整流程见 [WWDC23 Customize on-device speech recognition](https://developer.apple.com/videos/play/wwdc2023/10101/)。 + +适合： + +- 稳定的产品词库。 +- 用户反复纠正的人名。 +- 非标准读音和缩写。 + +不适合每次录音动态重建。第一阶段先验证 `contextualStrings`，只有频繁术语仍明显漏识别时再引入模型编译和版本管理。 + +## 9. Qwen3-ASR、火山和 MiMo + +### 9.1 Qwen3-ASR MLX + +[Qwen3-ASR 官方仓库](https://github.com/QwenLM/Qwen3-ASR)代码为 Apache-2.0，官方 [`qwen3_asr.py`](https://github.com/QwenLM/Qwen3-ASR/blob/7c6daf77a2421100f5fb066495372c00129d39ff/qwen_asr/inference/qwen3_asr.py)支持 context 和语言控制；模型覆盖和 benchmark 数字是上游自报，应在本项目语料复测。 + +OpenType 实际依赖社区 [qwen3-asr-mlx](https://github.com/gabrimatic/qwen3-asr-mlx)（MIT）。其 [PyPI `0.2.0`](https://pypi.org/project/qwen3-asr-mlx/) 于 2026-07-24 发布并增加 context/hotword 相关能力；当前项目基线仍固定 `0.1.1`。其[模型实现](https://github.com/gabrimatic/qwen3-asr-mlx/blob/7f8b420ad4188c6a2be038ba8e24d9f00cfedb72/src/qwen3_asr_mlx/model.py)暴露 greedy/temperature、repetition penalty 和 context。 + +建议用 canary 方式升级： + +1. 更新独立运行时版本和 marker，不破坏旧 runtime 回退。 +2. 请求协议新增短 `context`，来源只取净化后的个人词典与顶部 OCR 词。 +3. 默认确定性解码，测试 `repetition_penalty` 约 1.2 的候选值。 +4. 短音频返回空时允许一次低温重试，但不能无限重试。 +5. 记录 context 命中、重复率和普通词误吸附。 + +可直接参考同作者 [local-whisper 的 context builder](https://github.com/gabrimatic/local-whisper/blob/6ff31ffb99cfce6858ddf50f259ac1109010beb5/src/whisper_voice/engines/context.py)（MIT）：长度预算、去重、preferred spelling/spoken form、排除删除规则。其 [Qwen wrapper](https://github.com/gabrimatic/local-whisper/blob/6ff31ffb99cfce6858ddf50f259ac1109010beb5/src/whisper_voice/engines/qwen3_asr.py)还展示了 context 和重复惩罚的调用方式。 + +风险：Qwen 上游已有[热词导致重复的报告](https://github.com/QwenLM/Qwen3-ASR/issues/140)。因此不能把整页 OCR 或全部历史直接塞给模型。 + +### 9.2 火山 ASR + +火山官方[热词文档](https://www.volcengine.com/docs/6561/155739?lang=zh)支持请求传 `boosting_table_id` 或 `boosting_table_name`。官方约束包括最多 5000 词、单词长度和权重范围，并明确提醒常用单字/短语可能损伤整体准确率；[FAQ](https://www.volcengine.com/docs/6561/155743?lang=zh)说明热词本质是提高解码概率，效果依赖音频和基础模型。 + +本轮可做： + +- 在 provider 配置中增加可选 hotword table ID/name。 +- 请求体按用户配置传值。 +- 不自动上传/修改云端热词表；这需要额外授权和账号权限设计。 +- 文档提示热词表应放专名，不放常用词。 + +火山是专有云服务，不属于开源依赖。 + +### 9.3 MiMo + +[MiMo-V2.5-ASR 官方仓库](https://github.com/XiaomiMiMo/MiMo-V2.5-ASR)代码为 Apache-2.0，但上游安装要求是 Linux、Python 3.12、CUDA 12+ 和 FlashAttention。当前 macOS Apple Silicon 无官方原生推理路径。 + +产品上应： + +- 标记为“实验性/外部运行时”，不作为推荐的本地 Mac 引擎。 +- 不在质量主路线中投入大量兼容修补。 +- 如果将来出现经验证的 Metal/Core ML 端口，再单独做许可证、模型权重和质量评估。 + +## 10. 纠错后文本：从自由改写改为证据约束 + +### 10.1 拆成两种用户意图 + +默认模式应为： + +**忠实纠错** + +- 补标点、断句、大小写、明确的 ITN。 +- 清理嗯、啊、重复起句和说话者已完成的自我纠正。 +- 不改变否定、时态、数字、实体、术语和因果关系。 +- 实词替换必须有候选、词典、屏幕词或高可信上下文支持。 + +另设显式模式： + +**表达润色** + +- 允许重排语序、压缩冗余、改变格式。 +- 仍保护数字、专名、URL、邮箱、路径、代码 token 和引用。 +- 用户清楚知道这是改写，不把它包装成“识别原文”。 + +改造前中文 system prompt 要求“力度要高于轻度润色”。本轮已把默认提示改为忠实纠错，并为专业风格保留表达整理；产品上仍应进一步拆成两个显式模式，让用户知道何时允许自由改写。 + +### 10.2 三阶段流水线 + +```text +ASR evidence + → deterministic normalization + → evidence-constrained LLM correction + → semantic/protected-token validator + ├─ pass: output + └─ fail: per-sentence fallback or raw transcript +``` + +第一阶段只做可审计的确定性操作： + +- Unicode/空白规范化。 +- 引擎已知占位符清理。 +- 有边界的个人词典 exact replacement。 +- 语言明确时的 ITN。 +- 不处理语义不确定的同音词。 + +第二阶段要求严格结构化输出，例如： + +```json +{ + "final_text": "把 OpenType 2.5 的发布改到 8 月 3 日。", + "edits": [ + { + "source": "open type", + "replacement": "OpenType", + "reason": "personal_dictionary", + "evidence_id": "dictionary:42" + } + ], + "uncertain_spans": [] +} +``` + +允许的 `reason` 是枚举，而不是自由发挥： + +- `punctuation` +- `filler` +- `self_correction` +- `itn` +- `asr_alternative` +- `personal_dictionary` +- `screen_context` +- `formatting` + +第三阶段独立验证： + +- 所有 edit 的 source span 必须能在原文对齐。 +- 新增/删除的数字、百分比、日期、金额、计量单位必须一致。 +- URL、邮箱、文件路径、代码 token、版本号默认逐字符保护。 +- 否定词（不、没、未、不要、cannot 等）变化触发回退。 +- 未被 evidence 支持的人名/机构/产品实词替换触发回退。 +- 输出为空、异常变长、重复 n-gram 或语言突变时回退。 + +本轮已经落地其中的第一层整段 guard：保护有序数字/单位/URL/email/path、全部启用词典术语和否定邻近词，并检查短句及长文本的内容覆盖率与长度异常；确定性口述数字证据可放行有限 ITN。口述邮箱/URL/path 生成、严格 `edits` schema、声学候选对齐、重复 n-gram/语言突变检测和按句回退尚未实现，因此不能把它描述成完整的语义验证器。 + +回退粒度优先是句子/segment，不要因为一句风险把整段优质纠错全部丢掉。 + +### 10.3 N-best 和置信度的正确用途 + +[HyPoradise/Hypo2Trans](https://github.com/Hypotheses-Paradise/Hypo2Trans)（MIT）及其[论文](https://arxiv.org/abs/2309.15701)研究用 N-best 假设做 ASR 后纠错。其公开结果中也有部分 domain 被 LLM 改差，这正说明“总是让大模型重写一遍”不是安全策略。 + +后续研究进一步指出 N-best 提供更多声学证据，而不受约束的生成在未知 domain 可能退化：[Towards Robust and Generalizable ASR Error Correction](https://arxiv.org/abs/2409.09554)。 + +在 OpenType 中应这样用： + +1. Apple alternatives、Whisper 候选/低置信词先形成 evidence set。 +2. LLM 只在低置信 span 中从候选、词典或发音相近项选择。 +3. 高置信文本只补标点/格式，减少无意义改写。 +4. 两个引擎分歧很大时标为 uncertain，不要强行“猜一个更顺的”。 + +### 10.4 ITN 与标点 + +[text-processing-rs](https://github.com/FluidInference/text-processing-rs)（Apache-2.0）提供 Rust 实现和 Swift XCFramework wrapper，声称兼容 NeMo 的 TN/ITN 规则并覆盖中、英、日等语言。它比 Python/Pynini 更适合原生 macOS，但第一轮仍应做独立 spike： + +- 只在明确语言启用。 +- 测试中文日期、金额、百分比、电话号码、版本号。 +- URL、路径、代码段跳过 ITN。 +- 规则错误时允许保留 spoken form。 + +原始 [NeMo text processing](https://github.com/NVIDIA/NeMo-text-processing)为 Apache-2.0，但 Pynini/Linux 依赖不适合直接嵌入当前应用。 + +标点优先顺序： + +1. ASR 原生标点。 +2. 忠实 LLM 断句。 +3. 大型独立标点模型仅在前两者基准仍不够时考虑。 + +## 11. 低置信二次识别 + +成熟产品常见优势不是“永远运行两个最大模型”，而是只为困难片段花额外算力。 + +建议风险分： + +```text +risk = + low average logprob + + high no-speech probability + + low Apple confidence + + engine disagreement + + repeated n-grams + + expected dictionary term missing + + poor audio SNR / clipping +``` + +策略： + +- `low risk`：直接确定性规范化，必要时轻量纠错。 +- `medium risk`：同一引擎质量 profile 重解码，增加短 context。 +- `high risk`：只重识别对应音频 segment；调用第二引擎或 Apple alternatives。 +- `silence/hallucination risk`：拒绝生成长文本，提示未检测到清晰语音。 + +第二引擎候选： + +- 中文：Qwen3-ASR MLX 与 Whisper/Apple 互补。 +- 英文：[FluidAudio](https://github.com/FluidInference/FluidAudio)（Apache-2.0）的 Core ML Parakeet 可作为旁路 benchmark。其[模型支持表](https://github.com/FluidInference/FluidAudio/blob/main/Documentation/Models.md)显示 Parakeet v2 主要是英文、v3 是多种欧洲语言，不应被误当成中文主引擎。 +- 云端：用户已配置火山时可作为高质量候选，但必须遵守隐私和网络设置。 + +## 12. 评测方案 + +### 12.1 语料 + +先做 200–500 条版本化、用户授权、本地保存的短语料；CI 只放可公开/合成的极小子集。至少覆盖： + +| 维度 | 必测样例 | +|---|---| +| 语言 | 普通话、英语、粤语（若支持）、中英 code-switch | +| 设备 | 内置麦、AirPods/蓝牙、USB 麦、远场 | +| 环境 | 安静、风扇、咖啡店、键盘声、扬声器回声、背景视频 | +| 时长 | `<1s`、1–10 秒、30–120 秒 | +| 术语 | 人名、公司、产品、缩写、技术词、个人词典命中/不命中 | +| 保真 | 数字、日期、金额、百分比、单位、URL、邮箱、代码、文件路径 | +| 口语 | filler、重复、自我纠正、口述标点、犹豫 | +| 负样例 | 全静音、音乐、环境声、极低音量、削波 | + +每条音频需要两份 gold： + +1. `faithful_reference`：忠实记录说了什么，用于原始 ASR。 +2. `sendable_reference`：用户认可的可直接发送文本，用于纠错后质量。 + +不能用“sendable 文本”计算原始 ASR CER，否则模型正确保留口语词反而被算错。 + +### 12.2 指标 + +[JiWER](https://github.com/jitsi/jiwer)（Apache-2.0）可计算 WER、MER、WIL、WIP、CER；v4 对空 reference 有定义，因此能直接量化静音音频产生的插入幻觉。 + +最低指标集： + +- 中文 CER。 +- 英文 WER。 +- 中英混合的字符/词混合错误率。 +- 专名召回率与 preferred spelling exact match。 +- 数字/日期/金额/单位 exact match。 +- URL/email/path/code token exact match。 +- 静音插入率和 hallucinated characters/minute。 +- 重复 n-gram 率。 +- 标点 F1。 +- 后处理前后 semantic preservation 人审评分。 +- 用户最终编辑距离/删除字符数/重新口述率。 +- p50/p95 首 partial 延迟、最终延迟、real-time factor。 +- partial churn、rollback characters、commit latency。 +- 内存峰值、功耗和包体。 + +仓库中的第一版 evaluator 已实现 raw/processed CER、英文 WER、术语 exact match、空 reference 幻觉、延迟，以及数字/URL/email/path 的逐条有序 exact 比较；多出来、丢失和交换的 token 都计错，同时保留 multiset precision/recall 统计。真实音频、标点 F1、semantic 人审、partial churn、功耗等仍需后续采集。 + +### 12.3 人审 + +对至少 50 条难例做匿名、随机顺序、成对对比： + +- 语义忠实。 +- 可读性。 +- 专业词准确。 +- 数字和实体保真。 +- 格式是否可直接发送。 + +评审者不能知道引擎/参数。保留“二者相同/都不好”，避免强迫选边。 + +### 12.4 合入门禁 + +第一版可采用保守门禁： + +- 总体 CER/WER 不得显著退化。 +- 专名召回提升不能以普通词误吸附明显上升为代价。 +- 数字、URL、路径 exact match 不得因 LLM 纠错下降。 +- 静音幻觉必须下降或持平。 +- p95 最终延迟和内存必须在产品预算内。 +- 所有 `<1s`、空音频、长静音和长听写接缝测试通过。 + +等 baseline 建立后，再把“显著退化”和延迟预算固化为具体数值。 + +## 13. 可复用开源代码清单 + +| 项目 / 精确代码 | 许可证 | 可复用内容 | 对 OpenType 的可移植性 | 建议 | +|---|---|---|---|---| +| [WhisperKit](https://github.com/argmaxinc/argmax-oss-swift) / [`Configurations.swift`](https://github.com/argmaxinc/argmax-oss-swift/blob/8fcbfed028415b0b90f0f10ee7b0303c53b600a0/Sources/WhisperKit/Core/Configurations.swift) | MIT | 解码选项、阈值、timestamps、fallback | 高，已依赖 | 直接使用 | +| WhisperKit / [`EnergyVAD.swift`](https://github.com/argmaxinc/argmax-oss-swift/blob/8fcbfed028415b0b90f0f10ee7b0303c53b600a0/Sources/WhisperKit/Core/Audio/EnergyVAD.swift) / [`AudioChunker.swift`](https://github.com/argmaxinc/argmax-oss-swift/blob/8fcbfed028415b0b90f0f10ee7b0303c53b600a0/Sources/WhisperKit/Core/Audio/AudioChunker.swift) | MIT | 能量 VAD、音频切段 | 高 | 第一轮复用 | +| [whisper.cpp](https://github.com/ggml-org/whisper.cpp) / [stream example](https://github.com/ggml-org/whisper.cpp/blob/master/examples/stream/README.md) | MIT | rolling prompt、VAD、beam/streaming 参数参考 | 中 | 算法参考，不新增第二套 Whisper runtime | +| [WhisperStreaming](https://github.com/ufal/whisper_streaming) / [`HypothesisBuffer`](https://github.com/ufal/whisper_streaming/blob/6da90b44b7e50d79695e68166d2a2c7609c75abb/whisper_online.py) | MIT | LocalAgreement、时间戳去重、buffer trim | 高，算法可重写为 Swift | 第一轮移植 | +| [SimulStreaming](https://github.com/ufal/SimulStreaming) | MIT | AlignAtt/LocalAgreement 新版流式策略 | 低到中，PyTorch-centric | 后续研究 | +| [Silero VAD](https://github.com/snakers4/silero-vad) / [`utils_vad.py`](https://github.com/snakers4/silero-vad/blob/master/src/silero_vad/utils_vad.py) | MIT | 双阈值 VAD、padding、最长段切分 | 中，需 ONNX/Core ML 集成 | 先参考参数，后续 benchmark 决定是否引入 | +| [Qwen3-ASR](https://github.com/QwenLM/Qwen3-ASR) | Apache-2.0 | 官方 context、语言控制、对齐器 | 中，官方 Python 栈 | 用作 API/行为基准 | +| [qwen3-asr-mlx](https://github.com/gabrimatic/qwen3-asr-mlx) | MIT | Apple Silicon MLX 推理、context、重复抑制 | 高，已依赖旧版 | canary 升级 | +| [local-whisper context builder](https://github.com/gabrimatic/local-whisper/blob/6ff31ffb99cfce6858ddf50f259ac1109010beb5/src/whisper_voice/engines/context.py) | MIT | 有预算、净化、去重的术语 context | 高，逻辑可重写为 Swift | 直接借鉴算法并保留许可声明 | +| [FluidAudio](https://github.com/FluidInference/FluidAudio) | Apache-2.0 | Swift/Core ML ASR、VAD、英文 Parakeet | 高，但增加模型与包体 | 英文后续旁路 benchmark | +| [text-processing-rs](https://github.com/FluidInference/text-processing-rs) | Apache-2.0 | 中英日等 TN/ITN、Swift XCFramework | 中到高 | 后续独立 spike | +| [HyPoradise/Hypo2Trans](https://github.com/Hypotheses-Paradise/Hypo2Trans) | MIT | N-best 后纠错数据和方法 | 低，训练栈不嵌入；方法可借鉴 | 作为纠错设计证据 | +| [JiWER](https://github.com/jitsi/jiwer) | Apache-2.0 | WER/CER/空音频指标 | 高，离线测试工具 | 直接用于 benchmark 脚本 | +| [FunASR](https://github.com/modelscope/FunASR) | MIT（模型权重另查） | VAD + ASR + punctuation + hotword + ITN 流水线 | 低到中，Python/Torch | 架构/中文 benchmark | +| [WeNet](https://github.com/wenet-e2e/wenet) | Apache-2.0（模型权重另查） | context graph、WFST、N-best、流式 ASR | 低，C++/libtorch | 架构参考 | +| [MiMo-V2.5-ASR](https://github.com/XiaomiMiMo/MiMo-V2.5-ASR) | Apache-2.0（具体权重另查） | 中英 ASR 研究 | 低，上游要求 Linux/CUDA | 不作为原生 Mac 主路线 | +| [VoiceInk](https://github.com/Beingpax/VoiceInk) | GPL-3.0 | macOS 本地听写产品架构参考 | 代码复制会带来 GPL 义务 | 仅参考，不复制进当前项目 | + +许可证注意： + +- 表中“代码许可证”不自动覆盖模型权重、训练数据、服务条款。 +- 真正复制源码前应保留 copyright/license notice，并核对当前固定 commit 的许可证。 +- Apple Speech 和火山 ASR 是平台/服务接口，不是开源代码。 +- GPL-3.0 项目只作行为与架构参考，除非项目明确接受 GPL 传播义务。 + +## 14. 不建议的捷径 + +1. **只把 LLM prompt 写得更长**：ASR 没提供候选和声学证据时，大模型只能猜，越会写越可能把错误润色得像真的。 +2. **所有录音永远双引擎**：功耗、内存、延迟翻倍；应只处理低置信 segment。 +3. **把完整 OCR/历史当热词**：会泄露不必要内容，也会造成误吸附和重复。 +4. **默认开启强降噪/AGC**：通信音质更“干净”不等于 ASR CER 更低。 +5. **VAD 判静音就硬删除**：低音量字、塞音、句尾容易被切掉。 +6. **看到上游 benchmark 就更换主引擎**：数据集、语言、设备和量化方式都可能不同。 +7. **在没有 gold corpus 时调十几个阈值**：得到的只是不可复现的主观印象。 +8. **把 MiMo 当作现成的原生 Mac 引擎**：上游官方运行条件并不支持该结论。 + +## 15. 建议的首批验收任务 + +1. 建 30 条最小 corpus：10 条中文专名、5 条中英混说、5 条数字/URL/path、5 条短音频、5 条静音/噪声。 +2. 输出当前 Apple、Whisper、Qwen 的 baseline：raw CER/WER、专名、数字、静音幻觉、p95 延迟。 +3. 加 `TranscriptEvidence`，但先不改变 UI 输出。 +4. Whisper 开 timestamps/word confidence，比较 fallback 1/2/3。 +5. Apple 加 20 个 contextual strings，并取 alternatives/confidence。 +6. Qwen 0.2.0 走独立 canary runtime，只在 benchmark 开启 context。 +7. 用 LocalAgreement 做一个流式 feature flag，测 partial churn。 +8. 在现有整段 guard 上增加结构化 edits、声学候选证据和按句回退，再与改造前 prompt 做盲测。 +9. 只有上述结果明确后，才决定是否引入 Silero、FluidAudio 或 ITN 新依赖。 + +这批任务完成后，团队将第一次能回答三个关键问题： + +- 错误主要来自音频、ASR，还是后处理？ +- 哪个引擎在用户真实中文/英文场景最可靠？ +- 额外延迟和算力究竟换来了多少可测的准确率提升？ + +## 16. 主要一手来源 + +- Apple: [Speech framework](https://developer.apple.com/documentation/speech/), [SpeechAnalyzer](https://developer.apple.com/documentation/speech/speechanalyzer), [AnalysisContext](https://developer.apple.com/documentation/speech/analysiscontext), [WWDC25 SpeechAnalyzer](https://developer.apple.com/videos/play/wwdc2025/277/), [WWDC23 custom language model](https://developer.apple.com/videos/play/wwdc2023/10101/) +- Argmax: [WhisperKit source](https://github.com/argmaxinc/argmax-oss-swift) +- OpenAI-compatible Whisper runtime reference: [whisper.cpp source](https://github.com/ggml-org/whisper.cpp) +- UFAL: [WhisperStreaming source](https://github.com/ufal/whisper_streaming), [SimulStreaming source](https://github.com/ufal/SimulStreaming) +- Qwen: [Qwen3-ASR source](https://github.com/QwenLM/Qwen3-ASR), [Qwen3-ASR paper](https://arxiv.org/abs/2601.21337) +- Xiaomi: [MiMo-V2.5-ASR source](https://github.com/XiaomiMiMo/MiMo-V2.5-ASR) +- Volcano Engine: [hotword documentation](https://www.volcengine.com/docs/6561/155739?lang=zh), [hotword FAQ](https://www.volcengine.com/docs/6561/155743?lang=zh) +- ModelScope: [FunASR source](https://github.com/modelscope/FunASR) +- WeNet: [WeNet source](https://github.com/wenet-e2e/wenet) +- ASR correction: [HyPoradise/Hypo2Trans](https://github.com/Hypotheses-Paradise/Hypo2Trans), [Towards Robust and Generalizable ASR Error Correction](https://arxiv.org/abs/2409.09554) diff --git a/docs/superpowers/specs/voice-quality-corpus.example.jsonl b/docs/superpowers/specs/voice-quality-corpus.example.jsonl new file mode 100644 index 00000000..24aac85d --- /dev/null +++ b/docs/superpowers/specs/voice-quality-corpus.example.jsonl @@ -0,0 +1,4 @@ +{"id":"synthetic-zh-terminology-001","language":"zh-CN","faithful_reference":"请把 OpenType 2.5 的发布改到 8 月 3 日。","asr_text":"请把 OpenType 2.5 的发布改到 8 月 3 日。","sendable_reference":"请把 OpenType 2.5 的发布改到 8 月 3 日。","processed_text":"请把 OpenType 2.5 的发布改到 8 月 3 日。","terms":["OpenType"],"asr_latency_ms":480.0,"processing_latency_ms":125.0} +{"id":"synthetic-en-edit-001","language":"en-US","faithful_reference":"send the report to test@example.com","asr_text":"send report to test@example.com","sendable_reference":"Send the report to test@example.com.","processed_text":"Send the report to test@example.com.","terms":[],"asr_latency_ms":390.0,"processing_latency_ms":110.0} +{"id":"synthetic-code-switch-001","language":"zh-CN","faithful_reference":"打开 /tmp/demo.txt，然后访问 https://example.com。","asr_text":"打开 /tmp/demo.txt，然后访问 https://example.com。","terms":["demo.txt"],"asr_latency_ms":420.0} +{"id":"synthetic-silence-001","language":"zh-CN","faithful_reference":"","asr_text":"","asr_latency_ms":160.0} diff --git a/scripts/evaluate-voice-quality.py b/scripts/evaluate-voice-quality.py new file mode 100755 index 00000000..f09a6956 --- /dev/null +++ b/scripts/evaluate-voice-quality.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +"""Evaluate raw ASR and optional post-processed transcripts from JSONL.""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from pathlib import Path +from typing import Any + +from voice_quality_metrics import evaluate + + +REQUIRED_FIELDS = ("id", "language", "faithful_reference", "asr_text") +TEXT_FIELDS = ( + "language", + "faithful_reference", + "asr_text", + "sendable_reference", + "processed_text", +) +LATENCY_FIELDS = ("asr_latency_ms", "processing_latency_ms") + + +class CorpusError(ValueError): + """A JSONL record does not conform to the evaluator schema.""" + + +def validate_record(value: Any, line_number: int) -> dict[str, Any]: + if not isinstance(value, dict): + raise CorpusError(f"line {line_number}: expected a JSON object") + missing = [field for field in REQUIRED_FIELDS if field not in value] + if missing: + raise CorpusError( + f"line {line_number}: missing required field(s): {', '.join(missing)}" + ) + record = dict(value) + identifier = record["id"] + if isinstance(identifier, bool) or not isinstance(identifier, (str, int, float)): + raise CorpusError(f"line {line_number}: id must be a string or number") + record["id"] = str(identifier) + if not record["id"].strip(): + raise CorpusError(f"line {line_number}: id must not be empty") + for field in TEXT_FIELDS: + if field in record and not isinstance(record[field], str): + raise CorpusError(f"line {line_number}: {field} must be a string") + if not record["language"].strip(): + raise CorpusError(f"line {line_number}: language must not be empty") + terms = record.get("terms", []) + if not isinstance(terms, list) or any( + not isinstance(term, str) or not term for term in terms + ): + raise CorpusError( + f"line {line_number}: terms must be an array of non-empty strings" + ) + record["terms"] = terms + for field in LATENCY_FIELDS: + if field not in record: + continue + latency = record[field] + if ( + isinstance(latency, bool) + or not isinstance(latency, (int, float)) + or not math.isfinite(float(latency)) + or latency < 0 + ): + raise CorpusError( + f"line {line_number}: {field} must be a finite non-negative number" + ) + record[field] = float(latency) + return record + + +def load_records(path: str) -> list[dict[str, Any]]: + stream = sys.stdin if path == "-" else Path(path).open(encoding="utf-8") + records: list[dict[str, Any]] = [] + identifiers: set[str] = set() + try: + for line_number, line in enumerate(stream, start=1): + if not line.strip(): + continue + try: + value = json.loads(line) + except json.JSONDecodeError as error: + raise CorpusError( + f"line {line_number}: invalid JSON: {error.msg}" + ) from error + record = validate_record(value, line_number) + if record["id"] in identifiers: + raise CorpusError( + f"line {line_number}: duplicate id {record['id']!r}" + ) + identifiers.add(record["id"]) + records.append(record) + finally: + if stream is not sys.stdin: + stream.close() + if not records: + raise CorpusError("corpus contains no records") + return records + + +def format_rate(metric: dict[str, Any], numerator: str, denominator: str) -> str: + rate = metric["rate"] + if rate is None: + return "n/a" + return f"{rate * 100:.2f}% ({metric[numerator]}/{metric[denominator]})" + + +def render_view(label: str, view: dict[str, Any]) -> list[str]: + lines = [f"{label} ({view['records']} records)"] + lines.append( + " CER overall: " + + format_rate(view["cer"], "distance", "reference_units") + ) + for language, metric in view["cer_by_language"].items(): + lines.append( + f" CER {language}: " + + format_rate(metric, "distance", "reference_units") + ) + lines.append( + " English WER: " + + format_rate(view["english_wer"], "distance", "reference_units") + ) + hallucination = view["empty_reference_hallucination"] + lines.append( + " Empty-reference hallucination: " + f"{hallucination['hallucinated_characters']} characters, " + f"{hallucination['records_with_hallucination']}/" + f"{hallucination['empty_reference_records']} records" + ) + lines.append( + " Terms exact: " + + format_rate(view["terms_exact"], "matched", "expected") + ) + for category in ("number", "url", "email", "path", "combined"): + fidelity = view["protected_fidelity"][category] + lines.append( + f" {category} fidelity: " + + format_rate( + fidelity, "exact_records", "evaluated_records" + ) + + f", +{fidelity['insertions']}/-{fidelity['deletions']}" + ) + return lines + + +def render_text(report: dict[str, Any]) -> str: + lines = [f"Voice quality evaluation ({report['corpus_records']} records)"] + lines.extend(render_view("ASR vs faithful reference", report["asr"])) + lines.extend(render_view("Processed vs sendable reference", report["processed"])) + lines.append("Latency (ms)") + for phase in ("asr", "processing", "total"): + metric = report["latency_ms"][phase] + p50 = "n/a" if metric["p50"] is None else f"{metric['p50']:.2f}" + p95 = "n/a" if metric["p95"] is None else f"{metric['p95']:.2f}" + lines.append( + f" {phase}: p50={p50}, p95={p95} ({metric['records']} records)" + ) + return "\n".join(lines) + + +def parse_arguments(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Evaluate OpenType ASR and post-processing quality from JSONL.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog="""JSONL schema (one object per non-empty line): + Required: + id unique string or number + language BCP-47-like tag; metrics group by base language + faithful_reference literal spoken-content gold text; may be empty + asr_text raw ASR output + Optional: + sendable_reference edited gold text for processed_text + processed_text post-processed output; falls back to faithful_reference + as its gold text when sendable_reference is absent + terms array of preferred spellings expected verbatim + asr_latency_ms finite non-negative number + processing_latency_ms finite non-negative number + +CER uses NFKC text with whitespace removed and preserves case/punctuation. +English WER uses case-folded word tokens and ignores punctuation. Empty-reference +insertions contribute to aggregate CER and are also reported separately. Protected +fidelity requires the exact per-record multiset of numbers, URLs, emails, and +paths; additions and deletions both fail the record. Quoting is recommended for +directory paths that contain spaces and have no filename extension. + +Example: + scripts/evaluate-voice-quality.py \\ + docs/superpowers/specs/voice-quality-corpus.example.jsonl + scripts/evaluate-voice-quality.py --format json corpus.jsonl + cat corpus.jsonl | scripts/evaluate-voice-quality.py --format json -""", + ) + parser.add_argument("corpus", help="UTF-8 JSONL corpus path, or - for stdin") + parser.add_argument( + "--format", + choices=("text", "json"), + default="text", + help="output format (default: text)", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + arguments = parse_arguments(argv) + try: + report = evaluate(load_records(arguments.corpus)) + except (CorpusError, OSError) as error: + print(f"error: {error}", file=sys.stderr) + return 2 + if arguments.format == "json": + print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True)) + else: + print(render_text(report)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/test_evaluate_voice_quality.py b/scripts/tests/test_evaluate_voice_quality.py new file mode 100755 index 00000000..ae899ab9 --- /dev/null +++ b/scripts/tests/test_evaluate_voice_quality.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 + +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +SCRIPT = Path(__file__).resolve().parents[1] / "evaluate-voice-quality.py" + + +class VoiceQualityEvaluatorTests(unittest.TestCase): + def run_cli(self, records): + with tempfile.NamedTemporaryFile("w", suffix=".jsonl", encoding="utf-8") as fixture: + for record in records: + fixture.write(json.dumps(record, ensure_ascii=False) + "\n") + fixture.flush() + return subprocess.run( + [sys.executable, str(SCRIPT), "--format", "json", fixture.name], + check=False, + capture_output=True, + text=True, + ) + + def test_reports_asr_processed_fidelity_hallucination_and_latency(self): + exact = ( + "OpenType 2.5 访问 https://example.com，发到 test@example.com，" + "路径 /tmp/demo.txt" + ) + result = self.run_cli( + [ + { + "id": "synthetic-zh-1", + "language": "zh-CN", + "faithful_reference": exact, + "asr_text": exact, + "sendable_reference": exact, + "processed_text": exact, + "terms": ["OpenType"], + "asr_latency_ms": 100, + "processing_latency_ms": 20, + }, + { + "id": "synthetic-en-1", + "language": "en-US", + "faithful_reference": "hello world", + "asr_text": "hello brave world", + "asr_latency_ms": 200, + }, + { + "id": "synthetic-silence-1", + "language": "zh", + "faithful_reference": "", + "asr_text": "幻觉", + "asr_latency_ms": 300, + }, + ] + ) + self.assertEqual(result.returncode, 0, result.stderr) + report = json.loads(result.stdout) + self.assertEqual(report["corpus_records"], 3) + self.assertEqual(report["asr"]["english_wer"]["rate"], 0.5) + self.assertEqual( + report["asr"]["empty_reference_hallucination"][ + "hallucinated_characters" + ], + 2, + ) + self.assertEqual(report["asr"]["terms_exact"]["rate"], 1.0) + for category in ("number", "url", "email", "path", "combined"): + self.assertEqual( + report["asr"]["protected_fidelity"][category]["rate"], 1.0 + ) + self.assertEqual( + report["asr"]["protected_fidelity"][category]["insertions"], 0 + ) + self.assertEqual( + report["asr"]["protected_fidelity"][category]["deletions"], 0 + ) + self.assertEqual(report["processed"]["cer"]["rate"], 0.0) + self.assertEqual(report["latency_ms"]["asr"]["p50"], 200.0) + self.assertEqual(report["latency_ms"]["asr"]["p95"], 290.0) + + def test_fidelity_rejects_added_numbers_and_changed_unicode_space_path(self): + result = self.run_cli( + [ + { + "id": "added-number", + "language": "en", + "faithful_reference": "version 2", + "asr_text": "version 2 plus 999", + }, + { + "id": "changed-path", + "language": "zh", + "faithful_reference": "打开 /Users/陈丽/My File.txt。", + "asr_text": "打开 /Users/陈丽/My Other.txt。", + }, + ] + ) + self.assertEqual(result.returncode, 0, result.stderr) + report = json.loads(result.stdout)["asr"]["protected_fidelity"] + self.assertEqual(report["number"]["rate"], 0.0) + self.assertEqual(report["number"]["insertions"], 1) + self.assertEqual(report["path"]["rate"], 0.0) + self.assertEqual(report["path"]["insertions"], 1) + self.assertEqual(report["path"]["deletions"], 1) + self.assertEqual(report["combined"]["rate"], 0.0) + + def test_fidelity_preserves_sentence_email_opaque_paths_and_quotes(self): + result = self.run_cli( + [ + { + "id": "sentence-email", + "language": "zh", + "faithful_reference": "联系 test@example.com.", + "asr_text": "联系 nobody@example.com.", + }, + { + "id": "opaque-path", + "language": "zh", + "faithful_reference": "打开 /tmp/①.txt", + "asr_text": "打开 /tmp/1.txt", + }, + { + "id": "quoted-path", + "language": "zh", + "faithful_reference": '运行 "/tmp/My File.txt"', + "asr_text": "运行 /tmp/My File.txt", + }, + { + "id": "balanced-url", + "language": "en", + "faithful_reference": ( + "Read https://en.wikipedia.org/wiki/Function_(mathematics)" + ), + "asr_text": ( + "Read https://en.wikipedia.org/wiki/Function_(mathematics" + ), + }, + ] + ) + self.assertEqual(result.returncode, 0, result.stderr) + report = json.loads(result.stdout)["asr"]["protected_fidelity"] + self.assertEqual(report["email"]["rate"], 0.0) + self.assertEqual(report["email"]["insertions"], 1) + self.assertEqual(report["email"]["deletions"], 1) + self.assertEqual(report["path"]["rate"], 0.0) + self.assertEqual(report["path"]["insertions"], 2) + self.assertEqual(report["path"]["deletions"], 2) + self.assertEqual(report["url"]["rate"], 0.0) + self.assertEqual(report["url"]["insertions"], 1) + self.assertEqual(report["url"]["deletions"], 1) + self.assertEqual(report["combined"]["rate"], 0.0) + + def test_fidelity_rejects_reordered_number_ownership(self): + result = self.run_cli( + [ + { + "id": "reordered-numbers", + "language": "en", + "faithful_reference": "Alice 2, Bob 3", + "asr_text": "Alice 3, Bob 2", + } + ] + ) + self.assertEqual(result.returncode, 0, result.stderr) + report = json.loads(result.stdout)["asr"]["protected_fidelity"] + self.assertEqual(report["number"]["rate"], 0.0) + self.assertEqual(report["combined"]["rate"], 0.0) + + def test_rejects_missing_required_field_with_line_number(self): + result = self.run_cli( + [{"id": "broken", "language": "zh", "faithful_reference": "文本"}] + ) + self.assertEqual(result.returncode, 2) + self.assertIn("line 1", result.stderr) + self.assertIn("asr_text", result.stderr) + + def test_help_documents_schema(self): + result = subprocess.run( + [sys.executable, str(SCRIPT), "--help"], + check=False, + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0) + self.assertIn("JSONL schema", result.stdout) + self.assertIn("faithful_reference", result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/voice_quality_fidelity.py b/scripts/voice_quality_fidelity.py new file mode 100644 index 00000000..683620a0 --- /dev/null +++ b/scripts/voice_quality_fidelity.py @@ -0,0 +1,188 @@ +"""Protected-token extraction and exact-fidelity metrics.""" + +from __future__ import annotations + +import re +import unicodedata +from collections import Counter +from typing import Any, Iterable + + +URL_RE = re.compile(r"(?i)\b(?:https?://|www\.)[^\s<>\"']+") +EMAIL_RE = re.compile( + r"(?i)(?["'])(?:~|\.\.?|/|[A-Za-z]:\\|\\\\)[^"'\r\n]+(?P=quote)""" +) +SPACED_UNIX_FILE_PATH_RE = re.compile( + r"""(?\r\n"']*?""" + r"""\.[A-Za-z0-9]{1,16}(?=$|[\s,;!?，。；！？)\]}])""" +) +UNIX_PATH_RE = re.compile( + r"""(?\r\n"']+/)*""" + r"""[^\s,;!?，。；！？<>\r\n"']+""" +) +WINDOWS_PATH_RE = re.compile( + r"(?i)(? bool: + return any(span[0] < end and start < span[1] for start, end in occupied) + + +def trim_entity_punctuation(value: str) -> str: + always_trailing = ".,;:!?，。；：！？" + pairs = {")": "(", "）": "（", "]": "[", "】": "【", "}": "{"} + while value: + last = value[-1] + if last in always_trailing: + value = value[:-1] + continue + opening = pairs.get(last) + if opening is None or value.count(last) <= value.count(opening): + break + value = value[:-1] + return value + + +def extracted_protected_entities( + text: str, +) -> tuple[dict[str, list[str]], list[tuple[str, str]]]: + normalized = unicodedata.normalize("NFC", text) + entities: dict[str, list[str]] = {category: [] for category in CATEGORIES} + occupied: list[tuple[int, int]] = [] + ordered: list[tuple[int, str, str]] = [] + patterns = ( + ("url", URL_RE, None), + ("email", EMAIL_RE, None), + ("path", QUOTED_PATH_RE, None), + ("path", SPACED_UNIX_FILE_PATH_RE, None), + ("path", UNIX_PATH_RE, None), + ("path", WINDOWS_PATH_RE, None), + ("path", UNC_PATH_RE, None), + ("path", RELATIVE_FILE_PATH_RE, None), + ("path", BARE_FILE_PATH_RE, None), + ("number", NUMBER_RE, None), + ) + for category, pattern, value_group in patterns: + for match in pattern.finditer(normalized): + start, end = match.span(value_group) if value_group else match.span() + value = ( + match.group(value_group) if value_group else match.group() + ) + value = trim_entity_punctuation(value) + if category == "number": + value = unicodedata.normalize("NFKC", value) + end = start + len(value) + if not value or overlaps((start, end), occupied): + continue + entities[category].append(value) + ordered.append((start, category, value)) + occupied.append((start, end)) + ordered.sort(key=lambda item: item[0]) + return entities, [(category, value) for _, category, value in ordered] + + +def protected_entities(text: str) -> dict[str, list[str]]: + return extracted_protected_entities(text)[0] + + +def empty_category_metric() -> dict[str, Any]: + return { + "matched": 0, + "expected": 0, + "observed": 0, + "insertions": 0, + "deletions": 0, + "exact_records": 0, + "evaluated_records": 0, + "rate": None, + "recall": None, + "precision": None, + } + + +def finalize_category_metric(values: dict[str, Any]) -> None: + values["rate"] = ( + values["exact_records"] / values["evaluated_records"] + if values["evaluated_records"] + else None + ) + values["recall"] = ( + values["matched"] / values["expected"] if values["expected"] else None + ) + values["precision"] = ( + values["matched"] / values["observed"] if values["observed"] else None + ) + + +def fidelity_metric( + pairs: Iterable[tuple[dict[str, Any], str, str]] +) -> dict[str, Any]: + totals = {category: empty_category_metric() for category in CATEGORIES} + combined_exact_records = combined_evaluated_records = 0 + for _, reference, hypothesis in pairs: + expected_entities, expected_sequence = extracted_protected_entities(reference) + actual_entities, actual_sequence = extracted_protected_entities(hypothesis) + record_has_entities = False + for category in CATEGORIES: + expected_values = expected_entities[category] + observed_values = actual_entities[category] + expected = Counter(expected_values) + observed = Counter(observed_values) + matched = sum((expected & observed).values()) + expected_count = sum(expected.values()) + observed_count = sum(observed.values()) + values = totals[category] + values["matched"] += matched + values["expected"] += expected_count + values["observed"] += observed_count + values["deletions"] += expected_count - matched + values["insertions"] += observed_count - matched + if expected or observed: + record_has_entities = True + values["evaluated_records"] += 1 + if expected_values == observed_values: + values["exact_records"] += 1 + if record_has_entities: + combined_evaluated_records += 1 + combined_exact_records += expected_sequence == actual_sequence + + for values in totals.values(): + finalize_category_metric(values) + + combined = { + key: sum(values[key] for values in totals.values()) + for key in ( + "matched", + "expected", + "observed", + "insertions", + "deletions", + ) + } + combined.update( + { + "exact_records": combined_exact_records, + "evaluated_records": combined_evaluated_records, + } + ) + finalize_category_metric(combined) + totals["combined"] = combined + return totals diff --git a/scripts/voice_quality_metrics.py b/scripts/voice_quality_metrics.py new file mode 100644 index 00000000..4aab6992 --- /dev/null +++ b/scripts/voice_quality_metrics.py @@ -0,0 +1,217 @@ +"""Dependency-free transcript quality metrics used by the evaluator CLI.""" + +from __future__ import annotations + +import math +import re +import unicodedata +from collections import defaultdict +from typing import Any, Iterable, Sequence + +from voice_quality_fidelity import fidelity_metric + + +def normalize_unicode(text: str) -> str: + return unicodedata.normalize("NFKC", text) + + +def cer_units(text: str) -> list[str]: + return [ + character + for character in normalize_unicode(text) + if not character.isspace() + ] + + +def english_words(text: str) -> list[str]: + normalized = normalize_unicode(text).casefold().replace("’", "'") + return re.findall(r"[^\W_]+(?:'[^\W_]+)*", normalized, flags=re.UNICODE) + + +def edit_distance(reference: Sequence[str], hypothesis: Sequence[str]) -> int: + if len(reference) > len(hypothesis): + reference, hypothesis = hypothesis, reference + previous = list(range(len(reference) + 1)) + for hypothesis_index, hypothesis_item in enumerate(hypothesis, start=1): + current = [hypothesis_index] + for reference_index, reference_item in enumerate(reference, start=1): + current.append( + min( + current[-1] + 1, + previous[reference_index] + 1, + previous[reference_index - 1] + + (reference_item != hypothesis_item), + ) + ) + previous = current + return previous[-1] + + +def base_language(language: str) -> str: + return re.split(r"[-_]", language.strip().casefold(), maxsplit=1)[0] + + +def comparison_pairs( + records: Iterable[dict[str, Any]], processed: bool +) -> list[tuple[dict[str, Any], str, str]]: + pairs = [] + for record in records: + if processed: + if "processed_text" not in record: + continue + reference = record.get( + "sendable_reference", record["faithful_reference"] + ) + hypothesis = record["processed_text"] + else: + reference = record["faithful_reference"] + hypothesis = record["asr_text"] + pairs.append((record, reference, hypothesis)) + return pairs + + +def error_metric( + pairs: Iterable[tuple[dict[str, Any], str, str]], tokenizer +) -> dict[str, Any]: + distance = reference_units = records = 0 + for _, reference, hypothesis in pairs: + reference_tokens = tokenizer(reference) + hypothesis_tokens = tokenizer(hypothesis) + distance += edit_distance(reference_tokens, hypothesis_tokens) + reference_units += len(reference_tokens) + records += 1 + return { + "records": records, + "distance": distance, + "reference_units": reference_units, + "rate": distance / reference_units if reference_units else None, + } + + +def term_occurs_exactly(text: str, term: str) -> bool: + normalized_text = normalize_unicode(text) + normalized_term = normalize_unicode(term) + left = ( + r"(? dict[str, Any]: + expected = matched = 0 + for record, _, hypothesis in pairs: + unique_terms = dict.fromkeys( + normalize_unicode(term) for term in record["terms"] + ) + for term in unique_terms: + expected += 1 + matched += term_occurs_exactly(hypothesis, term) + return { + "matched": matched, + "expected": expected, + "rate": matched / expected if expected else None, + } + + +def hallucination_metric( + pairs: Iterable[tuple[dict[str, Any], str, str]] +) -> dict[str, int]: + empty_records = hallucinated_records = hallucinated_characters = 0 + for _, reference, hypothesis in pairs: + if cer_units(reference): + continue + empty_records += 1 + characters = len(cer_units(hypothesis)) + hallucinated_characters += characters + hallucinated_records += characters > 0 + return { + "empty_reference_records": empty_records, + "records_with_hallucination": hallucinated_records, + "hallucinated_characters": hallucinated_characters, + } + + +def percentile(values: list[float], quantile: float) -> float | None: + if not values: + return None + ordered = sorted(values) + position = (len(ordered) - 1) * quantile + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return ordered[lower] + return ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower) + + +def latency_metric(values: list[float]) -> dict[str, Any]: + return { + "records": len(values), + "p50": percentile(values, 0.5), + "p95": percentile(values, 0.95), + } + + +def evaluate_view( + pairs: list[tuple[dict[str, Any], str, str]] +) -> dict[str, Any]: + by_language: dict[ + str, list[tuple[dict[str, Any], str, str]] + ] = defaultdict(list) + for pair in pairs: + by_language[base_language(pair[0]["language"])].append(pair) + english = [ + pair for pair in pairs if base_language(pair[0]["language"]) == "en" + ] + return { + "records": len(pairs), + "cer": error_metric(pairs, cer_units), + "cer_by_language": { + language: error_metric(language_pairs, cer_units) + for language, language_pairs in sorted(by_language.items()) + }, + "english_wer": error_metric(english, english_words), + "empty_reference_hallucination": hallucination_metric(pairs), + "terms_exact": terms_metric(pairs), + "protected_fidelity": fidelity_metric(pairs), + } + + +def evaluate(records: list[dict[str, Any]]) -> dict[str, Any]: + asr_latencies = [ + record["asr_latency_ms"] + for record in records + if "asr_latency_ms" in record + ] + processing_latencies = [ + record["processing_latency_ms"] + for record in records + if "processing_latency_ms" in record + ] + total_latencies = [ + record["asr_latency_ms"] + record["processing_latency_ms"] + for record in records + if "asr_latency_ms" in record and "processing_latency_ms" in record + ] + return { + "schema_version": 1, + "corpus_records": len(records), + "asr": evaluate_view(comparison_pairs(records, processed=False)), + "processed": evaluate_view(comparison_pairs(records, processed=True)), + "latency_ms": { + "asr": latency_metric(asr_latencies), + "processing": latency_metric(processing_latencies), + "total": latency_metric(total_latencies), + }, + }