diff --git a/Package.swift b/Package.swift index 16d901a..2bee492 100644 --- a/Package.swift +++ b/Package.swift @@ -42,6 +42,7 @@ let package = Package( .copy("Resources/SettingsStyleIllustration.png"), .copy("Resources/SettingsIntegrationsIllustration.png"), .copy("Resources/SettingsAboutIllustration.png"), + .copy("Resources/IndustryLexicons.json"), .copy("Resources/Sounds"), .copy("Resources/AppIcon.icns"), .copy("Resources/AppIconLight.icns"), diff --git a/README.md b/README.md index 16fa9fe..01d2676 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ Three output modes are available: | **Screen Context OCR** | Captures on-screen text via ScreenCaptureKit + Vision to help the LLM correct homophones | | **Voice Command Mode** | Screen-aware voice assistant — summarize, reply, translate based on what's on screen | | **Input Memory** | Recent input history injected as LLM context for better continuity | +| **Industry Vocabulary** | Choose Medical, Legal, Finance & Accounting, or Software Technology terms for Apple Speech / Whisper biasing and output normalization; personal terms take priority | | **Edit Rules** | Personal text replacement rules applied on every output | | **Language Style Presets** | Concise / Formal / Casual / Custom prompt per language | | **Input History & Stats** | Full history with raw vs. processed comparison, word count stats, configurable retention | @@ -145,7 +146,7 @@ Sources/ ├── Hotkey/ # Global hotkey via CGEvent tap ├── LLM/ # LLMEngine (MLX), RemoteLLMClient (OpenAI/Anthropic) ├── Output/ # Text injection (Accessibility API + clipboard paste) -├── Processing/ # TextProcessor, InputHistory, MemoryStore, PersonalDictionary +├── Processing/ # TextProcessor, InputHistory, MemoryStore, personal and industry vocabulary ├── Prompts/ # PromptBuilder, prompt catalogs, style prompt presets ├── Screen/ # Screen OCR (ScreenCaptureKit + Vision) ├── Speech/ # SpeechEngine protocol, WhisperKit, Apple Speech, Doubao ASR, local ASR engines @@ -157,6 +158,7 @@ scripts/ ├── ci-basic-checks.sh # CI guardrails for linked files and resources ├── create-signing-cert.sh # Generate self-signed code signing certificate ├── generate-icon.swift # Generate AppIcon.icns from source PNG +├── test-industry-lexicons.sh # Validate vocabulary recall and non-target preservation ├── unit-test-coverage.sh # Run unit tests with coverage thresholds └── validate-volc-asr.swift # Validate Volcengine ASR configuration manually ``` diff --git a/README_zh.md b/README_zh.md index 50b0a6f..91aba89 100644 --- a/README_zh.md +++ b/README_zh.md @@ -49,6 +49,7 @@ | **屏幕上下文 OCR** | 通过 ScreenCaptureKit + Vision 截取屏幕文字，辅助 LLM 纠正同音字 | | **语音指令模式** | 屏幕感知的语音助手 — 总结、回复、翻译屏幕内容 | | **输入记忆** | 近期输入历史作为 LLM 上下文，提升连续输入准确度 | +| **行业词库** | 可选医疗、法律、金融财会或软件技术词库，为 Apple Speech / Whisper 提供优先术语并辅助输出规范化；个人词条优先 | | **编辑规则** | 自定义文本替换规则，每次输出自动应用 | | **语言风格预设** | 简洁精炼 / 正式书面 / 日常口语 / 自定义提示词 | | **输入历史与统计** | 完整历史记录，原始文本与润色结果对比，字数统计，可配置保留时长 | @@ -146,7 +147,7 @@ Sources/ ├── Hotkey/ # 全局快捷键（CGEvent tap） ├── LLM/ # 本地推理引擎（MLX）、远程客户端（OpenAI/Anthropic） ├── Output/ # 文本注入（Accessibility API + 剪贴板粘贴） -├── Processing/ # 文本处理器、输入历史、记忆系统、个人词库 +├── Processing/ # 文本处理器、输入历史、记忆系统、个人与行业词库 ├── Prompts/ # 提示词构建、固定提示词目录、风格提示词预设 ├── Screen/ # 屏幕 OCR（ScreenCaptureKit + Vision） ├── Speech/ # 语音识别协议、WhisperKit 引擎、Apple Speech 引擎、豆包语音识别引擎、本地语音识别引擎 @@ -158,6 +159,7 @@ scripts/ ├── ci-basic-checks.sh # CI 文件关联和资源检查 ├── create-signing-cert.sh # 生成自签名代码签名证书 ├── generate-icon.swift # 从源 PNG 生成 AppIcon.icns +├── test-industry-lexicons.sh # 验证行业词库、术语召回和非目标文本保真 ├── unit-test-coverage.sh # 运行单元测试并检查覆盖率 └── validate-volc-asr.swift # 手动验证火山引擎 ASR 配置 ``` diff --git a/Sources/App/VoicePipeline+Processing.swift b/Sources/App/VoicePipeline+Processing.swift index 1c44982..1e76715 100644 --- a/Sources/App/VoicePipeline+Processing.swift +++ b/Sources/App/VoicePipeline+Processing.swift @@ -131,6 +131,7 @@ extension VoicePipeline { return await processCommand(raw, settings: settings, targetApp: targetApp) case .direct: cancelScreenContextCapture() + let dictionarySnapshot = PersonalDictionary.shared.snapshot(settings: settings) let context = InputContext.capture( targetApp: targetApp, screenContext: "", @@ -139,7 +140,11 @@ extension VoicePipeline { source: .menuBar ) return VoicePipelineOutput( - text: textProcessor.basicClean(text: raw, inputLanguage: settings.inputLanguage), + text: textProcessor.basicClean( + text: raw, + inputLanguage: settings.inputLanguage, + dictionarySnapshot: dictionarySnapshot + ), context: context ) } @@ -155,7 +160,7 @@ extension VoicePipeline { let started = CFAbsoluteTimeGetCurrent() let processingOptions = TextProcessingOptions(settings: settings) - let dictionarySnapshot = PersonalDictionary.shared.snapshot() + let dictionarySnapshot = PersonalDictionary.shared.snapshot(settings: settings) let enableMemory = settings.enableMemory let memoryWindowMinutes = settings.memoryWindowMinutes let screenContext = await finishScreenContextCapture() @@ -200,7 +205,7 @@ extension VoicePipeline { let started = CFAbsoluteTimeGetCurrent() let processingOptions = TextProcessingOptions(settings: settings) - let dictionarySnapshot = PersonalDictionary.shared.snapshot() + let dictionarySnapshot = PersonalDictionary.shared.snapshot(settings: settings) let enableMemory = settings.enableMemory let memoryWindowMinutes = settings.memoryWindowMinutes let screenContext = await finishScreenContextCapture() diff --git a/Sources/App/VoicePipeline+Replacement.swift b/Sources/App/VoicePipeline+Replacement.swift index 7f96a20..418a9cf 100644 --- a/Sources/App/VoicePipeline+Replacement.swift +++ b/Sources/App/VoicePipeline+Replacement.swift @@ -39,7 +39,7 @@ extension VoicePipeline { targetApp: NSRunningApplication? ) async { let processingOptions = TextProcessingOptions(settings: settings) - let dictionarySnapshot = PersonalDictionary.shared.snapshot() + let dictionarySnapshot = PersonalDictionary.shared.snapshot(settings: settings) let enableMemory = settings.enableMemory let memoryWindowMinutes = settings.memoryWindowMinutes let quickText = immediateInsertText( diff --git a/Sources/App/VoicePipeline.swift b/Sources/App/VoicePipeline.swift index a12a0c5..96310fa 100644 --- a/Sources/App/VoicePipeline.swift +++ b/Sources/App/VoicePipeline.swift @@ -118,10 +118,11 @@ final class VoicePipeline { let micID = appState.settings.microphoneID let language = appState.settings.inputLanguage.whisperCode let streamingEnabled = appState.settings.enableStreamingRecognitionBeta + let vocabularySnapshot = PersonalDictionary.shared.snapshot( + settings: appState.settings + ) currentEngine?.configureRecognition( - context: SpeechRecognitionContext( - dictionaryEntries: PersonalDictionary.shared.entries - ) + context: SpeechRecognitionContext(phrases: vocabularySnapshot.recognitionPhrases) ) if streamingEnabled { currentEngine?.startListening(language: language) { [weak self] partialText in diff --git a/Sources/Config/AppSettings.swift b/Sources/Config/AppSettings.swift index fd57ab0..69e6cb8 100644 --- a/Sources/Config/AppSettings.swift +++ b/Sources/Config/AppSettings.swift @@ -280,6 +280,7 @@ final class AppSettings: ObservableObject { @Published var enableMemory: Bool @Published var memoryWindowMinutes: Int @Published var enableCorrectionLearning: Bool + @Published var industryLexicon: IndustryLexiconID @Published var useCustomSystemPrompt: Bool @Published var customSystemPrompt: String @Published var useRemoteLLM: Bool @@ -311,7 +312,7 @@ final class AppSettings: ObservableObject { case enableStreamingRecognitionBeta case inputLanguage, translationTargetLanguage case useScreenContext, screenContextMode, enableInstantInsert, hasCompletedOnboarding, uiLanguage, historyRetention - case enableMemory, memoryWindowMinutes, enableCorrectionLearning + case enableMemory, memoryWindowMinutes, enableCorrectionLearning, industryLexicon case useCustomSystemPrompt, customSystemPrompt case useRemoteLLM, remoteProvider, remoteAPIKey, remoteBaseURL, remoteModel case menuBarIcon, appIconAppearance @@ -385,6 +386,9 @@ final class AppSettings: ObservableObject { enableMemory = ud.object(forKey: Key.enableMemory.rawValue) as? Bool ?? true memoryWindowMinutes = (ud.integer(forKey: Key.memoryWindowMinutes.rawValue)).nonZeroInt ?? 30 enableCorrectionLearning = ud.object(forKey: Key.enableCorrectionLearning.rawValue) as? Bool ?? true + industryLexicon = IndustryLexiconID( + rawValue: ud.string(forKey: Key.industryLexicon.rawValue) ?? "" + ) ?? .general useCustomSystemPrompt = ud.bool(forKey: Key.useCustomSystemPrompt.rawValue) customSystemPrompt = ud.string(forKey: Key.customSystemPrompt.rawValue) ?? "" useRemoteLLM = ud.bool(forKey: Key.useRemoteLLM.rawValue) @@ -449,6 +453,9 @@ final class AppSettings: ObservableObject { $enableCorrectionLearning.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.enableCorrectionLearning.rawValue) }.store(in: &cancellables) + $industryLexicon.dropFirst().sink { + [defaults] in defaults.set($0.rawValue, forKey: Key.industryLexicon.rawValue) + }.store(in: &cancellables) $useCustomSystemPrompt.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.useCustomSystemPrompt.rawValue) }.store(in: &cancellables) $customSystemPrompt.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.customSystemPrompt.rawValue) }.store(in: &cancellables) $useRemoteLLM.dropFirst().sink { [defaults] in defaults.set($0, forKey: Key.useRemoteLLM.rawValue) }.store(in: &cancellables) diff --git a/Sources/Integration/InputSessionCoordinator+AudioFile.swift b/Sources/Integration/InputSessionCoordinator+AudioFile.swift index 0f3c1ce..f5c76b3 100644 --- a/Sources/Integration/InputSessionCoordinator+AudioFile.swift +++ b/Sources/Integration/InputSessionCoordinator+AudioFile.swift @@ -23,10 +23,9 @@ extension InputSessionCoordinator { guard let engine = await engineProvider.engine(settings: settings), engine.isReady else { throw IntegrationError.modelNotReady } + let vocabularySnapshot = PersonalDictionary.shared.snapshot(settings: settings) engine.configureRecognition( - context: SpeechRecognitionContext( - dictionaryEntries: PersonalDictionary.shared.entries - ) + context: SpeechRecognitionContext(phrases: vocabularySnapshot.recognitionPhrases) ) do { diff --git a/Sources/Integration/InputSessionCoordinator+Output.swift b/Sources/Integration/InputSessionCoordinator+Output.swift index 56353a2..a902da1 100644 --- a/Sources/Integration/InputSessionCoordinator+Output.swift +++ b/Sources/Integration/InputSessionCoordinator+Output.swift @@ -4,7 +4,7 @@ import Foundation extension InputSessionCoordinator { func outputText(for raw: String, active: ActiveSession) async throws -> String { let options = TextProcessingOptions(settings: settings, inputLanguage: active.inputLanguage) - let dictionarySnapshot = PersonalDictionary.shared.snapshot() + let dictionarySnapshot = PersonalDictionary.shared.snapshot(settings: settings) let enableMemory = settings.enableMemory let memoryWindowMinutes = settings.memoryWindowMinutes let text: String diff --git a/Sources/Integration/InputSessionCoordinator.swift b/Sources/Integration/InputSessionCoordinator.swift index b0d3b0f..981da0c 100644 --- a/Sources/Integration/InputSessionCoordinator.swift +++ b/Sources/Integration/InputSessionCoordinator.swift @@ -53,10 +53,9 @@ final class InputSessionCoordinator { guard let engine = await engineProvider.engine(settings: settings), engine.isReady else { throw IntegrationError.modelNotReady } + let vocabularySnapshot = PersonalDictionary.shared.snapshot(settings: settings) engine.configureRecognition( - context: SpeechRecognitionContext( - dictionaryEntries: PersonalDictionary.shared.entries - ) + context: SpeechRecognitionContext(phrases: vocabularySnapshot.recognitionPhrases) ) if effective.streamingEnabled, engine.supportsStreaming { diff --git a/Sources/Processing/IndustryLexicon.swift b/Sources/Processing/IndustryLexicon.swift new file mode 100644 index 0000000..d873cc8 --- /dev/null +++ b/Sources/Processing/IndustryLexicon.swift @@ -0,0 +1,218 @@ +import Foundation + +enum IndustryLexiconID: String, Codable, CaseIterable, Identifiable, Sendable { + case general + case medical + case legal + case finance + case technology + + var id: String { rawValue } + + var label: String { L("industry.lexicon.\(rawValue)") } + + var symbolName: String { + switch self { + case .general: return "text.book.closed" + case .medical: return "cross.case" + case .legal: return "building.columns" + case .finance: return "chart.line.uptrend.xyaxis" + case .technology: return "server.rack" + } + } +} + +struct IndustryLexiconCorrection: Codable, Equatable, Sendable { + let recognized: String + let preferred: String +} + +struct IndustryLexiconTerm: Codable, Equatable, Identifiable, Sendable { + let id: String + let term: String + let aliases: [String] + let corrections: [String] + let category: String +} + +struct IndustryLexiconSource: Codable, Equatable, Sendable { + let id: String + let title: String + let url: String + let usage: String + let redistribution: String +} + +struct IndustryLexiconPack: Codable, Equatable, Identifiable, Sendable { + let id: IndustryLexiconID + let version: String + let locale: String + let reviewStatus: String + let sourceIDs: [String] + let terms: [IndustryLexiconTerm] +} + +private struct IndustryLexiconDocument: Codable, Sendable { + let schemaVersion: Int + let version: String + let updatedAt: String + let rights: String + let sources: [IndustryLexiconSource] + let packs: [IndustryLexiconPack] +} + +struct IndustryLexiconSnapshot: Equatable, Sendable { + static let empty = IndustryLexiconSnapshot(pack: nil) + + let pack: IndustryLexiconPack? + + var recognitionPhrases: [String] { + guard let pack else { return [] } + return unique(pack.terms.flatMap { [$0.term] + $0.aliases }) + } + + var protectedTerms: [String] { + guard let pack else { return [] } + return unique(pack.terms.map(\.term)) + } + + var corrections: [IndustryLexiconCorrection] { + guard let pack else { return [] } + return pack.terms.flatMap { item in + item.corrections.map { + IndustryLexiconCorrection(recognized: $0, preferred: item.term) + } + } + } + + var promptDescription: String { + guard let pack else { return "" } + return pack.terms.map { item in + guard !item.aliases.isEmpty else { return item.term } + return "\(item.term)（\(item.aliases.joined(separator: "、"))）" + }.joined(separator: "\n") + } + + private func unique(_ values: [String]) -> [String] { + var seen = Set() + return values.compactMap { value in + let normalized = value.trimmingCharacters(in: .whitespacesAndNewlines) + guard !normalized.isEmpty, + seen.insert(normalized.lowercased()).inserted else { return nil } + return normalized + } + } +} + +struct IndustryLexiconCatalog: Sendable { + static let shared = loadBundled() + + let version: String + let updatedAt: String + let rights: String + let sources: [IndustryLexiconSource] + let packs: [IndustryLexiconPack] + + func pack(for id: IndustryLexiconID) -> IndustryLexiconPack? { + guard id != .general else { return nil } + return packs.first { $0.id == id } + } + + func snapshot(for id: IndustryLexiconID) -> IndustryLexiconSnapshot { + IndustryLexiconSnapshot(pack: pack(for: id)) + } + + static func decode(_ data: Data) throws -> IndustryLexiconCatalog { + let document = try JSONDecoder().decode(IndustryLexiconDocument.self, from: data) + guard document.schemaVersion == 1 else { throw IndustryLexiconError.unsupportedSchema } + guard !document.version.isEmpty, + !document.updatedAt.isEmpty, + !document.rights.isEmpty else { + throw IndustryLexiconError.invalidDocument + } + guard Set(document.sources.map(\.id)).count == document.sources.count, + document.sources.allSatisfy({ source in + !source.id.isEmpty + && !source.title.isEmpty + && URL(string: source.url)?.scheme?.hasPrefix("http") == true + && source.usage == "reference-only" + && source.redistribution == "not-redistributed" + }) else { + throw IndustryLexiconError.invalidSource + } + guard Set(document.packs.map(\.id)).count == document.packs.count, + !document.packs.contains(where: { $0.id == .general }) else { + throw IndustryLexiconError.duplicatePack + } + let sourceIDs = Set(document.sources.map(\.id)) + for pack in document.packs { + guard !pack.version.isEmpty, + pack.locale == "zh-CN", + pack.reviewStatus == "project-seed-needs-domain-review", + !pack.terms.isEmpty, + Set(pack.terms.map(\.id)).count == pack.terms.count, + !pack.sourceIDs.isEmpty, + Set(pack.sourceIDs).isSubset(of: sourceIDs) else { + throw IndustryLexiconError.invalidPack + } + var canonicalTerms = Set() + var recognizedCorrections = Set() + for item in pack.terms { + let term = item.term.trimmingCharacters(in: .whitespacesAndNewlines) + let canonicalKey = term.lowercased() + let aliasKeys = item.aliases.map { + $0.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + } + guard !item.id.isEmpty, + !term.isEmpty, + !item.category.isEmpty, + canonicalTerms.insert(canonicalKey).inserted, + !aliasKeys.contains(""), + !aliasKeys.contains(canonicalKey), + Set(aliasKeys).count == aliasKeys.count else { + throw IndustryLexiconError.invalidTerm + } + for correction in item.corrections { + let normalized = correction.trimmingCharacters(in: .whitespacesAndNewlines) + guard !normalized.isEmpty, + normalized.caseInsensitiveCompare(term) != .orderedSame, + recognizedCorrections.insert(normalized.lowercased()).inserted else { + throw IndustryLexiconError.invalidTerm + } + } + } + } + return IndustryLexiconCatalog( + version: document.version, + updatedAt: document.updatedAt, + rights: document.rights, + sources: document.sources, + packs: document.packs + ) + } + + private static func loadBundled() -> IndustryLexiconCatalog { + guard let url = AppResources.bundle.url( + forResource: "IndustryLexicons", + withExtension: "json" + ), let data = try? Data(contentsOf: url), let catalog = try? decode(data) else { + return IndustryLexiconCatalog( + version: "", + updatedAt: "", + rights: "", + sources: [], + packs: [] + ) + } + return catalog + } +} + +enum IndustryLexiconError: Error { + case unsupportedSchema + case invalidDocument + case invalidSource + case duplicatePack + case invalidPack + case invalidTerm +} diff --git a/Sources/Processing/PersonalDictionary.swift b/Sources/Processing/PersonalDictionary.swift index c998acc..2261612 100644 --- a/Sources/Processing/PersonalDictionary.swift +++ b/Sources/Processing/PersonalDictionary.swift @@ -9,45 +9,44 @@ struct EditRule: Codable, Identifiable, Sendable { struct PersonalDictionarySnapshot: Sendable { let entries: [DictionaryEntry] let editRules: [EditRule] + let industryLexicon: IndustryLexiconSnapshot + + init( + entries: [DictionaryEntry], + editRules: [EditRule], + industryLexicon: IndustryLexiconSnapshot = .empty + ) { + self.entries = entries + self.editRules = editRules + self.industryLexicon = industryLexicon + } func applyReplacements(to text: String) -> String { - let rules = entries.enumerated().compactMap { offset, entry -> ReplacementRule? in + let personalRules = entries.enumerated().compactMap { offset, entry -> VocabularyReplacementRule? in let original = entry.original let replacement = entry.replacement guard entry.isEffective, !original.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return nil } - return ReplacementRule( + return VocabularyReplacementRule( original: original, replacement: replacement, + sourcePriority: 0, insertionOrder: offset ) } - .sorted { - if $0.original.count != $1.original.count { - return $0.original.count > $1.original.count - } - return $0.insertionOrder < $1.insertionOrder - } - guard !rules.isEmpty, !text.isEmpty else { return text } - - var result = "" - result.reserveCapacity(text.count) - var cursor = text.startIndex - while cursor < text.endIndex { - if let match = rules.first(where: { - Self.matches($0.original, in: text, at: cursor) - }) { - result += match.replacement - cursor = text.index(cursor, offsetBy: match.original.count) - } else { - result.append(text[cursor]) - cursor = text.index(after: cursor) - } + let industryRules = industryLexicon.corrections.enumerated().map { offset, correction in + VocabularyReplacementRule( + original: correction.recognized, + replacement: correction.preferred, + sourcePriority: 1, + insertionOrder: offset + ) } - return result + + return VocabularyReplacementEngine.apply(personalRules + industryRules, to: text) } var activeEntriesDescription: String { @@ -70,9 +69,20 @@ struct PersonalDictionarySnapshot: Sendable { .joined(separator: "\n") } + var activeIndustryTermsDescription: String { + industryLexicon.promptDescription + } + + var recognitionPhrases: [String] { + let personal = SpeechRecognitionContext(dictionaryEntries: entries).phrases + return SpeechRecognitionContext( + phrases: personal + industryLexicon.recognitionPhrases + ).phrases + } + var protectedTerms: [String] { var seen = Set() - return entries.compactMap { entry in + let personalTerms = entries.compactMap { entry -> String? in guard entry.isEffective else { return nil } let term = entry.replacement.trimmingCharacters(in: .whitespacesAndNewlines) guard !term.isEmpty, @@ -81,37 +91,13 @@ struct PersonalDictionarySnapshot: Sendable { } return term } - } - - private static func matches( - _ original: String, - in text: String, - at start: String.Index - ) -> Bool { - guard let end = text.index(start, offsetBy: original.count, limitedBy: text.endIndex), - String(text[start.. text.startIndex, - isASCIIWordCharacter(text[text.index(before: start)]) { - return false - } - if let last = original.last, isASCIIWordCharacter(last), - end < text.endIndex, - isASCIIWordCharacter(text[end]) { - return false + let industryTerms = industryLexicon.protectedTerms.compactMap { term -> String? in + guard seen.insert(term.lowercased()).inserted else { return nil } + return term } - return true + return industryTerms + personalTerms } - private static func isASCIIWordCharacter(_ character: Character) -> Bool { - character.isASCIIWord - } } final class PersonalDictionary: ObservableObject { @@ -147,8 +133,20 @@ final class PersonalDictionary: ObservableObject { snapshot().activeRulesDescription } - func snapshot() -> PersonalDictionarySnapshot { - PersonalDictionarySnapshot(entries: entries, editRules: editRules) + func snapshot(industryLexicon: IndustryLexiconSnapshot = .empty) -> PersonalDictionarySnapshot { + PersonalDictionarySnapshot( + entries: entries, + editRules: editRules, + industryLexicon: industryLexicon + ) + } + + func snapshot(settings: AppSettings) -> PersonalDictionarySnapshot { + snapshot( + industryLexicon: IndustryLexiconCatalog.shared.snapshot( + for: settings.industryLexicon + ) + ) } @discardableResult @@ -257,9 +255,3 @@ final class PersonalDictionary: ObservableObject { } } - -private struct ReplacementRule { - let original: String - let replacement: String - let insertionOrder: Int -} diff --git a/Sources/Processing/TextProcessor+PromptConstruction.swift b/Sources/Processing/TextProcessor+PromptConstruction.swift index 1274178..fa037a2 100644 --- a/Sources/Processing/TextProcessor+PromptConstruction.swift +++ b/Sources/Processing/TextProcessor+PromptConstruction.swift @@ -72,8 +72,13 @@ extension TextProcessor { inputLanguage: InputLanguage, dictionarySnapshot: PersonalDictionarySnapshot? = nil ) -> String { - let snapshot = dictionarySnapshot ?? PersonalDictionary.shared.snapshot() + let snapshot = dictionarySnapshot ?? PersonalDictionary.shared.snapshot(settings: .shared) let extraSections = [ + PromptCatalog.activeIndustryLexiconSection( + snapshot.activeIndustryTermsDescription, + industry: snapshot.industryLexicon.pack?.id, + inputLanguage: inputLanguage + ), PromptCatalog.activePersonalDictionarySection( snapshot.activeEntriesDescription, inputLanguage: inputLanguage diff --git a/Sources/Processing/TextProcessor.swift b/Sources/Processing/TextProcessor.swift index dfa4193..031fb0f 100644 --- a/Sources/Processing/TextProcessor.swift +++ b/Sources/Processing/TextProcessor.swift @@ -37,7 +37,7 @@ final class TextProcessor { inputLanguage: InputLanguage = .auto, dictionarySnapshot: PersonalDictionarySnapshot? = nil ) -> String { - let snapshot = dictionarySnapshot ?? dictionary.snapshot() + let snapshot = dictionarySnapshot ?? dictionary.snapshot(settings: .shared) var result = snapshot.applyReplacements(to: text) result = normalizeWhitespace(result) return result.trimmingCharacters(in: .whitespacesAndNewlines) @@ -48,7 +48,7 @@ final class TextProcessor { inputLanguage: InputLanguage, dictionarySnapshot: PersonalDictionarySnapshot? = nil ) -> String { - let snapshot = dictionarySnapshot ?? dictionary.snapshot() + let snapshot = dictionarySnapshot ?? dictionary.snapshot(settings: .shared) var result = snapshot.applyReplacements(to: text) result = TranscriptionSanitizer.normalizeInput(result) return result.trimmingCharacters(in: .whitespacesAndNewlines) @@ -96,7 +96,7 @@ final class TextProcessor { dictionarySnapshot requestedDictionarySnapshot: PersonalDictionarySnapshot? = nil ) async -> String { let prepareStarted = CFAbsoluteTimeGetCurrent() - let dictionarySnapshot = requestedDictionarySnapshot ?? dictionary.snapshot() + let dictionarySnapshot = requestedDictionarySnapshot ?? dictionary.snapshot(settings: .shared) let cleanedText = prepareForFormatting( text: text, inputLanguage: options.inputLanguage, @@ -228,7 +228,7 @@ final class TextProcessor { inputContext: InputContext? = nil, dictionarySnapshot requestedDictionarySnapshot: PersonalDictionarySnapshot? = nil ) async -> String { - let dictionarySnapshot = requestedDictionarySnapshot ?? dictionary.snapshot() + let dictionarySnapshot = requestedDictionarySnapshot ?? dictionary.snapshot(settings: .shared) let useScreenImage = shouldUseScreenImage(options: options, image: screenImage) let systemPrompt = commandSystemPrompt( options: options, diff --git a/Sources/Processing/VocabularyReplacementEngine.swift b/Sources/Processing/VocabularyReplacementEngine.swift new file mode 100644 index 0000000..77df27a --- /dev/null +++ b/Sources/Processing/VocabularyReplacementEngine.swift @@ -0,0 +1,76 @@ +import Foundation + +struct VocabularyReplacementRule: Equatable, Sendable { + let original: String + let replacement: String + let sourcePriority: Int + let insertionOrder: Int +} + +enum VocabularyReplacementEngine { + static func apply(_ rules: [VocabularyReplacementRule], to text: String) -> String { + let rankedRules = rules.sorted { + if $0.original.count != $1.original.count { + return $0.original.count > $1.original.count + } + if $0.sourcePriority != $1.sourcePriority { + return $0.sourcePriority < $1.sourcePriority + } + return $0.insertionOrder < $1.insertionOrder + } + guard !rankedRules.isEmpty, !text.isEmpty else { return text } + + var result = "" + result.reserveCapacity(text.count) + var cursor = text.startIndex + while cursor < text.endIndex { + if let match = rankedRules.first(where: { + matches($0.original, in: text, at: cursor) + }) { + result += match.replacement + cursor = text.index(cursor, offsetBy: match.original.count) + } else { + result.append(text[cursor]) + cursor = text.index(after: cursor) + } + } + return result + } + + private static func matches( + _ original: String, + in text: String, + at start: String.Index + ) -> Bool { + guard let end = text.index(start, offsetBy: original.count, limitedBy: text.endIndex), + String(text[start.. text.startIndex, + isASCIIWord(text[text.index(before: start)]) { + return false + } + if let last = original.last, isASCIIWord(last), + end < text.endIndex, + isASCIIWord(text[end]) { + return false + } + return true + } + + private static func isASCIIWord(_ character: Character) -> Bool { + guard character.unicodeScalars.count == 1, + let value = character.unicodeScalars.first?.value else { + return false + } + return (48...57).contains(value) + || (65...90).contains(value) + || value == 95 + || (97...122).contains(value) + } +} diff --git a/Sources/Prompts/PromptCatalog+IndustryLexicon.swift b/Sources/Prompts/PromptCatalog+IndustryLexicon.swift new file mode 100644 index 0000000..7f0f8d8 --- /dev/null +++ b/Sources/Prompts/PromptCatalog+IndustryLexicon.swift @@ -0,0 +1,55 @@ +import Foundation + +extension PromptCatalog { + static func activeIndustryLexiconSection( + _ terms: String, + industry: IndustryLexiconID?, + inputLanguage: InputLanguage + ) -> String? { + let terms = terms.trimmingCharacters(in: .whitespacesAndNewlines) + guard let industry, industry != .general, !terms.isEmpty else { return nil } + + switch inputLanguage { + case .auto, .chinese, .cantonese: + return """ + \(chineseName(for: industry))行业词库，仅用于纠正语音中明确出现的术语；不要据此补充原文没有的信息： + \(PromptTextBlock.block(terms)) + """ + case .english: + return """ + \(englishName(for: industry)) industry vocabulary. Use it only to correct terms clearly present in the speech; do not add facts from this list: + \(PromptTextBlock.block(terms)) + """ + case .japanese: + return """ + \(englishName(for: industry))業界用語集。音声に明確に含まれる用語の訂正だけに使い、この一覧から情報を追加しないでください： + \(PromptTextBlock.block(terms)) + """ + case .korean: + return """ + \(englishName(for: industry)) 업계 용어집입니다. 음성에 명확히 포함된 용어를 교정할 때만 사용하고 목록의 정보를 추가하지 마세요: + \(PromptTextBlock.block(terms)) + """ + } + } + + private static func chineseName(for industry: IndustryLexiconID) -> String { + switch industry { + case .general: return "通用" + case .medical: return "医疗" + case .legal: return "法律" + case .finance: return "金融财会" + case .technology: return "软件技术" + } + } + + private static func englishName(for industry: IndustryLexiconID) -> String { + switch industry { + case .general: return "General" + case .medical: return "Medical" + case .legal: return "Legal" + case .finance: return "Finance and accounting" + case .technology: return "Software technology" + } + } +} diff --git a/Sources/Resources/IndustryLexicons.json b/Sources/Resources/IndustryLexicons.json new file mode 100644 index 0000000..cd744b3 --- /dev/null +++ b/Sources/Resources/IndustryLexicons.json @@ -0,0 +1,283 @@ +{ + "schemaVersion": 1, + "version": "2026.08.24-v1", + "updatedAt": "2026-08-24", + "rights": "Utter-curated seed lists of short public terminology; no external code set or source corpus is redistributed.", + "sources": [ + { + "id": "nhc-standards", + "title": "National Health Commission health standards", + "url": "http://wsbz.nhc.gov.cn/wsbzw/", + "usage": "reference-only", + "redistribution": "not-redistributed" + }, + { + "id": "who-icd11", + "title": "World Health Organization ICD-11", + "url": "https://icd.who.int/en", + "usage": "reference-only", + "redistribution": "not-redistributed" + }, + { + "id": "china-civil-code", + "title": "Civil Code of the People's Republic of China", + "url": "https://www.gov.cn/xinwen/2020-06/01/content_5516649.htm", + "usage": "reference-only", + "redistribution": "not-redistributed" + }, + { + "id": "china-criminal-law", + "title": "Criminal Law of the People's Republic of China", + "url": "https://www.npc.gov.cn/zgrdw/npc/lfzt/rlys/2008-08/21/content_1882895.htm", + "usage": "reference-only", + "redistribution": "not-redistributed" + }, + { + "id": "china-criminal-procedure", + "title": "Criminal Procedure Law of the People's Republic of China", + "url": "https://www.npc.gov.cn/npc/c2/c12435/201905/t20190521_276591.html", + "usage": "reference-only", + "redistribution": "not-redistributed" + }, + { + "id": "china-civil-procedure", + "title": "Civil Procedure Law of the People's Republic of China", + "url": "https://www.npc.gov.cn/c2/c12435/201905/t20190522_88402.html", + "usage": "reference-only", + "redistribution": "not-redistributed" + }, + { + "id": "china-administrative-reconsideration", + "title": "Administrative Reconsideration Law of the People's Republic of China", + "url": "https://www.samr.gov.cn/zw/zfxxgk/fdzdgknr/fgs/art/2023/art_70518816df484be18f6b38fa295750bb.html", + "usage": "reference-only", + "redistribution": "not-redistributed" + }, + { + "id": "china-administrative-litigation", + "title": "Administrative Litigation Law of the People's Republic of China", + "url": "https://www.npc.gov.cn/c2/c30834/201905/t20190521_278106.html", + "usage": "reference-only", + "redistribution": "not-redistributed" + }, + { + "id": "china-arbitration-law", + "title": "Arbitration Law of the People's Republic of China", + "url": "https://gongbao.court.gov.cn/Details/bf9acfc77d43545588ae53c4009d07.html", + "usage": "reference-only", + "redistribution": "not-redistributed" + }, + { + "id": "mof-accounting-standard-22", + "title": "Accounting Standard for Business Enterprises No. 22", + "url": "https://kjs.mof.gov.cn/zhengcefabu/201704/t20170406_2575699.htm", + "usage": "reference-only", + "redistribution": "not-redistributed" + }, + { + "id": "mof-accounting-standard-37", + "title": "Accounting Standard for Business Enterprises No. 37", + "url": "https://kjs.mof.gov.cn/zhengcefabu/201705/t20170515_2600144.htm", + "usage": "reference-only", + "redistribution": "not-redistributed" + }, + { + "id": "mof-accounting-standard-39", + "title": "Accounting Standard for Business Enterprises No. 39", + "url": "https://kjs.mof.gov.cn/zt/kjzzss/kuaijizhunzeshishi/201512/t20151208_1602631.htm", + "usage": "reference-only", + "redistribution": "not-redistributed" + }, + { + "id": "cncf-glossary", + "title": "Cloud Native Glossary", + "url": "https://glossary.cncf.io/", + "usage": "reference-only", + "redistribution": "not-redistributed" + }, + { + "id": "nist-zero-trust", + "title": "NIST SP 800-207 Zero Trust Architecture", + "url": "https://csrc.nist.gov/pubs/sp/800/207/final", + "usage": "reference-only", + "redistribution": "not-redistributed" + }, + { + "id": "spdx-spec", + "title": "SPDX Specification", + "url": "https://spdx.github.io/spdx-spec/v3.0/", + "usage": "reference-only", + "redistribution": "not-redistributed" + } + ], + "packs": [ + { + "id": "medical", + "version": "medical-zh-v1", + "locale": "zh-CN", + "reviewStatus": "project-seed-needs-domain-review", + "sourceIDs": ["nhc-standards", "who-icd11"], + "terms": [ + {"id":"medical-chief-complaint","term":"主诉","aliases":[],"corrections":["主愫"],"category":"documentation"}, + {"id":"medical-present-illness","term":"现病史","aliases":[],"corrections":["现病时"],"category":"documentation"}, + {"id":"medical-past-history","term":"既往史","aliases":[],"corrections":["既往式"],"category":"documentation"}, + {"id":"medical-allergy-history","term":"过敏史","aliases":[],"corrections":["过敏时"],"category":"documentation"}, + {"id":"medical-family-history","term":"家族史","aliases":[],"corrections":[],"category":"documentation"}, + {"id":"medical-physical-exam","term":"体格检查","aliases":[],"corrections":["体格捡查"],"category":"documentation"}, + {"id":"medical-differential","term":"鉴别诊断","aliases":[],"corrections":["鉴别珍断"],"category":"documentation"}, + {"id":"medical-critical-care","term":"呼吸与危重症医学科","aliases":[],"corrections":[],"category":"department"}, + {"id":"medical-palpitations","term":"心悸","aliases":[],"corrections":["心季"],"category":"symptom"}, + {"id":"medical-dyspnea","term":"呼吸困难","aliases":[],"corrections":[],"category":"symptom"}, + {"id":"medical-hemoptysis","term":"咯血","aliases":[],"corrections":[],"category":"symptom"}, + {"id":"medical-syncope","term":"晕厥","aliases":[],"corrections":[],"category":"symptom"}, + {"id":"medical-hypertension","term":"高血压","aliases":[],"corrections":[],"category":"diagnosis"}, + {"id":"medical-t2dm","term":"2型糖尿病","aliases":["T2DM"],"corrections":[],"category":"diagnosis"}, + {"id":"medical-copd","term":"慢性阻塞性肺疾病","aliases":["COPD"],"corrections":[],"category":"diagnosis"}, + {"id":"medical-chd","term":"冠状动脉粥样硬化性心脏病","aliases":["冠心病"],"corrections":[],"category":"diagnosis"}, + {"id":"medical-af","term":"心房颤动","aliases":["房颤"],"corrections":["心房禅动"],"category":"diagnosis"}, + {"id":"medical-heart-failure","term":"心力衰竭","aliases":["心衰"],"corrections":["心力衰揭"],"category":"diagnosis"}, + {"id":"medical-ckd","term":"慢性肾脏病","aliases":["CKD"],"corrections":["慢性肾张病"],"category":"diagnosis"}, + {"id":"medical-cbc","term":"血常规","aliases":["CBC"],"corrections":[],"category":"examination"}, + {"id":"medical-hba1c","term":"糖化血红蛋白","aliases":["HbA1c"],"corrections":["糖化血红旦白"],"category":"examination"}, + {"id":"medical-egfr","term":"估算肾小球滤过率","aliases":["eGFR"],"corrections":["估算肾小球虑过率"],"category":"examination"}, + {"id":"medical-crp","term":"C反应蛋白","aliases":["CRP"],"corrections":["C反映蛋白"],"category":"examination"}, + {"id":"medical-ecg","term":"心电图","aliases":["ECG"],"corrections":[],"category":"examination"}, + {"id":"medical-mri","term":"磁共振成像","aliases":["MRI"],"corrections":[],"category":"examination"}, + {"id":"medical-spo2","term":"血氧饱和度","aliases":["SpO2"],"corrections":["血氧包和度"],"category":"vital-sign"}, + {"id":"medical-infusion","term":"静脉滴注","aliases":[],"corrections":["静脉低注"],"category":"medication"}, + {"id":"medical-im","term":"肌内注射","aliases":[],"corrections":["肌内住射"],"category":"medication"}, + {"id":"medical-cpr","term":"心肺复苏","aliases":["CPR"],"corrections":["心肺腹苏"],"category":"procedure"}, + {"id":"medical-contraindication","term":"禁忌证","aliases":[],"corrections":["禁忌症"],"category":"safety"} + ] + }, + { + "id": "legal", + "version": "legal-zh-v1", + "locale": "zh-CN", + "reviewStatus": "project-seed-needs-domain-review", + "sourceIDs": [ + "china-civil-code", + "china-criminal-law", + "china-criminal-procedure", + "china-civil-procedure", + "china-administrative-reconsideration", + "china-administrative-litigation", + "china-arbitration-law" + ], + "terms": [ + {"id":"legal-civil-act","term":"民事法律行为","aliases":[],"corrections":[],"category":"civil-law"}, + {"id":"legal-expression-intent","term":"意思表示","aliases":[],"corrections":["意思表式"],"category":"civil-law"}, + {"id":"legal-limitation","term":"诉讼时效","aliases":[],"corrections":["诉讼实效"],"category":"procedure"}, + {"id":"legal-burden-proof","term":"举证责任","aliases":[],"corrections":["举证责认"],"category":"procedure"}, + {"id":"legal-contract-performance","term":"合同履行","aliases":[],"corrections":[],"category":"contract"}, + {"id":"legal-standard-terms","term":"格式条款","aliases":[],"corrections":["格式条宽"],"category":"contract"}, + {"id":"legal-force-majeure","term":"不可抗力","aliases":[],"corrections":["不可抗利"],"category":"contract"}, + {"id":"legal-breach","term":"违约责任","aliases":[],"corrections":["违约责认"],"category":"contract"}, + {"id":"legal-culpa","term":"缔约过失责任","aliases":[],"corrections":["缔约过时责任"],"category":"contract"}, + {"id":"legal-simultaneous-defense","term":"同时履行抗辩权","aliases":[],"corrections":[],"category":"contract"}, + {"id":"legal-insecurity-defense","term":"不安抗辩权","aliases":[],"corrections":["不安抗辨权"],"category":"contract"}, + {"id":"legal-subrogation","term":"代位权","aliases":[],"corrections":["代为权"],"category":"contract"}, + {"id":"legal-revocation","term":"撤销权","aliases":[],"corrections":[],"category":"contract"}, + {"id":"legal-good-faith","term":"善意取得","aliases":[],"corrections":["善意取的"],"category":"property"}, + {"id":"legal-unjust-enrichment","term":"不当得利","aliases":[],"corrections":["不当的利"],"category":"obligation"}, + {"id":"legal-negotiorum","term":"无因管理","aliases":[],"corrections":["无音管理"],"category":"obligation"}, + {"id":"legal-tort","term":"侵权责任","aliases":[],"corrections":[],"category":"tort"}, + {"id":"legal-presumed-fault","term":"过错推定","aliases":[],"corrections":[],"category":"tort"}, + {"id":"legal-joint-liability","term":"连带责任","aliases":[],"corrections":["连带责认"],"category":"liability"}, + {"id":"legal-self-defense","term":"正当防卫","aliases":[],"corrections":["正当防位"],"category":"criminal-law"}, + {"id":"legal-emergency","term":"紧急避险","aliases":[],"corrections":[],"category":"criminal-law"}, + {"id":"legal-bail","term":"取保候审","aliases":[],"corrections":["取保后审"],"category":"criminal-procedure"}, + {"id":"legal-plea","term":"认罪认罚","aliases":[],"corrections":[],"category":"criminal-procedure"}, + {"id":"legal-reconsideration","term":"行政复议","aliases":[],"corrections":["行政富议"],"category":"administrative-law"}, + {"id":"legal-administrative-litigation","term":"行政诉讼","aliases":[],"corrections":[],"category":"administrative-law"}, + {"id":"legal-arbitration-clause","term":"仲裁条款","aliases":[],"corrections":["仲裁条宽"],"category":"dispute-resolution"}, + {"id":"legal-jurisdiction-objection","term":"管辖权异议","aliases":[],"corrections":["管辖权意议"],"category":"procedure"}, + {"id":"legal-evidence-preservation","term":"证据保全","aliases":[],"corrections":[],"category":"procedure"}, + {"id":"legal-property-preservation","term":"财产保全","aliases":[],"corrections":[],"category":"procedure"}, + {"id":"legal-enforcement","term":"强制执行","aliases":[],"corrections":[],"category":"procedure"} + ] + }, + { + "id": "finance", + "version": "finance-zh-v1", + "locale": "zh-CN", + "reviewStatus": "project-seed-needs-domain-review", + "sourceIDs": [ + "mof-accounting-standard-22", + "mof-accounting-standard-37", + "mof-accounting-standard-39" + ], + "terms": [ + {"id":"finance-balance-sheet","term":"资产负债表","aliases":[],"corrections":["资产付债表"],"category":"accounting"}, + {"id":"finance-income-statement","term":"利润表","aliases":[],"corrections":[],"category":"accounting"}, + {"id":"finance-cash-flow","term":"现金流量表","aliases":[],"corrections":["现金刘量表"],"category":"accounting"}, + {"id":"finance-equity","term":"所有者权益","aliases":[],"corrections":[],"category":"accounting"}, + {"id":"finance-receivables","term":"应收账款","aliases":[],"corrections":["应收帐款"],"category":"accounting"}, + {"id":"finance-bad-debt","term":"坏账准备","aliases":[],"corrections":["怀账准备"],"category":"accounting"}, + {"id":"finance-fair-value","term":"公允价值","aliases":[],"corrections":["公允许价"],"category":"valuation"}, + {"id":"finance-amortized-cost","term":"摊余成本","aliases":[],"corrections":["滩余成本"],"category":"valuation"}, + {"id":"finance-effective-interest","term":"实际利率法","aliases":[],"corrections":[],"category":"accounting"}, + {"id":"finance-impairment","term":"资产减值","aliases":[],"corrections":[],"category":"accounting"}, + {"id":"finance-deferred-tax","term":"递延所得税","aliases":[],"corrections":["递延所的税"],"category":"tax"}, + {"id":"finance-wacc","term":"加权平均资本成本","aliases":["WACC"],"corrections":["加权平军资本成本"],"category":"corporate-finance"}, + {"id":"finance-irr","term":"内部收益率","aliases":["IRR"],"corrections":["内部收意率"],"category":"corporate-finance"}, + {"id":"finance-npv","term":"净现值","aliases":["NPV"],"corrections":[],"category":"corporate-finance"}, + {"id":"finance-pe","term":"市盈率","aliases":["P/E"],"corrections":[],"category":"valuation"}, + {"id":"finance-pb","term":"市净率","aliases":["P/B"],"corrections":[],"category":"valuation"}, + {"id":"finance-capital-adequacy","term":"资本充足率","aliases":[],"corrections":[],"category":"banking"}, + {"id":"finance-npl","term":"不良贷款率","aliases":[],"corrections":[],"category":"banking"}, + {"id":"finance-lcr","term":"流动性覆盖率","aliases":["LCR"],"corrections":["流动性复盖率"],"category":"banking"}, + {"id":"finance-leverage","term":"杠杆率","aliases":[],"corrections":[],"category":"risk"}, + {"id":"finance-qe","term":"量化宽松","aliases":[],"corrections":[],"category":"monetary-policy"}, + {"id":"finance-open-market","term":"公开市场操作","aliases":[],"corrections":[],"category":"monetary-policy"}, + {"id":"finance-reverse-repo","term":"逆回购","aliases":[],"corrections":["逆回够"],"category":"money-market"}, + {"id":"finance-repo","term":"回购协议","aliases":[],"corrections":[],"category":"money-market"}, + {"id":"finance-credit-spread","term":"信用利差","aliases":[],"corrections":["信用力差"],"category":"fixed-income"}, + {"id":"finance-duration","term":"久期","aliases":[],"corrections":[],"category":"fixed-income"}, + {"id":"finance-convexity","term":"凸性","aliases":[],"corrections":[],"category":"fixed-income"}, + {"id":"finance-hedging","term":"套期保值","aliases":[],"corrections":["套期保直"],"category":"risk"}, + {"id":"finance-fv-change","term":"公允价值变动损益","aliases":[],"corrections":[],"category":"accounting"}, + {"id":"finance-oci","term":"其他综合收益","aliases":[],"corrections":[],"category":"accounting"} + ] + }, + { + "id": "technology", + "version": "technology-zh-v1", + "locale": "zh-CN", + "reviewStatus": "project-seed-needs-domain-review", + "sourceIDs": ["cncf-glossary", "nist-zero-trust", "spdx-spec"], + "terms": [ + {"id":"technology-cloud-native","term":"云原生","aliases":[],"corrections":["云原声"],"category":"cloud"}, + {"id":"technology-orchestration","term":"容器编排","aliases":[],"corrections":[],"category":"cloud"}, + {"id":"technology-microservices","term":"微服务","aliases":[],"corrections":[],"category":"architecture"}, + {"id":"technology-service-mesh","term":"服务网格","aliases":[],"corrections":["服务网各"],"category":"cloud"}, + {"id":"technology-ci","term":"持续集成","aliases":["CI"],"corrections":["持续急成"],"category":"delivery"}, + {"id":"technology-cd","term":"持续交付","aliases":["CD"],"corrections":[],"category":"delivery"}, + {"id":"technology-iac","term":"基础设施即代码","aliases":["IaC"],"corrections":["基础设施即代马"],"category":"delivery"}, + {"id":"technology-observability","term":"可观测性","aliases":[],"corrections":["可观测行"],"category":"operations"}, + {"id":"technology-tracing","term":"分布式追踪","aliases":[],"corrections":["分布式追棕"],"category":"operations"}, + {"id":"technology-load-balancing","term":"负载均衡","aliases":[],"corrections":[],"category":"operations"}, + {"id":"technology-failover","term":"故障转移","aliases":[],"corrections":[],"category":"reliability"}, + {"id":"technology-idempotency","term":"幂等性","aliases":[],"corrections":["幂等幸"],"category":"distributed-systems"}, + {"id":"technology-eventual-consistency","term":"最终一致性","aliases":[],"corrections":["最终一至性"],"category":"distributed-systems"}, + {"id":"technology-strong-consistency","term":"强一致性","aliases":[],"corrections":[],"category":"distributed-systems"}, + {"id":"technology-event-driven","term":"事件驱动","aliases":[],"corrections":["事件区动"],"category":"architecture"}, + {"id":"technology-message-queue","term":"消息队列","aliases":[],"corrections":[],"category":"architecture"}, + {"id":"technology-ddd","term":"领域驱动设计","aliases":["DDD"],"corrections":[],"category":"architecture"}, + {"id":"technology-rag","term":"检索增强生成","aliases":["RAG"],"corrections":["检索增强生城"],"category":"ai"}, + {"id":"technology-llm","term":"大语言模型","aliases":["LLM"],"corrections":[],"category":"ai"}, + {"id":"technology-vector-database","term":"向量数据库","aliases":[],"corrections":["向量数聚库"],"category":"ai"}, + {"id":"technology-semantic-search","term":"语义检索","aliases":[],"corrections":[],"category":"ai"}, + {"id":"technology-prompt-injection","term":"提示词注入","aliases":[],"corrections":["提示词注人"],"category":"security"}, + {"id":"technology-zero-trust","term":"零信任","aliases":[],"corrections":["零性任"],"category":"security"}, + {"id":"technology-sbom","term":"软件物料清单","aliases":["SBOM"],"corrections":["软件物料青单"],"category":"security"}, + {"id":"technology-supply-chain","term":"供应链安全","aliases":[],"corrections":[],"category":"security"}, + {"id":"technology-dependency-injection","term":"依赖注入","aliases":[],"corrections":[],"category":"software"}, + {"id":"technology-reverse-proxy","term":"反向代理","aliases":[],"corrections":[],"category":"networking"}, + {"id":"technology-garbage-collection","term":"垃圾回收","aliases":[],"corrections":[],"category":"runtime"}, + {"id":"technology-concurrency-control","term":"并发控制","aliases":[],"corrections":[],"category":"distributed-systems"}, + {"id":"technology-version-control","term":"版本控制","aliases":[],"corrections":[],"category":"software"} + ] + } + ] +} diff --git a/Sources/Resources/en.lproj/Localizable.strings b/Sources/Resources/en.lproj/Localizable.strings index a7ab469..89bf4d3 100644 --- a/Sources/Resources/en.lproj/Localizable.strings +++ b/Sources/Resources/en.lproj/Localizable.strings @@ -314,6 +314,23 @@ "rules.subtitle" = "Additional instructions for the LLM when formatting text"; "rules.placeholder" = "e.g. Use Arabic numerals for all numbers"; "rules.empty" = "No rules yet"; +"industry.lexicon.title" = "Industry Vocabulary"; +"industry.lexicon.subtitle" = "Choose the current field to prioritize its terms in speech recognition and smart formatting. Personal terms keep higher priority."; +"industry.lexicon.selection" = "Industry"; +"industry.lexicon.general" = "General"; +"industry.lexicon.medical" = "Medical"; +"industry.lexicon.legal" = "Legal"; +"industry.lexicon.finance" = "Finance & Accounting"; +"industry.lexicon.technology" = "Software Technology"; +"industry.lexicon.medical.description" = "Medical records, common diagnoses, examinations, medication, and procedures."; +"industry.lexicon.legal.description" = "Civil and commercial law, litigation, administrative law, and dispute resolution."; +"industry.lexicon.finance.description" = "Financial statements, valuation, banking risk, and fixed income."; +"industry.lexicon.technology.description" = "Cloud native, distributed systems, AI, and software security."; +"industry.lexicon.term_count_fmt" = "%d built-in terms"; +"industry.lexicon.preview" = "View vocabulary"; +"industry.lexicon.search" = "Search terms"; +"industry.lexicon.no_results" = "No matching terms"; +"industry.lexicon.inactive" = "General recognition is active; no industry terms are added."; "dictionary.title" = "Personal Dictionary"; "dictionary.subtitle" = "Manual terms work immediately. With learning enabled, only local edits to the text Utter just inserted can become candidates."; "dictionary.auto_learning" = "Learn corrections"; diff --git a/Sources/Resources/zh-Hans.lproj/Localizable.strings b/Sources/Resources/zh-Hans.lproj/Localizable.strings index d9ce223..a1fc182 100644 --- a/Sources/Resources/zh-Hans.lproj/Localizable.strings +++ b/Sources/Resources/zh-Hans.lproj/Localizable.strings @@ -314,6 +314,23 @@ "rules.subtitle" = "LLM 整理文本时遵循的额外指令"; "rules.placeholder" = "如：所有数字用阿拉伯数字"; "rules.empty" = "暂无规则"; +"industry.lexicon.title" = "行业词库"; +"industry.lexicon.subtitle" = "选择当前领域，为语音识别和智能整理提供优先术语；个人词条仍具有更高优先级。"; +"industry.lexicon.selection" = "行业"; +"industry.lexicon.general" = "通用"; +"industry.lexicon.medical" = "医疗"; +"industry.lexicon.legal" = "法律"; +"industry.lexicon.finance" = "金融财会"; +"industry.lexicon.technology" = "软件技术"; +"industry.lexicon.medical.description" = "覆盖病历、常见诊断、检查、给药和处置术语。"; +"industry.lexicon.legal.description" = "覆盖民商事、诉讼、行政与常用争议解决术语。"; +"industry.lexicon.finance.description" = "覆盖财务报表、估值、银行风控与固定收益术语。"; +"industry.lexicon.technology.description" = "覆盖云原生、分布式系统、AI 与软件安全术语。"; +"industry.lexicon.term_count_fmt" = "%d 个内置术语"; +"industry.lexicon.preview" = "查看词库"; +"industry.lexicon.search" = "搜索术语"; +"industry.lexicon.no_results" = "没有匹配的术语"; +"industry.lexicon.inactive" = "当前使用通用识别，不添加行业术语。"; "dictionary.title" = "个人词库"; "dictionary.subtitle" = "手动词条立即生效；开启学习后，只有 Utter 刚插入文本里的局部修改才会成为候选。"; "dictionary.auto_learning" = "学习纠正"; diff --git a/Sources/Speech/AppleSpeechAnalyzer.swift b/Sources/Speech/AppleSpeechAnalyzer.swift index d687e17..9efe435 100644 --- a/Sources/Speech/AppleSpeechAnalyzer.swift +++ b/Sources/Speech/AppleSpeechAnalyzer.swift @@ -28,7 +28,8 @@ enum AppleSpeechAnalyzer { try await ensureModel(for: transcriber) return try await transcribe( file: AVAudioFile(forReading: audioURL), - with: transcriber + with: transcriber, + context: context ) } catch { Log.info( @@ -58,12 +59,14 @@ enum AppleSpeechAnalyzer { private static func transcribe( file: AVAudioFile, - with transcriber: SpeechTranscriber + with transcriber: SpeechTranscriber, + context: SpeechRecognitionContext ) async throws -> String { let analyzer = SpeechAnalyzer( modules: [transcriber], options: .init(priority: .userInitiated, modelRetention: .lingering) ) + try await apply(context, to: analyzer) let resultTask = Task { var transcript = "" for try await result in transcriber.results where result.isFinal { @@ -83,11 +86,7 @@ enum AppleSpeechAnalyzer { modules: [transcriber], options: .init(priority: .userInitiated, modelRetention: .lingering) ) - if !context.phrases.isEmpty { - let analysisContext = AnalysisContext() - analysisContext.contextualStrings[.general] = context.phrases - try await analyzer.setContext(analysisContext) - } + try await apply(context, to: analyzer) let resultTask = Task { var transcript = "" for try await result in transcriber.results where result.isFinal { @@ -98,6 +97,16 @@ enum AppleSpeechAnalyzer { return try await analyze(file: file, with: analyzer, resultTask: resultTask) } + private static func apply( + _ context: SpeechRecognitionContext, + to analyzer: SpeechAnalyzer + ) async throws { + guard !context.phrases.isEmpty else { return } + let analysisContext = AnalysisContext() + analysisContext.contextualStrings[.general] = context.phrases + try await analyzer.setContext(analysisContext) + } + private static func analyze( file: AVAudioFile, with analyzer: SpeechAnalyzer, diff --git a/Sources/UI/DictionaryStyleView.swift b/Sources/UI/DictionaryStyleView.swift index 09be7d7..c066d0f 100644 --- a/Sources/UI/DictionaryStyleView.swift +++ b/Sources/UI/DictionaryStyleView.swift @@ -22,6 +22,7 @@ struct DictionaryStyleView: View { if !settings.useCustomSystemPrompt { SettingsPanel { styleSection } } + SettingsPanel { IndustryLexiconView() } SettingsPanel { DictionaryManagementView() } SettingsPanel { editRulesSection } SettingsPanel { customSystemPromptSection } diff --git a/Sources/UI/IndustryLexiconView.swift b/Sources/UI/IndustryLexiconView.swift new file mode 100644 index 0000000..36af7d0 --- /dev/null +++ b/Sources/UI/IndustryLexiconView.swift @@ -0,0 +1,120 @@ +import SwiftUI + +struct IndustryLexiconView: View { + @EnvironmentObject private var settings: AppSettings + @State private var searchText = "" + @State private var showsTerms = false + + private let catalog = IndustryLexiconCatalog.shared + + var body: some View { + VStack(alignment: .leading, spacing: 10) { + HStack(spacing: 12) { + Label(L("industry.lexicon.title"), systemImage: "books.vertical") + .font(.headline) + Spacer() + Picker(L("industry.lexicon.selection"), selection: $settings.industryLexicon) { + ForEach(IndustryLexiconID.allCases) { industry in + Label(industry.label, systemImage: industry.symbolName) + .tag(industry) + } + } + .labelsHidden() + .frame(width: 190) + .accessibilityLabel(L("industry.lexicon.selection")) + } + + Text(L("industry.lexicon.subtitle")) + .font(.caption) + .foregroundStyle(.secondary) + + if let pack = activePack { + Label( + String(format: L("industry.lexicon.term_count_fmt"), pack.terms.count), + systemImage: "checkmark.circle.fill" + ) + .font(.caption) + .foregroundStyle(.secondary) + + Text(L("industry.lexicon.\(pack.id.rawValue).description")) + .font(.caption) + .foregroundStyle(.secondary) + + DisclosureGroup( + L("industry.lexicon.preview"), + isExpanded: $showsTerms + ) { + VStack(alignment: .leading, spacing: 8) { + TextField(L("industry.lexicon.search"), text: $searchText) + .textFieldStyle(.roundedBorder) + .accessibilityLabel(L("industry.lexicon.search")) + + termsList(pack) + } + .padding(.top, 8) + } + .font(.caption) + } else { + Label(L("industry.lexicon.inactive"), systemImage: "minus.circle") + .font(.caption) + .foregroundStyle(.secondary) + .frame(maxWidth: .infinity, alignment: .leading) + } + } + .onChange(of: settings.industryLexicon) { _, _ in + searchText = "" + } + } + + private var activePack: IndustryLexiconPack? { + catalog.pack(for: settings.industryLexicon) + } + + private func termsList(_ pack: IndustryLexiconPack) -> some View { + let terms = filteredTerms(in: pack) + return Group { + if terms.isEmpty { + Text(L("industry.lexicon.no_results")) + .foregroundStyle(.tertiary) + .frame(maxWidth: .infinity, minHeight: 48) + } else { + ScrollView { + LazyVStack(spacing: 0) { + ForEach(terms) { item in + HStack(alignment: .firstTextBaseline, spacing: 10) { + Text(item.term) + .textSelection(.enabled) + if !item.aliases.isEmpty { + Text(item.aliases.joined(separator: " · ")) + .foregroundStyle(.secondary) + .textSelection(.enabled) + } + Spacer() + } + .padding(.horizontal, 10) + .padding(.vertical, 6) + .accessibilityElement(children: .combine) + if item.id != terms.last?.id { Divider() } + } + } + } + .frame(maxHeight: 170) + .background(Color(nsColor: .controlBackgroundColor)) + .clipShape(RoundedRectangle(cornerRadius: 6)) + .overlay { + RoundedRectangle(cornerRadius: 6) + .stroke(Color(nsColor: .separatorColor), lineWidth: 0.5) + } + } + } + } + + private func filteredTerms(in pack: IndustryLexiconPack) -> [IndustryLexiconTerm] { + let query = searchText.trimmingCharacters(in: .whitespacesAndNewlines) + guard !query.isEmpty else { return pack.terms } + return pack.terms.filter { item in + item.term.localizedCaseInsensitiveContains(query) + || item.aliases.contains { $0.localizedCaseInsensitiveContains(query) } + } + } +} diff --git a/Tests/OpenTypeTests/IndustryLexiconTests.swift b/Tests/OpenTypeTests/IndustryLexiconTests.swift new file mode 100644 index 0000000..00f2b49 --- /dev/null +++ b/Tests/OpenTypeTests/IndustryLexiconTests.swift @@ -0,0 +1,173 @@ +import XCTest +@testable import OpenType + +final class IndustryLexiconTests: XCTestCase { + private let catalog = IndustryLexiconCatalog.shared + + func testBundledCatalogHasFourValidatedIndustryPacks() { + XCTAssertEqual(Set(catalog.packs.map(\.id)), Set([ + .medical, .legal, .finance, .technology, + ])) + XCTAssertFalse(catalog.version.isEmpty) + XCTAssertFalse(catalog.rights.isEmpty) + XCTAssertGreaterThanOrEqual(catalog.sources.count, 7) + for pack in catalog.packs { + XCTAssertGreaterThanOrEqual(pack.terms.count, 30, pack.id.rawValue) + XCTAssertFalse(pack.sourceIDs.isEmpty, pack.id.rawValue) + } + } + + func testSelectedPackReachesRecognitionContextWithAliases() throws { + let medical = try XCTUnwrap(catalog.pack(for: .medical)) + let snapshot = PersonalDictionarySnapshot( + entries: [DictionaryEntry(original: "utter", replacement: "Utter")], + editRules: [], + industryLexicon: IndustryLexiconSnapshot(pack: medical) + ) + + XCTAssertEqual(snapshot.recognitionPhrases.first, "Utter") + XCTAssertTrue(snapshot.recognitionPhrases.contains("糖化血红蛋白")) + XCTAssertTrue(snapshot.recognitionPhrases.contains("HbA1c")) + XCTAssertLessThanOrEqual( + SpeechRecognitionContext(phrases: snapshot.recognitionPhrases).phrases.count, + SpeechRecognitionContext.maximumPhraseCount + ) + } + + func testPersonalRecognitionPhrasesKeepPriorityAtContextLimit() throws { + let technology = try XCTUnwrap(catalog.pack(for: .technology)) + let personalEntries = (0..<95).map { + DictionaryEntry(original: "spoken-\($0)", replacement: "PersonalTerm\($0)") + } + let snapshot = PersonalDictionarySnapshot( + entries: personalEntries, + editRules: [], + industryLexicon: IndustryLexiconSnapshot(pack: technology) + ) + + XCTAssertEqual(snapshot.recognitionPhrases.count, 100) + for index in 0..<95 { + XCTAssertTrue(snapshot.recognitionPhrases.contains("PersonalTerm\(index)")) + } + } + + func testPersonalCorrectionWinsOverIndustryCorrection() throws { + let medical = try XCTUnwrap(catalog.pack(for: .medical)) + let snapshot = PersonalDictionarySnapshot( + entries: [DictionaryEntry(original: "禁忌症", replacement: "禁用情况")], + editRules: [], + industryLexicon: IndustryLexiconSnapshot(pack: medical) + ) + + XCTAssertEqual(snapshot.applyReplacements(to: "记录禁忌症"), "记录禁用情况") + } + + func testIndustrySelectionPersistsAndDefaultsToGeneral() { + let suiteName = "IndustryLexiconTests-\(UUID().uuidString)" + let defaults = UserDefaults(suiteName: suiteName)! + defaults.removePersistentDomain(forName: suiteName) + defer { defaults.removePersistentDomain(forName: suiteName) } + + let settings = AppSettings(defaults: defaults) + XCTAssertEqual(settings.industryLexicon, .general) + + settings.industryLexicon = .legal + XCTAssertEqual(AppSettings(defaults: defaults).industryLexicon, .legal) + } + + func testIndustryPromptIsCorrectionOnlyAndContainsCanonicalTerms() throws { + let technology = try XCTUnwrap(catalog.pack(for: .technology)) + let snapshot = PersonalDictionarySnapshot( + entries: [], + editRules: [], + industryLexicon: IndustryLexiconSnapshot(pack: technology) + ) + let prompt = TextProcessor().systemPromptWithPersonalContext( + "基础提示", + inputLanguage: .chinese, + dictionarySnapshot: snapshot + ) + + XCTAssertTrue(prompt.contains("软件技术行业词库")) + XCTAssertTrue(prompt.contains("检索增强生成（RAG）")) + XCTAssertTrue(prompt.contains("不要据此补充原文没有的信息")) + } + + func testEvaluationMeetsTermAccuracyAndRegressionGates() { + let cases = evaluationCases + let baselineAccuracy = termAccuracy(cases: cases, enhanced: false) + let enhancedAccuracy = termAccuracy(cases: cases, enhanced: true) + + XCTAssertGreaterThanOrEqual(enhancedAccuracy, 0.95) + XCTAssertGreaterThanOrEqual(enhancedAccuracy - baselineAccuracy, 0.50) + XCTAssertEqual(nonTargetPreservationRate, 1.0) + } + + private var evaluationCases: [EvaluationCase] { + [ + .init(.medical, "主愫是心季，糖化血红旦白偏高。", ["主诉", "心悸", "糖化血红蛋白"]), + .init(.medical, "建议复查估算肾小球虑过率和C反映蛋白。", ["估算肾小球滤过率", "C反应蛋白"]), + .init(.medical, "给予静脉低注，记录血氧包和度。", ["静脉滴注", "血氧饱和度"]), + .init(.medical, "患者高血压，心电图已完成。", ["高血压", "心电图"]), + .init(.legal, "本案超过诉讼实效，举证责认仍有争议。", ["诉讼时效", "举证责任"]), + .init(.legal, "不可抗利与缔约过时责任需要分别审查。", ["不可抗力", "缔约过失责任"]), + .init(.legal, "当事人提出管辖权意议并申请行政富议。", ["管辖权异议", "行政复议"]), + .init(.legal, "法院已经采取财产保全措施。", ["财产保全"]), + .init(.finance, "资产付债表和现金刘量表需要重编。", ["资产负债表", "现金流量表"]), + .init(.finance, "按滩余成本计算递延所的税。", ["摊余成本", "递延所得税"]), + .init(.finance, "流动性复盖率下降，逆回够规模上升。", ["流动性覆盖率", "逆回购"]), + .init(.finance, "净现值为正，资本充足率保持稳定。", ["净现值", "资本充足率"]), + .init(.technology, "云原声平台采用服务网各和持续急成。", ["云原生", "服务网格", "持续集成"]), + .init(.technology, "可观测行依赖分布式追棕和幂等幸。", ["可观测性", "分布式追踪", "幂等性"]), + .init(.technology, "检索增强生城使用向量数聚库。", ["检索增强生成", "向量数据库"]), + .init(.technology, "软件物料清单用于供应链安全。", ["软件物料清单", "供应链安全"]), + ] + } + + private func termAccuracy(cases: [EvaluationCase], enhanced: Bool) -> Double { + var hits = 0 + var total = 0 + for item in cases { + let snapshot = catalog.snapshot(for: item.industry) + let output = enhanced + ? PersonalDictionarySnapshot( + entries: [], + editRules: [], + industryLexicon: snapshot + ).applyReplacements(to: item.transcript) + : item.transcript + hits += item.expectedTerms.filter(output.contains).count + total += item.expectedTerms.count + } + return total == 0 ? 0 : Double(hits) / Double(total) + } + + private var nonTargetPreservationRate: Double { + let samples: [(IndustryLexiconID, String)] = [ + (.medical, "明天下午三点讨论项目排期。"), + (.legal, "请把会议纪要发给团队。"), + (.finance, "本周完成用户访谈和原型。"), + (.technology, "周末去公园散步。"), + ] + let preserved = samples.filter { industry, text in + PersonalDictionarySnapshot( + entries: [], + editRules: [], + industryLexicon: catalog.snapshot(for: industry) + ).applyReplacements(to: text) == text + }.count + return Double(preserved) / Double(samples.count) + } +} + +private struct EvaluationCase { + let industry: IndustryLexiconID + let transcript: String + let expectedTerms: [String] + + init(_ industry: IndustryLexiconID, _ transcript: String, _ expectedTerms: [String]) { + self.industry = industry + self.transcript = transcript + self.expectedTerms = expectedTerms + } +} diff --git a/docs/research/2026-08-24-chinese-industry-lexicon-landscape.md b/docs/research/2026-08-24-chinese-industry-lexicon-landscape.md new file mode 100644 index 0000000..7a95723 --- /dev/null +++ b/docs/research/2026-08-24-chinese-industry-lexicon-landscape.md @@ -0,0 +1,174 @@ +# Utter 中文本土多行业词库探索 + +## 摘要 + +[事实] 中文语音输入需要同时覆盖通用词、行业术语、机构名、地名、产品名和口语变体；单一通用词表不能代表多行业场景。 + +[事实] THUOCL 提供可直接评估的公开中文词库，官方仓库为 ，许可证为 MIT，README 表明允许研究和商业使用。 + +[事实] THUOCL README 列出 IT16000、财经3830、医学18749、法律9896、汽车1752、饮食8974、地名44805 等分类，主要更新时间集中在 2016—2017 年。 + +[事实] 本地归一化审计得到 157171 行、156668 个 unique；地名文件为 UTF-8 BOM 且使用 CR 行尾，不能假定所有文件是无 BOM 的 LF 文本。 + +[事实] 本地审计显示 IT 类约 3100 条含拉丁字母，需清洗、版本化并人工抽检后才适合进入生产候选集。 + +[事实] 本次审计固定在 THUOCL commit `a30ce79d895d01ab5132a5c74c29703ff7efb4cc`；仓库最后一次提交为 2018-11-21，因此“许可清晰”不代表“内容仍然新鲜”。 + +[建议] 首期以可审计、许可清晰的 THUOCL 作为 bundled 基线，再以行业官方术语作 reference-only 校准，以用户本地词库作 import-only 扩展。 + +[未确认] 本文没有把公开页面可见、可下载或可检索等同于商业整库再分发许可；每个候选来源仍需在发布前取得版本化许可证据。 + +## 三层使用边界 + +### bundled：可随产品发布的候选 + +[建议] 仅纳入许可证、来源、版本、变更记录和再分发范围都可复核的条目；默认保留原始文件哈希和归一化脚本版本。 + +[事实] THUOCL 的官方仓库声明 MIT，并在 README 中允许研究/商业用途，因此可作为 bundled 候选，但仍要保留归属与许可证文本。 + +[建议] bundled 词库只放经过 UTF-8/BOM/换行处理、重复合并、敏感词和误识别抽检的生产子集，不直接把原始下载目录打包进应用。 + +### reference-only：只供查阅或人工对照 + +[事实] 全国科技名词委 与术语在线 覆盖约百万术语、150 多个学科，是高权威参考来源。 + +[未确认] 全国科技名词委及术语在线的公开页面未提供可据以默认整库商业再分发的许可，因此不能按 bundled 处理。 + +[事实] [国家标准全文公开系统](https://openstd.samr.gov.cn/bzgk/gb/std_list?p.p1=0&p.p90=circulation&p.p91=release&p.p92=GB) 明确电子文本仅供个人学习研究，未经授权禁止复制、发行、汇编、翻译和网络传播。 + +[建议] reference-only 只保存来源 URL、标准号、术语释义的必要索引或人工核对结果，不复制受限全文，不离线重发布原文。 + +### import-only：用户本地导入 + +[事实] 搜狗细胞词库官方入口为 ，分类覆盖工程、医学、农林、社科等多个方向。 + +[事实] 百度输入法词库入口为 ，公开分类同样广泛。 + +[未确认] 尚未找到搜狗或百度允许第三方商业整库再分发的明确许可，因此归入 import-only，而不是 bundled。 + +[事实] Rime Ice 采用 GPL-3.0，且上游内容混合，不能仅凭项目许可证推导其中每份上游词表都可按 Utter 方式再分发。 + +[建议] import-only 让用户选择本地文件并在本机解析；产品不预置、不上传、不代替用户传播来源文件，并明确用户需自行承担来源许可责任。 + +## 来源比较 + +| 来源 | 覆盖/特点 | 许可判断 | 产品层 | 主要风险 | +| --- | --- | --- | --- | --- | +| THUOCL 官方仓库 | 共 11 类；其中 IT、财经、医学、法律、汽车、饮食、地名 7 类直接对应首批场景 | MIT；README 允许研究/商业 | bundled 候选 | 2016—2017 分类更新，仓库最后提交于 2018 年，需时效审计 | +| 搜狗细胞词库 | 工程、医学、农林、社科等分类广 | 整库商业再分发许可未找到 | import-only | 文件版本、授权边界和格式差异 | +| 百度输入法词库 | 行业分类广，便于用户本地扩展 | 整库商业再分发许可未找到 | import-only | 来源条目与更新状态不可控 | +| Rime Ice | 中文输入生态词库，上游混合 | GPL-3.0 与上游内容需逐项核对 | import-only | 传递许可、混合来源和再分发义务 | +| 全国科技名词委/术语在线 | 权威、学科广、术语规范 | 公开再分发许可未确认 | reference-only | 不得把检索权限当下载许可 | +| 国家标准全文公开系统 | 国家标准原文与标准元数据 | 电子文本用途受限 | reference-only | 仅个人学习研究等限制 | +| 国家统计局行业/产品分类 | 中文分层代码和行业名称，适合建立行业导航 | 商业再分发许可未确认 | reference-only | 是分类骨架，不是语音词库 | + +[事实] 国家统计局发布的[国民经济行业分类](https://www.stats.gov.cn/xxgk/tjbz/gjtjbz/201710/t20171017_1758922.html)可作为跨行业目录骨架，但其代码和名称不能替代每个行业的真实口述术语、缩写、品牌名和混淆词。 + +## 行业官方来源矩阵 + +| 行业 | 优先官方来源 | 建议用途 | 许可/状态边界 | +| --- | --- | --- | --- | +| 程序 IT | THUOCL IT；术语在线 | bundled 基线与 reference 校准 | THUOCL MIT；官方术语再分发未确认 | +| 网安 | TC260 官方信息 | 术语、控制项和缩写对照 | reference-only，许可未确认 | +| 医疗/中医 | 医学词码政策 ；医学标准 | 诊疗词、病种、方剂和编码校准 | reference-only，需核验具体许可 | +| 法律司法 | THUOCL 法律；[国家法律法规数据库](https://flk.npc.gov.cn/) | 案由、程序、法条短语 | bundled 候选仅限 THUOCL；法规全文不打包 | +| 金融会计证券保险 | [证监会证券期货统计指标标准](https://www.csrc.gov.cn/csrc/c100028/c1001697/content.shtml)及官方代码表 | 机构、证券、会计和险种术语 | 官方许可未确认，reference-only | +| 制造 | 工信部行业标准公告 | 工艺、设备、材料和型号词 | 标准正文许可未确认 | +| 能源电力石化 | 国家能源局标准查询 | 电力、油气、煤炭和安全词 | reference-only，需逐项授权 | +| 建筑 | [建筑幕墙术语国家标准](https://openstd.samr.gov.cn/bzgk/std/newGbInfo?hcno=A43C9BDD13E469F166BCB6E328E4BE3F)、住建及术语在线 | 构件、施工、造价和规范缩写 | reference-only，标准文本受限 | +| 交通物流汽车 | 交通运输部公开文件 ；THUOCL 汽车 | 车型、道路、仓储和运输词 | THUOCL 可作候选，其余许可未确认 | +| 农业林牧渔 | 农业标准目录 | 作物、农机、兽医和水产词 | reference-only；本地导入可扩展 | +| 教育 | 教育学科目录 | 学科、课程、考试和教育管理词 | 目录引用范围需确认 | +| 政务客服 | [数字政府术语国家标准](https://std.samr.gov.cn/gb/search/gbDetailed?id=55B3892F9612336AE06397BE0A0A4BF9)；政务公开术语 | 部门、事项、流程和客服表达 | 官方许可未确认，优先 reference | +| 电商零售 | 国家标准全文公开系统电子商务术语 | SKU、履约、售后和促销词 | 标准电子文本不 bundled | +| 通信 | 中国通信标准化协会通信词典 | 网络、协议、基站和运维词 | 公开检索不等于再分发许可 | +| 航空航天 | [民航标准](https://www.caac.gov.cn/XXGK/XXGK/TZTG/202203/t20220304_212195.html)；[航天术语：运输系统](https://openstd.samr.gov.cn/bzgk/std/newGbInfo?hcno=F1EA043CDC3796D804115FA575906563) | 航班、机场、航材、运载和飞控词 | reference-only，型号词需人工复核 | +| 出版传媒 | [出版物发行术语国家标准](https://std.samr.gov.cn/gb/search/gbDetailed?id=71F772D7E49FD3A7E05397BE0A0AB82A)及术语在线 | 版式、编辑、版权和媒介词 | 具体标准许可未确认 | +| 环保气象 | 环保工程术语 ；气象仪器术语 | 污染物、监测、气象观测和仪器词 | 标准/公告只作 reference | +| 旅游餐饮 | 文旅标准 ；THUOCL 饮食 | 景区、酒店、菜品和服务词 | THUOCL 饮食可作候选；菜谱来源需审计 | + +## 分期优先级 + +### P0：首个可交付闭环 + +[建议] 先纳入 THUOCL 的通用可审计子集，重点覆盖 IT、医学、法律、汽车、财经和饮食；地名另建高风险数据管线。 + +[建议] 建立词条记录：原始词、归一化词、领域、来源 URL、来源版本、许可证证据、哈希、审核状态和排除原因。 + +[建议] 处理 BOM、CR/LF、Unicode 规范化、全半角、大小写、空白、重复项、拉丁字母混排和明显乱码；每一步输出可复现统计。 + +[建议] 用固定短语集、领域混淆对和人工抽检建立回归集；词库命中只影响候选排序，不得绕过隐私、敏感词和输出保护规则。 + +### P1：行业扩展 + +[建议] 在 P0 稳定后增加网安、制造、能源、建筑、交通物流、农业、教育、政务客服、电商、通信和环保气象。 + +[建议] 每个行业至少形成“术语—同义词—缩写—易混淆词—保护词”五元组，并保留行业专家或官方出处的审核状态。 + +[建议] 对时效敏感的公司、型号、政策、标准号设置版本过期时间；过期词退回候选区，不静默删除历史输入。 + +### P2：长尾与用户生态 + +[建议] 再覆盖航空航天、出版传媒、旅游餐饮，以及地方机构、方言和企业内部术语。 + +[建议] 提供 import-only 的拖放导入、预览、去重、回滚、来源标注和本地删除；导入文件不上传云端。 + +[未确认] 行业官方来源是否允许批量下载、训练或再分发，必须在每个 P1/P2 包装前逐项确认。 + +## 数据质量门禁 + +[建议] 门禁一：输入编码必须可检测且可复现；发现 UTF-8 BOM、CR 行尾或异常字节时记录并转化，不覆盖原始快照。 + +[建议] 门禁二：归一化后按 Unicode、空白、全半角和大小写规则生成 canonical key；重复条目合并时保留全部来源。 + +[建议] 门禁三：每个词条必须有领域、来源和审核状态；无法追溯的条目只能进入 quarantine，不能 bundled。 + +[建议] 门禁四：抽检按行业、长度、汉字/拉丁字母比例、数字符号、低频度和高混淆度分层；IT 约 3100 条拉丁混排项必须单独抽检。 + +[建议] 门禁五：拒绝乱码、孤立标点、异常超长字符串、明显个人信息、恶意提示片段和未经确认的敏感词；拒绝原因可查询。 + +[建议] 门禁六：发布包提供行数、unique 数、领域分布、删除数、冲突数、版本日期、SHA-256 和审计报告；任何统计变化都能定位到输入版本。 + +[建议] 门禁七：变更采用版本化清单和可回滚包，禁止下载源变化后无记录地重生成生产词库。 + +[建议] 门禁八：保护词槽位必须 100% 保留；词库不得把密码、令牌、身份证号、银行卡号等模式变成可自动注入内容。 + +## 真实音频验收门禁 + +[事实] 现有确定性单元测试、固定文本回归和词库命中统计只能说明代码路径或文本处理行为，不能等同于真实音频通过。 + +[建议] 真实音频集按行业、口音、语速、噪声、设备、远近场、代码混说、数字和专名建立可追溯子桶，并固定训练/验收隔离。 + +[建议] 通过条件为 exact recall ≥90%，相对基线提升 ≥10 个百分点；每个关键子桶 recall ≥85%。 + +[建议] 通过条件为 CER/WER 相对基线恶化 ≤0.3 个百分点；词库引入的假插入率 ≤0.5%。 + +[建议] 保护槽位保留率 100%，静音片段不得产生新增输出；新增处理 p95 延迟 ≤100 ms。 + +[建议] 实时因子（RTF）相对基线恶化 ≤5%；同时记录冷启动、热缓存和模型版本，避免平均值掩盖尾延迟。 + +[建议] 任何一项指标不达标都停留在候选或灰度阶段；按行业和错误类型回收词条，修订后用同一音频集复测。 + +## 发布记录最小字段 + +[建议] 每次发布记录词库包版本、上游快照日期、构建时间、构建工具版本和输入文件 SHA-256。 + +[建议] 每个词条保留领域标签、原始拼写、canonical 拼写、来源层级、许可证状态、审核人和审核日期。 + +[建议] 每次剔除记录原因码，例如乱码、重复、过期、许可不明、隐私风险、音频误识别或保护冲突。 + +[建议] 质量报告同时保存基线模型、测试集版本、分桶样本数、exact recall、CER、WER、假插入率和延迟分位数。 + +[建议] 灰度期间记录用户撤回、误识别反馈和回滚点；反馈只用于本地或经授权的聚合统计。 + +[未确认] 任何行业来源若只有搜索结果摘要、转载页面或个人整理，不能作为最终授权证据。 + +## 结论与下一步 + +[建议] 采用“THUOCL bundled 基线 + 官方术语 reference-only + 用户文件 import-only”的三层架构，能在覆盖面、许可风险和可回滚性之间取得可审计的起点。 + +[建议] P0 只发布经过质量门禁且拥有来源与 MIT 证据的子集；其余行业先建立索引和人工校准，不因分类数量多而直接打包。 + +[未确认] 本报告未替代法律许可审查、行业专家验收或真实音频实验；在这些证据补齐前，不宣称任何 reference-only 或 import-only 来源可商业再分发。 + +[建议] 最小交付顺序为：冻结来源快照 → 归一化与审计 → 人工分层抽检 → 确定性回归 → 真实音频门禁 → 灰度发布 → 版本化复盘。 diff --git a/docs/research/2026-08-24-industry-lexicon-asr-research.md b/docs/research/2026-08-24-industry-lexicon-asr-research.md new file mode 100644 index 0000000..d66323f --- /dev/null +++ b/docs/research/2026-08-24-industry-lexicon-asr-research.md @@ -0,0 +1,271 @@ +# Utter 行业词库、上下文偏置与可重复评测调研 + +> 日期：2026-08-24 +> 状态：研究结论与评测门禁提案；**没有把尚未跑过的真实音频评测写成“已通过”** +> 范围：桌面语音输入的行业术语增强，医疗优先；兼顾本地 Apple Speech、Whisper/Qwen 与可选云端 ASR +> 证据口径：`[事实]` 来自官方标准、政府/专业组织或厂商原始文档；`[建议]` 是针对 Utter 的产品与工程判断；`[未知]` 表示原始页面没有给出足以支持商业再分发的许可。 + +## 1. 结论先行 + +1. `[事实]` 主流 ASR 的“行业词库”本质是**上下文偏置**，不是输入法式的全量词典替换。Apple 建议 contextual phrases 尽量只有一两个词、总数不超过 100；Google 明确提醒 boost 越高越容易产生假阳性；火山也要求避免单字、常见词和无实体意义的高频词。[Apple `contextualStrings`](https://developer.apple.com/documentation/speech/sfspeechrecognitionrequest/contextualstrings)、[Google PhraseSet boost](https://cloud.google.com/speech-to-text/docs/reference/rest/v1/projects.locations.phraseSets)、[火山引擎热词](https://www.volcengine.com/docs/6561/155739?lang=zh) +2. `[建议]` Utter 不应把几千或几万条行业词全部塞进每次解码，而应维护一个有来源的完整行业包，再按语言、当前 App/窗口、屏幕词、最近使用和用户个人词典，**每次动态选择最多 100 条短词**送给 Apple；Whisper/Qwen 使用同一选择结果映射到各自的 prompt/context。 +3. `[建议]` 首批内置顺序应为：**医疗 + IT/网络安全（P0）**，然后金融、能源（P1），航空航天（P2）。医疗感知价值最高，但中文权威资源的商业再分发权最不清晰；IT/网络安全有微软多语言术语和 NIST JSON，最容易形成许可可审计的第二个行业包。 +4. `[事实]` 医疗领域不能把“权威”与“可随 App 分发”混为一谈。SNOMED CT 要求供应商成为 Affiliate 并管理 sublicense；WHO ICD-11 是 CC BY-ND 3.0 IGO；术语在线明确标注版权所有；国家卫健委下载页没有为 App 内置词库给出明确再分发许可。[SNOMED 供应商许可](https://docs.snomed.org/snomed-ct-practical-guides/vendor-introduction-to-snomed-ct/7-licensing)、[ICD-11 许可](https://icd.who.int/docs/icd-api/license/)、[术语在线](https://www.termonline.cn/)、[国家卫健委疾病代码表](https://www.nhc.gov.cn/mohwsbwstjxxzx/dczlxz/201809/8e46a2feccac441a9d080fce33aba60d.shtml) +5. `[建议]` 第一版医疗包应只使用许可已核清的 MeSH/LOINC 子集和团队原创、经医学专家复核的中文 spoken-form 映射；不得抓取术语在线、SNOMED 或国家临床版整表后直接打包。 +6. `[建议]` “测试通过”必须表示同一批真实录音在**行业包关闭/开启**两种条件下做配对比较，同时满足术语召回提升、通用 CER/WER 不退化、负样例不误吸附、数字/单位完全保真和延迟门禁。单元测试、TTS 样例或几个手工口述只证明管线可运行，不能证明行业识别效果已通过。 + +## 2. 官方 ASR 能力说明了怎样的产品形态 + +| 能力 | 官方事实 | 对 Utter 的直接含义 | +|---|---|---| +| Apple 运行时短语 | `SFSpeechRecognitionRequest.contextualStrings` 和 macOS 新 Speech framework 的 `AnalysisContext.contextualStrings` 用于提高系统词表外短语的识别概率；建议一到两个词，总计不超过 100。[旧 API](https://developer.apple.com/documentation/speech/sfspeechrecognitionrequest/contextualstrings)、[新 API](https://developer.apple.com/documentation/speech/analysiscontext/contextualstrings) | 每次只挑最相关的 100 条，不能把全行业包直接送入 Apple ASR。 | +| Apple 自定义语言模型 | `SFCustomLanguageModelData` 能加入带 count 的偏置短语，以及用 X-SAMPA 表示的自定义发音；已有系统词汇或不支持的音素会被忽略。[训练数据对象](https://developer.apple.com/documentation/speech/sfcustomlanguagemodeldata)、[自定义发音](https://developer.apple.com/documentation/speech/sfcustomlanguagemodeldata/custompronunciation)、[加权短语](https://developer.apple.com/documentation/speech/sfcustomlanguagemodeldata/phrasecount) | 稳定行业包可在运行时短语 A/B 通过后，再升级为按 locale/version 编译的本地语言模型；发音不是简单别名字段。 | +| Whisper 初始提示 | OpenAI Whisper 官方实现提供 `initial_prompt`，并把它编码成首窗 prompt tokens；prompt 受模型文本上下文预算约束。[官方 `transcribe.py`](https://github.com/openai/whisper/blob/main/whisper/transcribe.py) | 传规范拼写和短术语序列，不传定义、说明或“请正确识别”之类编辑指令。每段滚动且严格限 token。 | +| Qwen3-ASR context | Qwen 官方 API toolkit 的 `--context` 是“引导 ASR、改善特定术语识别”的文本上下文。[Qwen3-ASR Toolkit](https://github.com/QwenLM/Qwen3-ASR-Toolkit) | 与 Whisper 共用经过净化和限量的 context 文本；是否支持当前本地运行器仍须以实际版本和调用链验证。 | +| 火山热词表 | 官方文档支持中英文，每表最多 5000 个热词、每词少于 10 个字、权重 1–10；数字/特殊符号要转为口述形式，并明确要求避免常见词和无实体高频词。[热词管理](https://www.volcengine.com/docs/6561/155739?lang=zh) | 源数据应同时保存 `canonical` 与 `spokenForms`；向火山发送时做供应商特定序列化，不能把供应商约束污染通用词库。 | +| Google PhraseSet | PhraseSet/CustomClass 用于提高稀有词和常用短语概率；官方称推荐 boost 通常为 0–20，并明确更高 boost 会提高假阳性风险，建议二分调参。[模型适配](https://cloud.google.com/speech-to-text/docs/adaptation-model)、[PhraseSet API](https://cloud.google.com/speech-to-text/docs/reference/rest/v1/projects.locations.phraseSets) | 必须有“词未出现”的负样例；权重只能由评测调出，不能按词库来源主观设成最高。 | +| AWS 自定义词表 | Amazon Transcribe 表格包含 `Phrase` 与 `DisplayAs`；当前官方说明 `IPA` 和 `SoundsLike` 已不再支持并会被忽略，`DisplayAs` 可保存 C++ 等符号形式。[自定义词表表格](https://docs.aws.amazon.com/transcribe/latest/dg/custom-vocabulary-create-table.html) | 通用 schema 可保留 spoken form，但供应商 adapter 必须声明真实支持范围；不能因为字段存在就声称该引擎使用了发音。 | + +由这些接口可以得到一个稳定结论：**行业包是候选词来源，ASR adapter 决定如何偏置，后处理器只做有证据的规范化**。三者需要分层，否则很容易出现“词库里有某词，所以把所有近音普通词都替换成它”的误改。 + +## 3. 可用行业术语源与再分发边界 + +### 3.1 医疗：权威资源多，但许可必须逐源处理 + +| 资源 | 能提供什么 | 可再分发性判断 | 第一版用法 | +|---|---|---|---| +| NLM MeSH | 生物医学主题词、同义词和层级；NLM 提供 XML、RDF、MARC 下载及 API。[MeSH 下载](https://www.nlm.nih.gov/databases/download/mesh.html) | **有条件可用。** NLM 条款要求显著注明来源；再分发者要保持最新版本，或明确说明不是最新数据。NLM 同时提醒数据可能包含在美国或美国之外受版权保护的材料。[NLM 数据条款](https://www.nlm.nih.gov/databases/download.html) | 优先抽取 descriptor/entry term 的英文规范拼写和同义词；保留 production year、ID、源 URL 和 NLM acknowledgement。不要打包 scope note/第三方文本。 | +| LOINC | 实验室检查与临床观察的代码、long common name、short name、display name。[LOINC](https://loinc.org/) | **有条件可用。** 官方许可允许商业/非商业使用、复制和分发，但要求产品内 notice、版本/发布日期；带第三方版权的行必须一并保留相应 notice，Part/RSNA/SNOMED 关联还有额外限制。[LOINC License](https://loinc.org/license) | 只取最常听写的检验/生命体征名称；构建时排除或单独审计 `EXTERNAL_COPYRIGHT_NOTICE` 非空记录，随 App 提供要求的 notice。 | +| WHO ICD-11 | 疾病分类，多语言内容与 API；官方 API 首页提供注册访问和本地部署选项。[ICD API](https://icd.who.int/icdapi) | **先不打包。** 内容是 CC BY-ND 3.0 IGO；官方说明映射、crosswalk 和翻译不在通用许可内，需要另行书面协议。[许可页](https://icd.who.int/docs/icd-api/license/)、[许可说明 PDF](https://icd.who.int/en/docs/icd11-license.pdf) | 可用作人工核对来源；在法务确认“抽取子集 + 添加 spoken form”是否构成改编前，不纳入可再分发包。 | +| SNOMED CT | 广泛的临床概念、描述和关系。[获取 SNOMED CT](https://www.snomed.org/get-snomed) | **不可作为无条件内置包。** 产品供应商需成为 Affiliate；向用户提供含 SNOMED 的产品还要管理 sublicense，非成员地区可能收费和申报。[供应商许可指南](https://docs.snomed.org/snomed-ct-practical-guides/vendor-introduction-to-snomed-ct/7-licensing) | 只作为未来持牌企业版选项；不放入普通安装包和公共测试语料。 | +| 全国科技名词委 / 术语在线 | 国务院授权的科技名词审定机构；官网称已审定公布 60 万条规范名词，术语在线称汇聚 80 多万条。[全国科技名词委](https://www.cnctst.cn/)、[术语在线服务介绍](https://new.termonline.cn/languageCommittee/) | **未知，默认不可再分发。** 委员会官网和术语在线均标明版权所有，未找到允许商业 App 批量抓取、修改和再分发的公开许可。[术语在线](https://www.termonline.cn/) | 中文词形的人工核对与专家审核入口；未经书面许可不抓整库、不复制定义、不把搜索结果批量打包。 | +| 国家卫健委 / 国家中医药管理局公开附件 | 疾病分类国家临床版使用线索、中医病证分类与临床诊疗术语附件。[疾病分类标准说明](https://www.nhc.gov.cn/mohwsbwstjxxzx/s8553/201610/25a695abe55c464897777c30290cc4d3.shtml)、[中医术语通知及附件](https://www.gov.cn/zhengce/zhengceku/2020-11/24/content_5563703.htm) | **未知。** 公开下载与政策要求使用不等于授权第三方 App 批量复制、改写和再分发；原始页面未给出适用于该场景的许可。 | 作为需求/规范核对；若要入包，先取得权利方书面确认，并固定具体附件版本与校验和。 | +| RxNorm | 美国药品规范名称、RXCUI 等；NLM 提供 full release 和 current prescribable content。[RxNorm files](https://www.nlm.nih.gov/research/umls/rxnorm/docs/rxnormfiles.html) | **不适合直接整包。** full release 需要 UMLS 协议，且可能含第三方专有词表；NLM FAQ 说明即使 RxNorm 本身不收费，源词表仍可能需要额外许可。[RxNorm FAQ](https://www.nlm.nih.gov/research/umls/rxnorm/faq.html) | 只在逐行 source vocabulary 审计后考虑美国药名子集；中国市场药品名应另找有明确许可的权威来源。 | + +医疗包的实用边界：它是**听写辅助，不是诊断编码器**。疾病名、药名、检验名和术式即使拼写正确，也不能证明医学事实或编码正确；任何数字、剂量、左右侧、阴阳性和否定词都必须保真，并允许用户快速回看原始识别。 + +### 3.2 适合作为后续内置包的其他行业 + +| 优先级 | 行业与来源 | 权威性和数据形态 | 许可/风险 | 建议 | +|---|---|---|---|---| +| P0 | IT/软件：Microsoft Terminology | 微软官方称术语可作为 IT glossary 基础，支持近 100 种语言并提供 TBX 下载，适合直接取得 `zh-CN ↔ en-US` 规范界面和技术词。[术语资源](https://learn.microsoft.com/en-us/globalization/reference/microsoft-terminology) | 官方 Globalization License 授予使用、复制和分发权，但要求下游条款至少同等保护、显示版权 notice，并含 indemnity 条款；上线前需把条件落实到 App 许可和第三方 notices。[许可](https://learn.microsoft.com/en-us/globalization/license-agreement) | 与医疗同时做 P0。只选技术名词，不把完整产品句子当词条；保留 Microsoft attribution。 | +| P0 | 网络安全：NIST CSRC Glossary | NIST 官方在线 glossary 聚合 NIST/CNSSI 标准、指南和技术出版物术语，提供每日更新 JSON；页面当前显示 10,000+ term records，并提示同一词可能有多种来源定义。[NIST Glossary](https://csrc.nist.gov/glossary) | NIST 网页除明确标注版权内容外视为公共信息，可分发/复制并建议署名；但 glossary 也聚合其他来源，不能把所有定义自动视为 NIST 原创。[NIST reuse](https://www.nist.gov/copyrights-disclaimers) | 优先取 term/abbreviation，不打包定义；按 source document 去重，保留 NIST/CNSSI 来源，过滤第三方版权标记。 | +| P1 | 金融：EDM Council FIBO | 专业组织维护的金融业务本体，覆盖实体、贷款、证券、衍生品、指数、市场数据；官方 ontology guide 说明其目标包含金融术语标准化，发布 ontology 使用 MIT。[FIBO repo](https://github.com/edmcouncil/fibo)、[Ontology Guide](https://github.com/edmcouncil/fibo/blob/master/ONTOLOGY_GUIDE.md) | MIT 许可清晰；标签以英文为主，自动机翻不能冒充行业规范中文。 | 提取 release maturity 的 `rdfs:label`/synonym，中文由金融专家与公开可授权来源复核。优先高频产品、指标和机构词，不把关系本体整体塞入 ASR。 | +| P1 | 能源：U.S. EIA Glossary | 美国能源信息署官方 glossary，覆盖煤、电力、天然气、核能、石油、可再生能源等分类。[EIA Glossary](https://www.eia.gov/tools/glossary/index.php) | EIA 明确允许使用/分发其网站数据、文件、数据库和信息产品，建议注明来源与日期；第三方受保护材料例外。[EIA copyrights and reuse](https://www.eia.gov/about/copyrights_reuse.php) | 许可风险低，适合做英文能源包；中文名称需独立复核并标注翻译责任方。 | +| P2 | 航空/航天：FAA Pilot/Controller Glossary + NASA Thesaurus | FAA glossary 当前有约 1,300 个空管通信术语；NASA Thesaurus 提供 SKOS、OWL、ZThes、CSV/TXT 的完整机器可读下载。[FAA glossary](https://www.faa.gov/air_traffic/publications/atpubs/pcg_html/)、[NASA Thesaurus](https://www.sti.nasa.gov/nasa-thesaurus/) | 美国政府内容通常可复用，但 FAA 明确混入标记为 `[ICAO]` 的术语，NASA/FAA 页面也可能包含第三方材料；全球分发前应过滤并做一次源级许可审计。航空又是安全关键场景。 | 先做内部 benchmark，不作为第一批默认开启；排除 `[ICAO]`/第三方来源，用户主动选择后启用。 | +| P2 | 欧盟公共事务/法律：EuroVoc | 欧盟出版局维护的 24 个欧盟语言加 3 个候选国语言的多学科词库，提供多种 RDF/SKOS/XML/Excel 发布格式。[EuroVoc](https://op.europa.eu/en/web/eu-vocabularies/dataset/-/resource?uri=http%3A%2F%2Fpublications.europa.eu%2Fresource%2Fdataset%2Feurovoc) | 欧委会有开放复用文件政策，但具体数据包仍应读取其版本 metadata/rights；没有中文，中文翻译不能继承原标签权威性。[欧委会文件复用决定](https://eur-lex.europa.eu/legal-content/EN/ALL/?uri=CELEX%3A32011D0833) | 仅在欧洲语言用户需求明确时加入，不占首批中文资源。 | + +## 4. 首批内置包建议 + +### 4.1 P0：医疗 + +`[建议]` 完整包目标不是越大越好，而是覆盖高价值、易错、常用且可授权的术语。第一版建议按以下桶采样，每条都要有至少一个真实音频回归样例： + +- 疾病/症候与常用别称：例如规范名、临床简称、容易与普通词混淆的近音词。 +- 检验与生命体征：LOINC 高频项目、完整中文名、常见英文缩写；数值和单位不作为可自由替换文本。 +- 常用通用名药物：仅纳入许可来源清楚的通用名；商品名和不同市场药名另表管理。 +- 手术/检查/治疗：长术式、英文缩写和中英混说。 +- 解剖部位、左右侧、分级、阴阳性、给药途径：这些属于高风险保护项，宁可不改也不能猜。 + +完整行业包可以有数百至数千条，但 Apple 单次激活集合仍不得超过 100。默认行业包不能覆盖医院内部简称、医生姓名、院内药品商品名和科室模板，这些应由组织词库/个人词典覆盖，优先级高于内置行业包。 + +### 4.2 P0：IT/网络安全 + +`[建议]` 这是最适合验证通用框架的第二个包：中英混说、大小写、缩写和符号形式多，且 Microsoft TBX 与 NIST JSON 都是机器可读官方来源。优先覆盖： + +- 操作系统、云服务、网络协议、编程语言和常见工具名； +- 认证、加密、漏洞、威胁建模、零信任等安全术语； +- `canonical` 与口述形式分开，例如显示 `Kubernetes`，口述可记录为用户实际读法； +- C++、C#、IPv6 等符号词必须通过引擎 adapter 转换，不能让通用清洗删除符号。 + +### 4.3 P1：金融与能源 + +金融词通常涉及数值、百分比、币种、期限和否定条件，能源词通常涉及单位、化学式和缩写。两者都应复用医疗的“保护数字/单位 + 负样例”门禁。FIBO/EIA 只提供英文权威基础；中文正式发布前必须由相应行业人员复核，不应直接把机器翻译标为 `approved`。 + +## 5. 词库数据模型与构建规则 + +W3C SKOS 把 `prefLabel`、`altLabel`、`hiddenLabel` 分开；这很适合作为 canonical、可显示别名和只用于匹配的口述形式的概念基础。[SKOS Reference](https://www.w3.org/TR/skos-reference/)、[SKOS Primer](https://www.w3.org/TR/skos-primer/Overview.html) + +建议的最小运行时条目： + +```json +{ + "id": "medical:loinc:4548-4", + "industry": "medical", + "locale": "zh-CN", + "canonical": "糖化血红蛋白", + "aliases": ["血红蛋白 A1c", "HbA1c"], + "spokenForms": ["糖化血红蛋白", "艾尺比艾万西"], + "doNotReplace": ["糖化血红蛋白数值", "阴性", "阳性"], + "baseWeight": 0.6, + "sourceId": "LOINC:4548-4", + "sourceVersion": "", + "sourceURL": "https://loinc.org/4548-4/", + "licenseId": "LicenseRef-LOINC", + "redistribution": "conditional", + "reviewStatus": "needs-medical-review" +} +``` + +上例只说明字段，不代表该中文 spoken form 已获医学/语音学审核，也不代表该 LOINC 行没有第三方 notice。 + +构建规则： + +1. `locale` 使用 BCP 47 标签，不用含糊的 `Chinese`/`English`；BCP 47 定义了语言标签结构与语义。[RFC 5646](https://www.rfc-editor.org/info/rfc5646/) +2. `canonical` 是最终可显示形式；`aliases` 可以显示；`spokenForms` 只用于 ASR/context 匹配，默认不能直接覆盖输出。 +3. 每个同形异义概念独立 ID；不能仅以字符串做主键。 +4. 统一做 Unicode normalization，但原始 spelling 仍须保留；Unicode UAX #15 说明 NFC/NFKC 的等价与兼容分解差异。[Unicode normalization](https://unicode.org/reports/tr15/) +5. 每个 source 有单独 manifest：版本、下载时间、原始 URL、校验和、许可全文/URL、attribution、允许字段、排除条件、是否允许翻译/修改/商业分发。 +6. `licenseId` 优先使用 SPDX short identifier；非 SPDX 许可用稳定的 `LicenseRef-*`，不能为了方便把自定义条款写成 `CC0` 或 `MIT`。[SPDX License List](https://spdx.org/licenses/) +7. 构建器默认拒绝 `redistribution = unknown/restricted` 的条目进入 App bundle；内部研究集和可分发产品集必须物理分离。 +8. 定义、scope note 和例句默认不进入运行时包。ASR 只需要短词、别名、发音和少量权重；少复制文本也降低许可和包体风险。 + +## 6. 建议的识别与后处理链路 + +```text +用户选择行业包 + → 合并组织词库、个人词典、当前屏幕/窗口候选 + → 按 locale、近期度、稀有度、当前上下文和历史命中选 Top-K + → 引擎 adapter（Apple contextualStrings / Whisper prompt / Qwen context / 云端热词表） + → 原始 ASR + → 仅基于 ASR 候选、canonical/alias 与上下文做忠实规范化 + → 数字、单位、左右侧、阴阳性、否定词和药物剂量 guard + → 最终文本 + 本地质量诊断 +``` + +关键约束： + +- 常见词、单字、否定词和数字不参与正向 boost。 +- 同一概念的多个 spoken form 共享总权重，避免枚举越多权重越高。 +- 行业包只改变候选概率，不允许 LLM 因“医学上更合理”而补充未说出的诊断、剂量、结论或动作。 +- 当词典命中与原始声学结果冲突或后处理 guard 失败时，回退原始 ASR，不进行强制替换。 +- 离线行业版的词库选择、prompt 生成、评测日志和更新都应留在本机；远程 ASR 只有用户显式启用时才上传其所需热词。 +- 行业包版本随评测结果一起固定。换源版本、翻译、权重、模型或 prompt 都视为新的待评测配置。 + +## 7. 可重复评测标准 + +### 7.1 评测设计 + +`[事实]` NIST SCTK 的 `sclite` 按插入、删除、替换进行标准 ASR 对齐计分；其文档还指出普通话在词边界不明确时常以字符错误为主要指标。[SCTK `sclite`](https://github.com/usnistgov/SCTK/blob/master/doc/sclite.htm)、[Mandarin scoring options](https://github.com/usnistgov/SCTK/blob/master/doc/options.htm) + +`[建议]` 每个行业使用冻结的音频 corpus，对完全相同的音频、模型、解码参数运行两次： + +- `control`：行业包关闭，个人词典和其他上下文保持相同； +- `treatment`：只打开指定行业包； +- raw ASR 与 processed text 分别计分，不能用后处理改善掩盖 ASR 退化； +- 按 utterance ID 做配对差异，报告 95% paired-bootstrap confidence interval；区间跨过门禁时结论为“证据不足”，而不是“通过”。 + +首期每个行业至少包含： + +| 分层 | 最低规模 | 目的 | +|---|---:|---| +| 术语正样例 | 200 条 utterance，至少 300 次目标术语出现 | 测 canonical/alias 的 exact recall；每个目标词至少 3 个不同上下文，不用背诵词表。 | +| 近音负样例 | 100 条 utterance | 目标行业词**没有被说出**，但包含常见词、近音词或相邻领域词，测误吸附。 | +| 通用回归 | 100 条 utterance | 日常聊天、邮件、开发文本、静音/短音频，确认开启行业包没有伤害一般输入。 | +| 高风险槽位 | 100 条 utterance | 数字、单位、剂量、左右侧、阴阳性、否定、日期、币种/百分比，要求 exact fidelity。 | + +真实录音至少覆盖 10 名说话人、男女声、不同年龄/口音、内置麦与常见蓝牙/USB 麦、安静与两档噪声。TTS 可用于确定性 CI smoke test，但不能代替真实口音和声学条件。 + +### 7.2 指标 + +| 指标 | 定义 | 为什么需要 | +|---|---|---| +| 中文 CER / 英文 WER | 与冻结 reference 做字符/词编辑距离；分 raw ASR 与 processed 两套报告。 | 防止只挑术语样例而整体退化。 | +| Term Exact Recall | `正确出现的目标 canonical 数 / reference 中目标 canonical 数`；多次出现逐 occurrence 计数，不只看“本条是否出现过”。 | 衡量行业词真正被识别的比例。当前 evaluator 的 `terms exact` 是 per-record presence，正式门禁应升级为 occurrence 级。 | +| Term False Insertion Rate | `未说目标词但输出了行业词的负样例数 / 负样例数`，同时列出混淆对。 | 直接发现 boost 过高和盲目后处理替换。 | +| Protected Fidelity | 数字、单位、药物剂量、左右侧、阴阳性、否定、URL/email/path 的有序 exact match；新增、删除、交换都失败。 | 行业正确拼写不能以改变事实为代价。 | +| Term CER | 只在 reference 的目标术语 span 上做字符编辑距离。 | exact recall 不区分错一个字和完全错词，Term CER 用于诊断。 | +| Latency / RTF | 端到端 p50/p95，另报 ASR、context selection、post-process；长音频报告 real-time factor。 | 防止大词库造成可感知延迟。 | +| Empty/negative hallucination | 静音、非本行业语音及近音负样例中的新增字符/行业词。 | 发现词库把无关输入吸向行业术语。 | + +仓库已有 `scripts/evaluate-voice-quality.py`，当前已能报告 CER、英文 WER、per-record `terms exact`、数字/URL/email/path 保真、静音幻觉和 p50/p95。行业评测至少还需要在 corpus 或对比层补充：`industry`、`condition`、`pair_id`、`negative_terms`、`speaker_id`、`device/noise`，并把术语 presence 升级为 occurrence/span 级。 + +### 7.3 “通过”的产品门禁 + +以下阈值是 `[建议]` 的首期 release gate，不是厂商或标准组织给出的通用行业标准： + +1. 术语正样例 exact recall：开启后 **≥ 90%**，且相对关闭状态绝对提升 **≥ 10 个百分点**；若 control 已 ≥ 90%，则允许以不低于 control 1 个百分点的非劣结论通过。 +2. 每个行业子桶（疾病/药名/检验/术式等）exact recall **≥ 85%**，不能用大量容易词掩盖某一高风险桶失败。 +3. 通用回归集 CER/WER 相对 control 的绝对恶化 **≤ 0.3 个百分点**，并且 paired-bootstrap 95% CI 上界不超过该值。 +4. 近音负样例 Term False Insertion Rate **≤ 0.5%**，且比 control 增加不超过 0.5 个百分点。 +5. 高风险槽位的 exact fidelity **100%**；任何剂量、数字、单位、左右侧、阴阳性或否定翻转都直接失败，不以平均分抵消。 +6. 静音集不得新增行业词；空 reference 输出字符数为 0。 +7. 开启行业包新增的端到端 p95 延迟 **≤ 100 ms**；同一模型/音频上的 p95 RTF 不恶化超过 5%。 +8. 固定随机种子或确定性解码配置重复 3 次，逐条最终文本一致；非确定性云端引擎则报告三次均值、最差值和版本/endpoint，不只报最好一次。 + +只要任何高风险门禁失败，结论应是“词库管线可运行，但该行业包未通过质量门禁”。 + +### 7.4 样例设计 + +以下句子是原创评测模板，仅用于说明覆盖方式，不是医学建议或第三方词库内容： + +| 行业 | 正样例 | 对应负样例/保护点 | +|---|---|---| +| 医疗 | “患者的糖化血红蛋白是百分之七点二。” | “请把糖化的步骤写清楚。”不能吸成医学词；`7.2%` 必须保真。 | +| 医疗 | “右侧股骨颈骨折，否认药物过敏。” | “左/右”“否认/存在”任何翻转直接失败。 | +| 医疗中英混说 | “计划做 PCI，继续服用 atorvastatin。” | 缩写大小写和药名拼写可增强，但不得新增剂量。 | +| IT/安全 | “Kubernetes 的 Ingress Controller 使用 mTLS。” | 普通句“控制器进来了”不能因 context 输出 `Ingress Controller`。 | +| 金融 | “净资产收益率是百分之十二点五，期限三年。” | `12.5%`、三年与币种/方向必须 exact。 | +| 能源 | “光伏逆变器的额定功率是五十千瓦。” | `50 kW` 可规范化，但不能变成 15 kW 或 MW。 | +| 航空 | “保持三千英尺，联系进近管制。” | 高度、航向、频率属于零容忍槽位；该包未通过前不默认开启。 | + +每个目标术语还应覆盖：单独出现、句首/句中/句尾、多词短语、缩写逐字读/整体读、中英切换、同一句两个行业词、不同语速、停顿拆分、噪声、近音普通词。训练/调权样例与最终 holdout 说话人和句子必须分离。 + +## 8. 测试层级与交付定义 + +### 8.1 无音频的确定性测试 + +- source manifest：所有 bundled source 必须有版本、URL、校验和、许可与 redistribution 状态；unknown/restricted 条目构建失败。 +- schema：ID 唯一、locale 合法、canonical 非空、alias/spoken form 去重、Unicode normalization 后无碰撞。 +- selector：Apple Top-K 不超过 100；常见词/否定词/数字不会被选；个人/组织词典优先于行业包；跨 locale 不泄漏。 +- adapter：同一条目对 Apple、Whisper、Qwen、火山/AWS 产生符合各自限制的输出；不支持 IPA/SoundsLike 的引擎不会伪称使用。 +- post-process：只有完整词边界或中文明确子串/候选证据才规范化，不做级联替换，不改变保护项。 +- license notices：LOINC、Microsoft、NLM、NIST/EIA/FIBO 等 attribution 随实际入包来源生成，禁止 logo/商标误用和“官方认可 Utter”暗示。 + +### 8.2 真实音频评测 + +真实音频评测必须保存：App commit、行业包 version/checksum、源版本、ASR 模型 ID、decoder 参数、prompt/context、macOS/硬件、输入设备、音频 checksum、control/treatment 原始输出、processed 输出、逐条 diff 和聚合报告。这样才能复现某个“通过”结论。 + +### 8.3 可接受的交付声明 + +- 仅完成 schema、数据包和单元测试：**“行业词库功能已接通，确定性测试通过；真实识别提升尚未验证。”** +- 跑过少量开发者口述：**“smoke test 通过；不代表行业评测通过。”** +- 完成第 7 节冻结 corpus、配对报告和全部门禁：**“该词库版本在指定模型、设备和 corpus 上通过；不外推到其他模型/语言/医院或安全关键用途。”** + +## 9. 上线前仍需确认的事项 + +1. 取得中文医疗术语的商业再分发书面许可，或由医学专家基于可授权来源独立编写、复核第一版中文 canonical/spoken forms。 +2. 对每条 LOINC 记录检查第三方 copyright 字段，并把要求的 notice 放到产品许可与下载页。 +3. 审核 Microsoft Globalization License 的下游条款和 indemnity 是否与现有 App EULA/开源分发方式兼容。 +4. 确认当前 macOS 26 Apple Speech 的 custom language model 对目标 zh-CN locale 的实际支持、编译体积和首次加载延迟；官方 API 存在不等于每个 locale 都适合发布。 +5. 固定 Qwen/Whisper 本地运行器版本并验证 context 真正进入了调用链；命令行/上游支持不等于当前 app adapter 已接入。 +6. 由医疗、金融、能源等行业专家审批各自高风险保护词与允许的 display normalization。 +7. 建立来源更新策略：自动发现新版本可以，但任何词条增删、翻译或权重变化都先生成 diff、重跑许可检查和完整 corpus，再发布。 + +## 10. 本次实现与验证结果 + +本次实现内置了医疗、法律、金融财会和软件技术四个中文种子包，每包 30 个项目自编术语。外部权威页面只作为规范核对引用，不复制其代码集、定义或原始语料；每个引用在运行时清单中明确标为 `reference-only` / `not-redistributed`。四个包仍标记为 `project-seed-needs-domain-review`，不能据此宣称已经得到医疗、法律或金融专业审核。 + +运行时链路已接到： + +- 设置页行业选择； +- Apple / Whisper 已有的短语上下文入口，个人词条排在行业词之前，总数仍限制为 100； +- 直出和智能整理前的非级联、精确错写规范化，个人规则优先； +- LLM 的“只纠正明确出现术语、不得从词表补事实”约束； +- 个人词典、行业词库和来源元数据的确定性测试。 + +2026-08-24 的确定性 corpus 结果（词库 `2026.08.24-v1`，SHA-256 `d9da4adc01a814d9f0eb54d5d9f6c6788365d7affca55f83536b6838f8b8452c`）： + +| 指标 | 关闭行业纠错 | 开启行业纠错 | 本层门槛 | 结果 | +|---|---:|---:|---:|---| +| 目标术语 exact recall | 20.59% | 100.00% | 开启后 ≥ 90%，绝对提升 ≥ 10 个百分点 | 通过 | +| 绝对提升 | — | +79.41 个百分点 | ≥ 10 个百分点 | 通过 | +| 非目标文本原样保留 | — | 100.00% | 100% | 通过 | +| 单包 ASR context | — | ≤ 100 条 | ≤ 100 条 | 通过 | + +这些结果来自人为构造的错写文本，用于验证 schema、选择、优先级和后处理管线，**不等于第 7 节的真实音频 ASR 质量通过**。真实行业效果仍需要冻结 corpus 上的 control/treatment 配对音频评测；当前环境仅配置 Command Line Tools，且缺少 `metal` 编译器，无法运行依赖 MLX 的完整 SwiftPM XCTest。可在安装带 Metal Toolchain 的完整 Xcode 后执行： + +```bash +./scripts/test-industry-lexicons.sh +swift test --filter IndustryLexiconTests +``` diff --git a/scripts/ci-basic-checks.sh b/scripts/ci-basic-checks.sh index 6bd4208..5bc8a74 100755 --- a/scripts/ci-basic-checks.sh +++ b/scripts/ci-basic-checks.sh @@ -52,6 +52,9 @@ if ! diff -u "$en_keys" "$zh_keys"; then fail "localized string keys differ between en and zh-Hans" fi +step "Checking industry vocabulary" +./scripts/test-industry-lexicons.sh + step "Checking required app resources" test -f Sources/Resources/Sounds/start.caf || fail "missing start sound" test -f Sources/Resources/Sounds/stop.caf || fail "missing stop sound" diff --git a/scripts/test-industry-lexicons.sh b/scripts/test-industry-lexicons.sh new file mode 100755 index 0000000..9ae3352 --- /dev/null +++ b/scripts/test-industry-lexicons.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TEST_TEMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/utter-industry-lexicon.XXXXXX")" +trap 'rm -rf "$TEST_TEMP_DIR"' EXIT + +swiftc -parse-as-library \ + "$ROOT_DIR/Sources/Processing/IndustryLexicon.swift" \ + "$ROOT_DIR/Sources/Processing/VocabularyReplacementEngine.swift" \ + "$ROOT_DIR/scripts/tests/industry_lexicon_deterministic.swift" \ + -o "$TEST_TEMP_DIR/industry-lexicon-tests" + +"$TEST_TEMP_DIR/industry-lexicon-tests" \ + "$ROOT_DIR/Sources/Resources/IndustryLexicons.json" diff --git a/scripts/tests/industry_lexicon_deterministic.swift b/scripts/tests/industry_lexicon_deterministic.swift new file mode 100644 index 0000000..69a12b0 --- /dev/null +++ b/scripts/tests/industry_lexicon_deterministic.swift @@ -0,0 +1,149 @@ +import Foundation + +func L(_ key: String) -> String { key } + +enum AppResources { + static let bundle = Bundle.main +} + +@main +enum IndustryLexiconDeterministicTest { + static func main() throws { + guard CommandLine.arguments.count == 2 else { + throw TestFailure("expected the IndustryLexicons.json path") + } + let data = try Data(contentsOf: URL(fileURLWithPath: CommandLine.arguments[1])) + let catalog = try IndustryLexiconCatalog.decode(data) + try require(catalog.packs.count == 4, "expected four industry packs") + try require(catalog.packs.allSatisfy { $0.terms.count >= 30 }, "each pack needs 30 terms") + try require( + catalog.packs.allSatisfy { + IndustryLexiconSnapshot(pack: $0).recognitionPhrases.count <= 100 + }, + "an industry pack exceeded the 100-phrase ASR context budget" + ) + + let cases = evaluationCases + let baseline = termRecall(cases, catalog: catalog, enhanced: false) + let treatment = termRecall(cases, catalog: catalog, enhanced: true) + let preservation = nonTargetPreservation(catalog: catalog) + + try require(treatment >= 0.90, "treatment term recall below 90%") + try require(treatment - baseline >= 0.10, "term recall improvement below 10 points") + try require(preservation == 1, "non-target text changed") + try verifyPersonalPriority(catalog: catalog) + + print("Industry lexicon deterministic evaluation passed") + print(String(format: " baseline term recall: %.2f%%", baseline * 100)) + print(String(format: " treatment term recall: %.2f%%", treatment * 100)) + print(String(format: " absolute improvement: %.2f points", (treatment - baseline) * 100)) + print(String(format: " non-target preservation: %.2f%%", preservation * 100)) + } + + private static let evaluationCases: [EvaluationCase] = [ + .init(.medical, "主愫是心季，糖化血红旦白偏高。", ["主诉", "心悸", "糖化血红蛋白"]), + .init(.medical, "建议复查估算肾小球虑过率和C反映蛋白。", ["估算肾小球滤过率", "C反应蛋白"]), + .init(.medical, "给予静脉低注，记录血氧包和度。", ["静脉滴注", "血氧饱和度"]), + .init(.medical, "患者高血压，心电图已完成。", ["高血压", "心电图"]), + .init(.legal, "本案超过诉讼实效，举证责认仍有争议。", ["诉讼时效", "举证责任"]), + .init(.legal, "不可抗利与缔约过时责任需要分别审查。", ["不可抗力", "缔约过失责任"]), + .init(.legal, "当事人提出管辖权意议并申请行政富议。", ["管辖权异议", "行政复议"]), + .init(.legal, "法院已经采取财产保全措施。", ["财产保全"]), + .init(.finance, "资产付债表和现金刘量表需要重编。", ["资产负债表", "现金流量表"]), + .init(.finance, "按滩余成本计算递延所的税。", ["摊余成本", "递延所得税"]), + .init(.finance, "流动性复盖率下降，逆回够规模上升。", ["流动性覆盖率", "逆回购"]), + .init(.finance, "净现值为正，资本充足率保持稳定。", ["净现值", "资本充足率"]), + .init(.technology, "云原声平台采用服务网各和持续急成。", ["云原生", "服务网格", "持续集成"]), + .init(.technology, "可观测行依赖分布式追棕和幂等幸。", ["可观测性", "分布式追踪", "幂等性"]), + .init(.technology, "检索增强生城使用向量数聚库。", ["检索增强生成", "向量数据库"]), + .init(.technology, "软件物料清单用于供应链安全。", ["软件物料清单", "供应链安全"]), + ] + + private static func termRecall( + _ cases: [EvaluationCase], + catalog: IndustryLexiconCatalog, + enhanced: Bool + ) -> Double { + var matched = 0 + var expected = 0 + for item in cases { + let output = enhanced + ? apply(catalog.snapshot(for: item.industry), to: item.transcript) + : item.transcript + matched += item.expectedTerms.filter(output.contains).count + expected += item.expectedTerms.count + } + return Double(matched) / Double(expected) + } + + private static func nonTargetPreservation(catalog: IndustryLexiconCatalog) -> Double { + let samples: [(IndustryLexiconID, String)] = [ + (.medical, "明天下午三点讨论项目排期。"), + (.legal, "请把会议纪要发给团队。"), + (.finance, "本周完成用户访谈和原型。"), + (.technology, "周末去公园散步。"), + ] + let preserved = samples.filter { industry, text in + apply(catalog.snapshot(for: industry), to: text) == text + }.count + return Double(preserved) / Double(samples.count) + } + + private static func apply(_ snapshot: IndustryLexiconSnapshot, to text: String) -> String { + let rules = snapshot.corrections.enumerated().map { offset, correction in + VocabularyReplacementRule( + original: correction.recognized, + replacement: correction.preferred, + sourcePriority: 1, + insertionOrder: offset + ) + } + return VocabularyReplacementEngine.apply(rules, to: text) + } + + private static func verifyPersonalPriority(catalog: IndustryLexiconCatalog) throws { + let industry = catalog.snapshot(for: .medical).corrections.enumerated().map { + VocabularyReplacementRule( + original: $0.element.recognized, + replacement: $0.element.preferred, + sourcePriority: 1, + insertionOrder: $0.offset + ) + } + let personal = VocabularyReplacementRule( + original: "禁忌症", + replacement: "禁用情况", + sourcePriority: 0, + insertionOrder: 0 + ) + try require( + VocabularyReplacementEngine.apply([personal] + industry, to: "记录禁忌症") + == "记录禁用情况", + "personal correction did not outrank the industry pack" + ) + } + + private static func require(_ condition: @autoclosure () -> Bool, _ message: String) throws { + if !condition() { throw TestFailure(message) } + } +} + +private struct EvaluationCase { + let industry: IndustryLexiconID + let transcript: String + let expectedTerms: [String] + + init(_ industry: IndustryLexiconID, _ transcript: String, _ expectedTerms: [String]) { + self.industry = industry + self.transcript = transcript + self.expectedTerms = expectedTerms + } +} + +private struct TestFailure: Error, CustomStringConvertible { + let description: String + + init(_ description: String) { + self.description = description + } +}