diff --git a/README.md b/README.md index 534c8b77..f8da86be 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,7 @@ Mob.Notify.register_push(socket) def handle_info({:push_token, :ios, token}, socket), do: ... ``` -Also: `Mob.Clipboard`, `Mob.Share`, `Mob.Photos`, `Mob.Files`, `Mob.Audio`, `Mob.Motion`, `Mob.Biometric`, `Mob.Scanner`, `Mob.Permissions`. +Also: `Mob.Clipboard`, `Mob.Share`, `Mob.Photos`, `Mob.Files`, `Mob.Audio`, `Mob.IOS.FoundationModels`, `Mob.IOS.Vision`, `Mob.IOS.Speech`, `Mob.Motion`, `Mob.Biometric`, `Mob.Scanner`, `Mob.Permissions`. ## What's in the box @@ -156,6 +156,7 @@ The pre-built OTP runtime that ships with each app includes: Native APIs surfaced via `Mob.*` modules (above) cover camera, location, audio, files, biometrics, push, clipboard, share, scanner, +Foundation Models, Vision, Speech, motion sensors, permissions. The OTP runtime tarball is ~80 MB compressed; sliced per-arch by diff --git a/android/jni/mob_nif.zig b/android/jni/mob_nif.zig index ac12cab6..ec8486a3 100644 --- a/android/jni/mob_nif.zig +++ b/android/jni/mob_nif.zig @@ -2433,6 +2433,36 @@ export fn nif_audio_set_volume( return erts.ok(env); } +export fn nif_foundation_models_generate_text( + env: ?*erts.ErlNifEnv, + argc: c_int, + argv: [*]const erts.ERL_NIF_TERM, +) callconv(.c) erts.ERL_NIF_TERM { + _ = argc; + _ = argv; + return erts.atom(env, "unsupported"); +} + +export fn nif_vision_recognize_text( + env: ?*erts.ErlNifEnv, + argc: c_int, + argv: [*]const erts.ERL_NIF_TERM, +) callconv(.c) erts.ERL_NIF_TERM { + _ = argc; + _ = argv; + return erts.atom(env, "unsupported"); +} + +export fn nif_speech_transcribe_audio( + env: ?*erts.ErlNifEnv, + argc: c_int, + argv: [*]const erts.ERL_NIF_TERM, +) callconv(.c) erts.ERL_NIF_TERM { + _ = argc; + _ = argv; + return erts.atom(env, "unsupported"); +} + export fn nif_motion_start( env: ?*erts.ErlNifEnv, argc: c_int, @@ -3273,6 +3303,9 @@ const nif_funcs = [_]erts.ErlNifFunc{ .{ .name = "audio_play", .arity = 2, .fptr = nif_audio_play, .flags = 0 }, .{ .name = "audio_stop_playback", .arity = 0, .fptr = nif_audio_stop_playback, .flags = 0 }, .{ .name = "audio_set_volume", .arity = 1, .fptr = nif_audio_set_volume, .flags = 0 }, + .{ .name = "foundation_models_generate_text", .arity = 2, .fptr = nif_foundation_models_generate_text, .flags = 0 }, + .{ .name = "vision_recognize_text", .arity = 2, .fptr = nif_vision_recognize_text, .flags = 0 }, + .{ .name = "speech_transcribe_audio", .arity = 2, .fptr = nif_speech_transcribe_audio, .flags = 0 }, .{ .name = "motion_start", .arity = 2, .fptr = nif_motion_start, .flags = 0 }, .{ .name = "motion_stop", .arity = 0, .fptr = nif_motion_stop, .flags = 0 }, .{ .name = "scanner_scan", .arity = 1, .fptr = nif_scanner_scan, .flags = 0 }, diff --git a/guides/device_capabilities.md b/guides/device_capabilities.md index dee72e73..6bc021de 100644 --- a/guides/device_capabilities.md +++ b/guides/device_capabilities.md @@ -205,6 +205,153 @@ end iOS uses `AVAudioPlayer` / `AVPlayer`. Android uses `MediaPlayer`. +## iOS Foundation Models + +Generates text with Apple's on-device Foundation Models framework. +Requires an eligible physical device with Apple Intelligence enabled; the iOS +simulator reports this capability as unavailable. + +Apple docs: +[Foundation Models](https://developer.apple.com/documentation/foundationmodels) and +[Adding intelligent app features with generative models](https://developer.apple.com/documentation/foundationmodels/adding-intelligent-app-features-with-generative-models). + +```elixir +socket = + Mob.IOS.FoundationModels.generate_text(socket, "Turn this note into a short action list", + instructions: "Return compact plain text.", + temperature: 0.2, + maximum_response_tokens: 240 + ) + +def handle_info({:foundation_models, :generated_text, %{text: text}}, socket) do + {:noreply, Mob.Socket.assign(socket, :result, text)} +end + +def handle_info({:foundation_models, :error, %{reason: reason}}, socket) do + {:noreply, Mob.Socket.assign(socket, :error, reason)} +end +``` + +## iOS Vision text recognition + +Recognizes text in a local image file with Apple's Vision framework. Combine +with `Mob.Photos.pick/2` to OCR a user-selected image. + +Apple docs: +[VNRecognizeTextRequest](https://developer.apple.com/documentation/vision/vnrecognizetextrequest). + +```elixir +socket = + Mob.Photos.pick(socket, max: 1, types: [:image]) + +def handle_info({:photos, :picked, [%{path: path} | _]}, socket) do + socket = + Mob.IOS.Vision.recognize_text(socket, path, + recognition_level: :accurate, + uses_language_correction: true + ) + + {:noreply, socket} +end + +def handle_info({:vision, :recognized_text, %{text: text}}, socket) do + {:noreply, Mob.Socket.assign(socket, :ocr_text, text)} +end +``` + +## iOS Speech transcription + +Transcribes an existing audio file with Apple's Speech framework. Use +`Mob.Audio` to record microphone input first, then transcribe the saved +recording path. + +Apple docs: +[SFSpeechRecognizer](https://developer.apple.com/documentation/speech/sfspeechrecognizer) and +[SFSpeechURLRecognitionRequest](https://developer.apple.com/documentation/speech/sfspeechurlrecognitionrequest). + +```elixir +socket = Mob.Audio.start_recording(socket, format: :aac, quality: :high) +socket = Mob.Audio.stop_recording(socket) + +def handle_info({:audio, :recorded, %{path: path}}, socket) do + socket = + Mob.IOS.Speech.transcribe_audio(socket, path, + locale: "en-US", + requires_on_device_recognition: false + ) + + {:noreply, socket} +end + +def handle_info({:speech, :transcribed_audio, %{text: text}}, socket) do + {:noreply, Mob.Socket.assign(socket, :transcript, text)} +end +``` + +Speech recognition requires iOS speech authorization. If +`requires_on_device_recognition` is true, iOS may reject locales that do not +support local recognition. + +### iOS native intelligence testing + +| Capability | iOS simulator | Physical iPhone | +|---|---:|---:| +| `Mob.IOS.FoundationModels.generate_text/3` | No. The simulator does not provide the on-device system language model. | Yes, on Apple Intelligence-capable devices with Apple Intelligence enabled and the model ready. | +| `Mob.IOS.Vision.recognize_text/3` | Yes. Pass a readable image path in the simulator app container or pick a simulator photo. | Yes. | +| `Mob.IOS.Speech.transcribe_audio/3` | Usually yes for file transcription, subject to simulator Speech authorization and runtime locale/service availability. | Yes, subject to Speech authorization and locale support. | + +The lightest simulator smoke test is: + +1. Build a Mob app that exposes a screen with `Mob.Photos.pick/2` and calls + `Mob.IOS.Vision.recognize_text/3` with the selected photo path. +2. Add a photo with readable text to the simulator photo library, select it in + the picker, and confirm OCR returns the expected text. +3. Build two audio actions: one calls `Mob.Audio.start_recording/2`; the other + calls `Mob.Audio.stop_recording/1`. Pass the recorded file path from + `{:audio, :recorded, %{path: path}}` to `Mob.IOS.Speech.transcribe_audio/3`. +4. Confirm Foundation Models returns the expected simulator-unavailable error. + +For path-based debugging in a simulator app that is already installed, the +useful setup commands are: + +```sh +SIM_ID="booted" +BUNDLE_ID="com.example.my_mob_app" +CONTAINER="$(xcrun simctl get_app_container "$SIM_ID" "$BUNDLE_ID" data)" + +cp ./ocr_fixture.png "$CONTAINER/Documents/ocr_fixture.png" +``` + +Then pass the copied path to the app: + +```elixir +image_path = "/path/from/xcrun/simctl/get_app_container/Documents/ocr_fixture.png" +Mob.IOS.Vision.recognize_text(socket, image_path) +``` + +For Speech, recording inside the Mob app is the most representative smoke test: + +```elixir +socket = Mob.Audio.start_recording(socket) +socket = Mob.Audio.stop_recording(socket) + +def handle_info({:audio, :recorded, %{path: path}}, socket) do + {:noreply, Mob.IOS.Speech.transcribe_audio(socket, path)} +end +``` + +### Scope and follow-up ideas + +This first bridge includes plain Foundation Models text generation, Vision OCR +from a local image path, and Speech transcription from a local audio file. + +Not included yet: Foundation Models structured generation with `@Generable`, +streaming partial Foundation Models responses, tool calling, multi-turn session +persistence, Vision requests beyond text recognition, live Speech recognition, +custom Speech language models, Natural Language framework features, image +generation, Private Cloud Compute-backed server features, and Android ML Kit or +platform-equivalent implementations. + ## Location Requires `:location` permission. diff --git a/ios/MobFoundationModels.swift b/ios/MobFoundationModels.swift new file mode 100644 index 00000000..d9e6f53a --- /dev/null +++ b/ios/MobFoundationModels.swift @@ -0,0 +1,91 @@ +import Foundation + +#if canImport(FoundationModels) +import FoundationModels +#endif + +@objcMembers +public final class MobFoundationModels: NSObject { + public static func generateText( + _ prompt: String, + optionsJSON: String, + completion: @escaping (String?, String?) -> Void + ) { + #if targetEnvironment(simulator) + completion(nil, "Foundation Models does not run in the iOS simulator.") + #else + guard #available(iOS 26.0, *) else { + completion(nil, "Foundation Models requires iOS 26.0 or newer.") + return + } + + #if canImport(FoundationModels) + let opts = decodeOptions(optionsJSON) + let instructions = opts["instructions"] as? String ?? "" + let temperature = opts["temperature"] as? Double ?? 0.2 + let maximumResponseTokens = opts["maximum_response_tokens"] as? Int ?? 256 + + Task { + let model = SystemLanguageModel.default + + guard model.supportsLocale(Locale.current) else { + completion(nil, "Foundation Models does not support the current locale: \(Locale.current.identifier).") + return + } + + switch model.availability { + case .available: + do { + let session = instructions.isEmpty + ? LanguageModelSession() + : LanguageModelSession(instructions: instructions) + let response = try await session.respond( + to: prompt, + options: GenerationOptions( + temperature: temperature, + maximumResponseTokens: maximumResponseTokens + ) + ) + completion(response.content, nil) + } catch { + completion(nil, "Foundation Models generation error: \(error.localizedDescription)") + } + + case .unavailable(let reason): + completion(nil, availabilityMessage(reason)) + } + } + #else + completion(nil, "This build of Xcode does not expose the FoundationModels module.") + #endif + #endif + } + + private static func decodeOptions(_ optionsJSON: String) -> [String: Any] { + guard let data = optionsJSON.data(using: .utf8), + let object = try? JSONSerialization.jsonObject(with: data), + let dict = object as? [String: Any] else { + return [:] + } + + return dict + } + + #if canImport(FoundationModels) + @available(iOS 26.0, *) + private static func availabilityMessage( + _ reason: SystemLanguageModel.Availability.UnavailableReason + ) -> String { + switch reason { + case .deviceNotEligible: + return "Foundation Models unavailable: this device is not eligible for Apple Intelligence." + case .appleIntelligenceNotEnabled: + return "Foundation Models unavailable: Apple Intelligence is not enabled in Settings." + case .modelNotReady: + return "Foundation Models unavailable: the on-device model is not ready yet." + @unknown default: + return "Foundation Models unavailable: \(String(describing: reason))." + } + } + #endif +} diff --git a/ios/MobSpeech.swift b/ios/MobSpeech.swift new file mode 100644 index 00000000..f1211147 --- /dev/null +++ b/ios/MobSpeech.swift @@ -0,0 +1,120 @@ +import Foundation +import Speech + +@objcMembers +public final class MobSpeech: NSObject { + private static var tasks: [SFSpeechRecognitionTask] = [] + + public static func transcribeAudio( + atPath path: String, + optionsJSON: String, + completion: @escaping (String?, String?) -> Void + ) { + guard FileManager.default.fileExists(atPath: path) else { + completion(nil, "Speech audio file does not exist: \(path)") + return + } + + let opts = decodeOptions(optionsJSON) + let localeIdentifier = opts["locale"] as? String ?? "" + let requiresOnDeviceRecognition = opts["requires_on_device_recognition"] as? Bool ?? false + let url = URL(fileURLWithPath: path) + + SFSpeechRecognizer.requestAuthorization { status in + guard status == .authorized else { + completion(nil, "Speech recognition authorization: \(authorizationName(status)).") + return + } + + guard let recognizer = makeRecognizer(localeIdentifier: localeIdentifier) else { + completion(nil, localeDiagnostic(localeIdentifier: localeIdentifier)) + return + } + + guard recognizer.isAvailable else { + completion(nil, "Speech recognizer for \(recognizer.locale.identifier) is not currently available.") + return + } + + let request = SFSpeechURLRecognitionRequest(url: url) + request.shouldReportPartialResults = false + request.requiresOnDeviceRecognition = requiresOnDeviceRecognition + + let task = recognizer.recognitionTask(with: request) { result, error in + if let result, result.isFinal { + completion(result.bestTranscription.formattedString, nil) + tasks.removeAll { $0.isFinishing || $0.isCancelled } + return + } + + if let error { + completion(nil, "Speech transcription error for \(recognizer.locale.identifier): \(error.localizedDescription)") + tasks.removeAll { $0.isFinishing || $0.isCancelled } + } + } + + tasks.append(task) + } + } + + private static func decodeOptions(_ optionsJSON: String) -> [String: Any] { + guard let data = optionsJSON.data(using: .utf8), + let object = try? JSONSerialization.jsonObject(with: data), + let dict = object as? [String: Any] else { + return [:] + } + + return dict + } + + private static func makeRecognizer(localeIdentifier: String) -> SFSpeechRecognizer? { + if !localeIdentifier.isEmpty { + return SFSpeechRecognizer(locale: Locale(identifier: localeIdentifier)) + } + + if let recognizer = SFSpeechRecognizer() { + return recognizer + } + + let preferredIdentifiers = [ + Locale.current.identifier, + Locale.preferredLanguages.first ?? "", + "en-US", + "en_US" + ] + + for identifier in preferredIdentifiers where !identifier.isEmpty { + if let recognizer = SFSpeechRecognizer(locale: Locale(identifier: identifier)) { + return recognizer + } + } + + return nil + } + + private static func localeDiagnostic(localeIdentifier: String) -> String { + let requested = localeIdentifier.isEmpty ? "default locale" : localeIdentifier + let supported = SFSpeechRecognizer.supportedLocales() + .map(\.identifier) + .sorted() + .prefix(12) + .joined(separator: ", ") + + return "Failed to initialize speech recognizer for \(requested). Supported locales include: \(supported)." + } + + private static func authorizationName(_ status: SFSpeechRecognizerAuthorizationStatus) -> String { + switch status { + case .authorized: + return "authorized" + case .denied: + return "denied" + case .restricted: + return "restricted" + case .notDetermined: + return "not determined" + @unknown default: + return "unknown" + } + } +} diff --git a/ios/MobVision.swift b/ios/MobVision.swift new file mode 100644 index 00000000..f000e54f --- /dev/null +++ b/ios/MobVision.swift @@ -0,0 +1,52 @@ +import Foundation +import Vision + +@objcMembers +public final class MobVision: NSObject { + public static func recognizeText( + atPath path: String, + optionsJSON: String, + completion: @escaping (String?, String?) -> Void + ) { + DispatchQueue.global(qos: .userInitiated).async { + guard FileManager.default.fileExists(atPath: path) else { + completion(nil, "Image file does not exist: \(path)") + return + } + + let opts = decodeOptions(optionsJSON) + let url = URL(fileURLWithPath: path) + let request = VNRecognizeTextRequest { request, error in + if let error { + completion(nil, "Vision OCR error: \(error.localizedDescription)") + return + } + + let observations = request.results as? [VNRecognizedTextObservation] ?? [] + let lines = observations.compactMap { $0.topCandidates(1).first?.string } + completion(lines.joined(separator: "\n"), nil) + } + + let level = opts["recognition_level"] as? String ?? "accurate" + request.recognitionLevel = level == "fast" ? .fast : .accurate + request.usesLanguageCorrection = opts["uses_language_correction"] as? Bool ?? true + request.automaticallyDetectsLanguage = true + + do { + try VNImageRequestHandler(url: url, options: [:]).perform([request]) + } catch { + completion(nil, "Vision OCR error: \(error.localizedDescription)") + } + } + } + + private static func decodeOptions(_ optionsJSON: String) -> [String: Any] { + guard let data = optionsJSON.data(using: .utf8), + let object = try? JSONSerialization.jsonObject(with: data), + let dict = object as? [String: Any] else { + return [:] + } + + return dict + } +} diff --git a/ios/mob_nif.m b/ios/mob_nif.m index c9a46c3d..ab2d6bdc 100644 --- a/ios/mob_nif.m +++ b/ios/mob_nif.m @@ -1971,6 +1971,122 @@ static ERL_NIF_TERM nif_share_text(ErlNifEnv *env, int argc, const ERL_NIF_TERM return enif_make_atom(env, "ok"); } +// ── Native text capability helpers ───────────────────────────────────────── + +static NSString *mob_native_text_string_arg(ErlNifEnv *env, ERL_NIF_TERM term) { + ErlNifBinary bin; + if (!enif_inspect_binary(env, term, &bin) && + !enif_inspect_iolist_as_binary(env, term, &bin)) + return nil; + + return [[NSString alloc] initWithBytes:bin.data length:bin.size encoding:NSUTF8StringEncoding]; +} + +static ERL_NIF_TERM mob_native_text_make_utf8_binary(ErlNifEnv *env, NSString *text) { + const char *utf8 = text ? text.UTF8String : ""; + size_t len = utf8 ? strlen(utf8) : 0; + ErlNifBinary bin; + enif_alloc_binary(len, &bin); + if (len > 0) + memcpy(bin.data, utf8, len); + return enif_make_binary(env, &bin); +} + +static void mob_native_text_send_success(ErlNifPid pid, const char *family, const char *event, + NSString *text) { + ErlNifEnv *e = enif_alloc_env(); + ERL_NIF_TERM keys[1] = {enif_make_atom(e, "text")}; + ERL_NIF_TERM vals[1] = {mob_native_text_make_utf8_binary(e, text)}; + ERL_NIF_TERM payload; + enif_make_map_from_arrays(e, keys, vals, 1, &payload); + ERL_NIF_TERM msg = enif_make_tuple3(e, enif_make_atom(e, family), enif_make_atom(e, event), payload); + enif_send(NULL, &pid, e, msg); + enif_free_env(e); +} + +static void mob_native_text_send_error(ErlNifPid pid, const char *family, const char *operation, + NSString *reason) { + ErlNifEnv *e = enif_alloc_env(); + ERL_NIF_TERM keys[2] = {enif_make_atom(e, "operation"), enif_make_atom(e, "reason")}; + ERL_NIF_TERM vals[2] = {enif_make_atom(e, operation), mob_native_text_make_utf8_binary(e, reason)}; + ERL_NIF_TERM payload; + enif_make_map_from_arrays(e, keys, vals, 2, &payload); + ERL_NIF_TERM msg = enif_make_tuple3(e, enif_make_atom(e, family), enif_make_atom(e, "error"), payload); + enif_send(NULL, &pid, e, msg); + enif_free_env(e); +} + +static ERL_NIF_TERM nif_foundation_models_generate_text(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + // Apple Foundation Models: + // https://developer.apple.com/documentation/foundationmodels + NSString *prompt = mob_native_text_string_arg(env, argv[0]); + NSString *optionsJSON = mob_native_text_string_arg(env, argv[1]); + if (!prompt || !optionsJSON) + return enif_make_badarg(env); + + ErlNifPid pid; + enif_self(env, &pid); + + [MobFoundationModels generateText:prompt + optionsJSON:optionsJSON + completion:^(NSString *text, NSString *error) { + if (error) + mob_native_text_send_error(pid, "foundation_models", "generate_text", + error); + else + mob_native_text_send_success(pid, "foundation_models", + "generated_text", text); + }]; + + return enif_make_atom(env, "ok"); +} + +static ERL_NIF_TERM nif_vision_recognize_text(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + // Apple Vision VNRecognizeTextRequest: + // https://developer.apple.com/documentation/vision/vnrecognizetextrequest + NSString *path = mob_native_text_string_arg(env, argv[0]); + NSString *optionsJSON = mob_native_text_string_arg(env, argv[1]); + if (!path || !optionsJSON) + return enif_make_badarg(env); + + ErlNifPid pid; + enif_self(env, &pid); + + [MobVision recognizeTextAtPath:path + optionsJSON:optionsJSON + completion:^(NSString *text, NSString *error) { + if (error) + mob_native_text_send_error(pid, "vision", "recognize_text", error); + else + mob_native_text_send_success(pid, "vision", "recognized_text", text); + }]; + + return enif_make_atom(env, "ok"); +} + +static ERL_NIF_TERM nif_speech_transcribe_audio(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + // Apple Speech SFSpeechURLRecognitionRequest: + // https://developer.apple.com/documentation/speech/sfspeechurlrecognitionrequest + NSString *path = mob_native_text_string_arg(env, argv[0]); + NSString *optionsJSON = mob_native_text_string_arg(env, argv[1]); + if (!path || !optionsJSON) + return enif_make_badarg(env); + + ErlNifPid pid; + enif_self(env, &pid); + + [MobSpeech transcribeAudioAtPath:path + optionsJSON:optionsJSON + completion:^(NSString *text, NSString *error) { + if (error) + mob_native_text_send_error(pid, "speech", "transcribe_audio", error); + else + mob_native_text_send_success(pid, "speech", "transcribed_audio", text); + }]; + + return enif_make_atom(env, "ok"); +} + // ════════════════════════════════════════════════════════════════════════════ // Device capability NIFs // ════════════════════════════════════════════════════════════════════════════ @@ -5789,6 +5905,9 @@ static ERL_NIF_TERM nif_vendor_usb_close(ErlNifEnv *env, int argc, const ERL_NIF {"audio_play", 2, nif_audio_play, 0}, {"audio_stop_playback", 0, nif_audio_stop_playback, 0}, {"audio_set_volume", 1, nif_audio_set_volume, 0}, + {"foundation_models_generate_text", 2, nif_foundation_models_generate_text, 0}, + {"vision_recognize_text", 2, nif_vision_recognize_text, 0}, + {"speech_transcribe_audio", 2, nif_speech_transcribe_audio, 0}, {"motion_start", 2, nif_motion_start, 0}, {"motion_stop", 0, nif_motion_stop, 0}, {"scanner_scan", 1, nif_scanner_scan, 0}, diff --git a/lib/mob/ios/foundation_models.ex b/lib/mob/ios/foundation_models.ex new file mode 100644 index 00000000..bcdd16c0 --- /dev/null +++ b/lib/mob/ios/foundation_models.ex @@ -0,0 +1,66 @@ +defmodule Mob.IOS.FoundationModels do + @moduledoc """ + iOS Foundation Models text generation. + + This wraps Apple's Foundation Models framework, specifically + `SystemLanguageModel`, `LanguageModelSession`, and `GenerationOptions`. + See Apple's guide: + https://developer.apple.com/documentation/foundationmodels/adding-intelligent-app-features-with-generative-models + + Calls are asynchronous. Results are delivered to the calling process: + + Mob.IOS.FoundationModels.generate_text(socket, "Summarize this note") + + def handle_info({:foundation_models, :generated_text, %{text: text}}, socket) do + {:noreply, Mob.Socket.assign(socket, :summary, text)} + end + + def handle_info({:foundation_models, :error, %{reason: reason}}, socket) do + {:noreply, Mob.Socket.assign(socket, :error, reason)} + end + + Foundation Models requires an eligible physical iOS device with Apple + Intelligence enabled. The iOS simulator does not provide the on-device system + language model. + """ + + @type generate_option :: + {:instructions, String.t()} + | {:temperature, number()} + | {:maximum_response_tokens, pos_integer()} + + @doc """ + Generate text using Apple's on-device Foundation Models framework. + """ + @spec generate_text(Mob.Socket.t(), String.t(), [generate_option()]) :: Mob.Socket.t() + def generate_text(socket, prompt, opts \\ []) when is_binary(prompt) and is_list(opts) do + invoke( + :mob_nif.foundation_models_generate_text(prompt, :json.encode(generate_text_opts(opts))), + :generate_text + ) + + socket + end + + @doc false + @spec generate_text_opts(keyword()) :: %{String.t() => term()} + def generate_text_opts(opts) do + %{ + "instructions" => Keyword.get(opts, :instructions, ""), + "temperature" => Keyword.get(opts, :temperature, 0.2) / 1, + "maximum_response_tokens" => Keyword.get(opts, :maximum_response_tokens, 256) + } + end + + defp invoke(:ok, _operation), do: :ok + + defp invoke(:unsupported, operation) do + send( + self(), + {:foundation_models, :error, + %{operation: operation, reason: "Foundation Models is not supported on this platform."}} + ) + + :ok + end +end diff --git a/lib/mob/ios/speech.ex b/lib/mob/ios/speech.ex new file mode 100644 index 00000000..eb9a3e02 --- /dev/null +++ b/lib/mob/ios/speech.ex @@ -0,0 +1,70 @@ +defmodule Mob.IOS.Speech do + @moduledoc """ + iOS Speech framework transcription. + + File transcription wraps Apple's `SFSpeechRecognizer` and + `SFSpeechURLRecognitionRequest`. + See Apple's API documentation: + https://developer.apple.com/documentation/speech/sfspeechrecognizer + https://developer.apple.com/documentation/speech/sfspeechurlrecognitionrequest + + Calls are asynchronous. Results are delivered to the calling process. Pair + this with `Mob.Audio.start_recording/2` and `Mob.Audio.stop_recording/1` when + the audio should come from the microphone: + + socket = Mob.Audio.start_recording(socket, format: :aac, quality: :high) + socket = Mob.Audio.stop_recording(socket) + + def handle_info({:audio, :recorded, %{path: path}}, socket) do + {:noreply, Mob.IOS.Speech.transcribe_audio(socket, path)} + end + + def handle_info({:speech, :transcribed_audio, %{text: text}}, socket) do + {:noreply, Mob.Socket.assign(socket, :transcript, text)} + end + + On iOS, speech transcription requires speech recognition authorization. Audio + recording still uses `Mob.Audio`; this module only transcribes an existing + audio file. + """ + + @type transcribe_audio_option :: + {:locale, String.t()} + | {:requires_on_device_recognition, boolean()} + + @doc """ + Transcribe an audio file using Apple's Speech framework. + """ + @spec transcribe_audio(Mob.Socket.t(), String.t(), [transcribe_audio_option()]) :: + Mob.Socket.t() + def transcribe_audio(socket, path, opts \\ []) when is_binary(path) and is_list(opts) do + invoke( + :mob_nif.speech_transcribe_audio(path, :json.encode(transcribe_audio_opts(opts))), + :transcribe_audio + ) + + socket + end + + @doc false + @spec transcribe_audio_opts(keyword()) :: %{String.t() => term()} + def transcribe_audio_opts(opts) do + %{ + "locale" => Keyword.get(opts, :locale, ""), + "requires_on_device_recognition" => + Keyword.get(opts, :requires_on_device_recognition, false) + } + end + + defp invoke(:ok, _operation), do: :ok + + defp invoke(:unsupported, operation) do + send( + self(), + {:speech, :error, + %{operation: operation, reason: "Speech transcription is not supported on this platform."}} + ) + + :ok + end +end diff --git a/lib/mob/ios/vision.ex b/lib/mob/ios/vision.ex new file mode 100644 index 00000000..15fbe32f --- /dev/null +++ b/lib/mob/ios/vision.ex @@ -0,0 +1,64 @@ +defmodule Mob.IOS.Vision do + @moduledoc """ + iOS Vision framework capabilities. + + Text recognition wraps Apple's `VNRecognizeTextRequest`. + See Apple's API documentation: + https://developer.apple.com/documentation/vision/vnrecognizetextrequest + + Calls are asynchronous. Results are delivered to the calling process. Pair + this with `Mob.Photos.pick/2` when the image should come from the user's + photo library: + + socket = Mob.Photos.pick(socket, max: 1, types: [:image]) + + def handle_info({:photos, :picked, [%{path: path} | _]}, socket) do + {:noreply, Mob.IOS.Vision.recognize_text(socket, path)} + end + + def handle_info({:vision, :recognized_text, %{text: text}}, socket) do + {:noreply, Mob.Socket.assign(socket, :ocr_text, text)} + end + """ + + @type recognize_text_option :: + {:recognition_level, :fast | :accurate} + | {:uses_language_correction, boolean()} + + @doc """ + Recognize text in an image file using Apple's Vision framework. + """ + @spec recognize_text(Mob.Socket.t(), String.t(), [recognize_text_option()]) :: Mob.Socket.t() + def recognize_text(socket, path, opts \\ []) when is_binary(path) and is_list(opts) do + invoke( + :mob_nif.vision_recognize_text(path, :json.encode(recognize_text_opts(opts))), + :recognize_text + ) + + socket + end + + @doc false + @spec recognize_text_opts(keyword()) :: %{String.t() => term()} + def recognize_text_opts(opts) do + %{ + "recognition_level" => Keyword.get(opts, :recognition_level, :accurate) |> Atom.to_string(), + "uses_language_correction" => Keyword.get(opts, :uses_language_correction, true) + } + end + + defp invoke(:ok, _operation), do: :ok + + defp invoke(:unsupported, operation) do + send( + self(), + {:vision, :error, + %{ + operation: operation, + reason: "Vision text recognition is not supported on this platform." + }} + ) + + :ok + end +end diff --git a/mix.exs b/mix.exs index 24939d65..cc3aa31d 100644 --- a/mix.exs +++ b/mix.exs @@ -111,6 +111,9 @@ defmodule Mob.MixProject do Mob.Photos, Mob.Files, Mob.Audio, + Mob.IOS.FoundationModels, + Mob.IOS.Vision, + Mob.IOS.Speech, Mob.Motion, Mob.Scanner, Mob.Notify diff --git a/src/mob_nif.erl b/src/mob_nif.erl index 722932d2..216ea254 100644 --- a/src/mob_nif.erl +++ b/src/mob_nif.erl @@ -41,6 +41,12 @@ audio_play/2, audio_stop_playback/0, audio_set_volume/1, + %% Foundation Models + foundation_models_generate_text/2, + %% Vision + vision_recognize_text/2, + %% Speech + speech_transcribe_audio/2, %% Motion sensors motion_start/2, motion_stop/0, @@ -140,6 +146,9 @@ audio_play/2, audio_stop_playback/0, audio_set_volume/1, + foundation_models_generate_text/2, + vision_recognize_text/2, + speech_transcribe_audio/2, motion_start/2, motion_stop/0, scanner_scan/1, @@ -237,6 +246,9 @@ audio_stop_recording() -> erlang:nif_error(not_loaded). audio_play(_Path, _OptsJson) -> erlang:nif_error(not_loaded). audio_stop_playback() -> erlang:nif_error(not_loaded). audio_set_volume(_Volume) -> erlang:nif_error(not_loaded). +foundation_models_generate_text(_Prompt, _OptsJson) -> erlang:nif_error(not_loaded). +vision_recognize_text(_Path, _OptsJson) -> erlang:nif_error(not_loaded). +speech_transcribe_audio(_Path, _OptsJson) -> erlang:nif_error(not_loaded). motion_start(_Sensors, _Interval) -> erlang:nif_error(not_loaded). motion_stop() -> erlang:nif_error(not_loaded). scanner_scan(_FormatsJson) -> erlang:nif_error(not_loaded). diff --git a/test/mob/ios/foundation_models_test.exs b/test/mob/ios/foundation_models_test.exs new file mode 100644 index 00000000..3f2638c0 --- /dev/null +++ b/test/mob/ios/foundation_models_test.exs @@ -0,0 +1,27 @@ +defmodule Mob.IOS.FoundationModelsTest do + use ExUnit.Case, async: true + + alias Mob.IOS.FoundationModels + + describe "generate_text_opts/1" do + test "defaults are string keyed" do + assert FoundationModels.generate_text_opts([]) == %{ + "instructions" => "", + "temperature" => 0.2, + "maximum_response_tokens" => 256 + } + end + + test "custom values are passed through" do + assert FoundationModels.generate_text_opts( + instructions: "Be concise", + temperature: 0.7, + maximum_response_tokens: 64 + ) == %{ + "instructions" => "Be concise", + "temperature" => 0.7, + "maximum_response_tokens" => 64 + } + end + end +end diff --git a/test/mob/ios/speech_test.exs b/test/mob/ios/speech_test.exs new file mode 100644 index 00000000..c009428f --- /dev/null +++ b/test/mob/ios/speech_test.exs @@ -0,0 +1,22 @@ +defmodule Mob.IOS.SpeechTest do + use ExUnit.Case, async: true + + alias Mob.IOS.Speech + + describe "transcribe_audio_opts/1" do + test "defaults to platform locale and server-capable recognition" do + assert Speech.transcribe_audio_opts([]) == %{ + "locale" => "", + "requires_on_device_recognition" => false + } + end + + test "custom locale and on-device setting are passed through" do + assert Speech.transcribe_audio_opts(locale: "en-US", requires_on_device_recognition: true) == + %{ + "locale" => "en-US", + "requires_on_device_recognition" => true + } + end + end +end diff --git a/test/mob/ios/vision_test.exs b/test/mob/ios/vision_test.exs new file mode 100644 index 00000000..2af04917 --- /dev/null +++ b/test/mob/ios/vision_test.exs @@ -0,0 +1,21 @@ +defmodule Mob.IOS.VisionTest do + use ExUnit.Case, async: true + + alias Mob.IOS.Vision + + describe "recognize_text_opts/1" do + test "defaults to accurate OCR with language correction" do + assert Vision.recognize_text_opts([]) == %{ + "recognition_level" => "accurate", + "uses_language_correction" => true + } + end + + test "recognition level is encoded as a string" do + assert Vision.recognize_text_opts(recognition_level: :fast) == %{ + "recognition_level" => "fast", + "uses_language_correction" => true + } + end + end +end