From 22954621b3c25beda9f8f9e8dfd5b21816d285a0 Mon Sep 17 00:00:00 2001 From: Yurko Hoshko Date: Thu, 14 May 2026 21:51:58 -0700 Subject: [PATCH 1/6] Add iOS native AI bridge --- android/jni/mob_nif.zig | 33 ++++++ ios/MobAI.swift | 235 ++++++++++++++++++++++++++++++++++++++++ ios/mob_nif.m | 109 +++++++++++++++++++ lib/mob/ai.ex | 137 +++++++++++++++++++++++ src/mob_nif.erl | 10 ++ test/mob/ai_test.exs | 59 ++++++++++ 6 files changed, 583 insertions(+) create mode 100644 ios/MobAI.swift create mode 100644 lib/mob/ai.ex create mode 100644 test/mob/ai_test.exs diff --git a/android/jni/mob_nif.zig b/android/jni/mob_nif.zig index ac12cab6..984dc6fa 100644 --- a/android/jni/mob_nif.zig +++ b/android/jni/mob_nif.zig @@ -2433,6 +2433,36 @@ export fn nif_audio_set_volume( return erts.ok(env); } +export fn nif_ai_generate_text( + env: ?*erts.ErlNifEnv, + argc: c_int, + argv: [*]const erts.ERL_NIF_TERM, +) callconv(.c) erts.ERL_NIF_TERM { + _ = argc; + _ = argv; + return erts.atom(env, "unsupported"); +} + +export fn nif_ai_recognize_text( + env: ?*erts.ErlNifEnv, + argc: c_int, + argv: [*]const erts.ERL_NIF_TERM, +) callconv(.c) erts.ERL_NIF_TERM { + _ = argc; + _ = argv; + return erts.atom(env, "unsupported"); +} + +export fn nif_ai_transcribe_audio( + env: ?*erts.ErlNifEnv, + argc: c_int, + argv: [*]const erts.ERL_NIF_TERM, +) callconv(.c) erts.ERL_NIF_TERM { + _ = argc; + _ = argv; + return erts.atom(env, "unsupported"); +} + export fn nif_motion_start( env: ?*erts.ErlNifEnv, argc: c_int, @@ -3273,6 +3303,9 @@ const nif_funcs = [_]erts.ErlNifFunc{ .{ .name = "audio_play", .arity = 2, .fptr = nif_audio_play, .flags = 0 }, .{ .name = "audio_stop_playback", .arity = 0, .fptr = nif_audio_stop_playback, .flags = 0 }, .{ .name = "audio_set_volume", .arity = 1, .fptr = nif_audio_set_volume, .flags = 0 }, + .{ .name = "ai_generate_text", .arity = 2, .fptr = nif_ai_generate_text, .flags = 0 }, + .{ .name = "ai_recognize_text", .arity = 2, .fptr = nif_ai_recognize_text, .flags = 0 }, + .{ .name = "ai_transcribe_audio", .arity = 2, .fptr = nif_ai_transcribe_audio, .flags = 0 }, .{ .name = "motion_start", .arity = 2, .fptr = nif_motion_start, .flags = 0 }, .{ .name = "motion_stop", .arity = 0, .fptr = nif_motion_stop, .flags = 0 }, .{ .name = "scanner_scan", .arity = 1, .fptr = nif_scanner_scan, .flags = 0 }, diff --git a/ios/MobAI.swift b/ios/MobAI.swift new file mode 100644 index 00000000..7c8f101d --- /dev/null +++ b/ios/MobAI.swift @@ -0,0 +1,235 @@ +import Foundation +import Speech +import Vision + +#if canImport(FoundationModels) +import FoundationModels +#endif + +@objcMembers +public final class MobAI: NSObject { + private static var speechTasks: [SFSpeechRecognitionTask] = [] + + public static func generateText( + _ prompt: String, + optionsJSON: String, + completion: @escaping (String?, String?) -> Void + ) { + #if targetEnvironment(simulator) + completion(nil, "Foundation Models does not run in the iOS simulator.") + #else + guard #available(iOS 26.0, *) else { + completion(nil, "Foundation Models requires iOS 26.0 or newer.") + return + } + + #if canImport(FoundationModels) + let opts = decodeOptions(optionsJSON) + let instructions = opts["instructions"] as? String ?? "" + let temperature = opts["temperature"] as? Double ?? 0.2 + let maximumResponseTokens = opts["maximum_response_tokens"] as? Int ?? 256 + + Task { + let model = SystemLanguageModel.default + + guard model.supportsLocale(Locale.current) else { + completion(nil, "Foundation Models does not support the current locale: \(Locale.current.identifier).") + return + } + + switch model.availability { + case .available: + do { + let session = instructions.isEmpty + ? LanguageModelSession() + : LanguageModelSession(instructions: instructions) + let response = try await session.respond( + to: prompt, + options: GenerationOptions( + temperature: temperature, + maximumResponseTokens: maximumResponseTokens + ) + ) + completion(response.content, nil) + } catch { + completion(nil, "Foundation Models generation error: \(error.localizedDescription)") + } + + case .unavailable(let reason): + completion(nil, foundationAvailabilityMessage(reason)) + } + } + #else + completion(nil, "This build of Xcode does not expose the FoundationModels module.") + #endif + #endif + } + + public static func recognizeText( + atPath path: String, + optionsJSON: String, + completion: @escaping (String?, String?) -> Void + ) { + DispatchQueue.global(qos: .userInitiated).async { + guard FileManager.default.fileExists(atPath: path) else { + completion(nil, "Image file does not exist: \(path)") + return + } + + let opts = decodeOptions(optionsJSON) + let url = URL(fileURLWithPath: path) + let request = VNRecognizeTextRequest { request, error in + if let error { + completion(nil, "Vision OCR error: \(error.localizedDescription)") + return + } + + let observations = request.results as? [VNRecognizedTextObservation] ?? [] + let lines = observations.compactMap { $0.topCandidates(1).first?.string } + completion(lines.joined(separator: "\n"), nil) + } + + let level = opts["recognition_level"] as? String ?? "accurate" + request.recognitionLevel = level == "fast" ? .fast : .accurate + request.usesLanguageCorrection = opts["uses_language_correction"] as? Bool ?? true + request.automaticallyDetectsLanguage = true + + do { + try VNImageRequestHandler(url: url, options: [:]).perform([request]) + } catch { + completion(nil, "Vision OCR error: \(error.localizedDescription)") + } + } + } + + public static func transcribeAudio( + atPath path: String, + optionsJSON: String, + completion: @escaping (String?, String?) -> Void + ) { + guard FileManager.default.fileExists(atPath: path) else { + completion(nil, "Speech audio file does not exist: \(path)") + return + } + + let opts = decodeOptions(optionsJSON) + let localeIdentifier = opts["locale"] as? String ?? "" + let requiresOnDeviceRecognition = opts["requires_on_device_recognition"] as? Bool ?? false + let url = URL(fileURLWithPath: path) + + SFSpeechRecognizer.requestAuthorization { status in + guard status == .authorized else { + completion(nil, "Speech recognition authorization: \(speechAuthorizationName(status)).") + return + } + + guard let recognizer = makeSpeechRecognizer(localeIdentifier: localeIdentifier) else { + completion(nil, speechLocaleDiagnostic(localeIdentifier: localeIdentifier)) + return + } + + guard recognizer.isAvailable else { + completion(nil, "Speech recognizer for \(recognizer.locale.identifier) is not currently available.") + return + } + + let request = SFSpeechURLRecognitionRequest(url: url) + request.shouldReportPartialResults = false + request.requiresOnDeviceRecognition = requiresOnDeviceRecognition + + let task = recognizer.recognitionTask(with: request) { result, error in + if let result, result.isFinal { + completion(result.bestTranscription.formattedString, nil) + speechTasks.removeAll { $0.isFinishing || $0.isCancelled } + return + } + + if let error { + completion(nil, "Speech transcription error for \(recognizer.locale.identifier): \(error.localizedDescription)") + speechTasks.removeAll { $0.isFinishing || $0.isCancelled } + } + } + + speechTasks.append(task) + } + } + + private static func decodeOptions(_ optionsJSON: String) -> [String: Any] { + guard let data = optionsJSON.data(using: .utf8), + let object = try? JSONSerialization.jsonObject(with: data), + let dict = object as? [String: Any] else { + return [:] + } + + return dict + } + + #if canImport(FoundationModels) + @available(iOS 26.0, *) + private static func foundationAvailabilityMessage( + _ reason: SystemLanguageModel.Availability.UnavailableReason + ) -> String { + switch reason { + case .deviceNotEligible: + return "Foundation Models unavailable: this device is not eligible for Apple Intelligence." + case .appleIntelligenceNotEnabled: + return "Foundation Models unavailable: Apple Intelligence is not enabled in Settings." + case .modelNotReady: + return "Foundation Models unavailable: the on-device model is not ready yet." + @unknown default: + return "Foundation Models unavailable: \(String(describing: reason))." + } + } + #endif + + private static func makeSpeechRecognizer(localeIdentifier: String) -> SFSpeechRecognizer? { + if !localeIdentifier.isEmpty { + return SFSpeechRecognizer(locale: Locale(identifier: localeIdentifier)) + } + + if let recognizer = SFSpeechRecognizer() { + return recognizer + } + + let preferredIdentifiers = [ + Locale.current.identifier, + Locale.preferredLanguages.first ?? "", + "en-US", + "en_US" + ] + + for identifier in preferredIdentifiers where !identifier.isEmpty { + if let recognizer = SFSpeechRecognizer(locale: Locale(identifier: identifier)) { + return recognizer + } + } + + return nil + } + + private static func speechLocaleDiagnostic(localeIdentifier: String) -> String { + let requested = localeIdentifier.isEmpty ? "default locale" : localeIdentifier + let supported = SFSpeechRecognizer.supportedLocales() + .map(\.identifier) + .sorted() + .prefix(12) + .joined(separator: ", ") + + return "Failed to initialize speech recognizer for \(requested). Supported locales include: \(supported)." + } + + private static func speechAuthorizationName(_ status: SFSpeechRecognizerAuthorizationStatus) -> String { + switch status { + case .authorized: + return "authorized" + case .denied: + return "denied" + case .restricted: + return "restricted" + case .notDetermined: + return "not determined" + @unknown default: + return "unknown" + } + } +} diff --git a/ios/mob_nif.m b/ios/mob_nif.m index c9a46c3d..011b3d42 100644 --- a/ios/mob_nif.m +++ b/ios/mob_nif.m @@ -1971,6 +1971,112 @@ static ERL_NIF_TERM nif_share_text(ErlNifEnv *env, int argc, const ERL_NIF_TERM return enif_make_atom(env, "ok"); } +// ── Native AI helpers ─────────────────────────────────────────────────────── + +static NSString *mob_ai_string_arg(ErlNifEnv *env, ERL_NIF_TERM term) { + ErlNifBinary bin; + if (!enif_inspect_binary(env, term, &bin) && + !enif_inspect_iolist_as_binary(env, term, &bin)) + return nil; + + return [[NSString alloc] initWithBytes:bin.data length:bin.size encoding:NSUTF8StringEncoding]; +} + +static ERL_NIF_TERM mob_ai_make_utf8_binary(ErlNifEnv *env, NSString *text) { + const char *utf8 = text ? text.UTF8String : ""; + size_t len = utf8 ? strlen(utf8) : 0; + ErlNifBinary bin; + enif_alloc_binary(len, &bin); + if (len > 0) + memcpy(bin.data, utf8, len); + return enif_make_binary(env, &bin); +} + +static void mob_ai_send_success(ErlNifPid pid, const char *event, NSString *text) { + ErlNifEnv *e = enif_alloc_env(); + ERL_NIF_TERM keys[1] = {enif_make_atom(e, "text")}; + ERL_NIF_TERM vals[1] = {mob_ai_make_utf8_binary(e, text)}; + ERL_NIF_TERM payload; + enif_make_map_from_arrays(e, keys, vals, 1, &payload); + ERL_NIF_TERM msg = enif_make_tuple3(e, enif_make_atom(e, "ai"), enif_make_atom(e, event), payload); + enif_send(NULL, &pid, e, msg); + enif_free_env(e); +} + +static void mob_ai_send_error(ErlNifPid pid, const char *operation, NSString *reason) { + ErlNifEnv *e = enif_alloc_env(); + ERL_NIF_TERM keys[2] = {enif_make_atom(e, "operation"), enif_make_atom(e, "reason")}; + ERL_NIF_TERM vals[2] = {enif_make_atom(e, operation), mob_ai_make_utf8_binary(e, reason)}; + ERL_NIF_TERM payload; + enif_make_map_from_arrays(e, keys, vals, 2, &payload); + ERL_NIF_TERM msg = enif_make_tuple3(e, enif_make_atom(e, "ai"), enif_make_atom(e, "error"), payload); + enif_send(NULL, &pid, e, msg); + enif_free_env(e); +} + +static ERL_NIF_TERM nif_ai_generate_text(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + NSString *prompt = mob_ai_string_arg(env, argv[0]); + NSString *optionsJSON = mob_ai_string_arg(env, argv[1]); + if (!prompt || !optionsJSON) + return enif_make_badarg(env); + + ErlNifPid pid; + enif_self(env, &pid); + + [MobAI generateText:prompt + optionsJSON:optionsJSON + completion:^(NSString *text, NSString *error) { + if (error) + mob_ai_send_error(pid, "generate_text", error); + else + mob_ai_send_success(pid, "generated_text", text); + }]; + + return enif_make_atom(env, "ok"); +} + +static ERL_NIF_TERM nif_ai_recognize_text(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + NSString *path = mob_ai_string_arg(env, argv[0]); + NSString *optionsJSON = mob_ai_string_arg(env, argv[1]); + if (!path || !optionsJSON) + return enif_make_badarg(env); + + ErlNifPid pid; + enif_self(env, &pid); + + [MobAI recognizeTextAtPath:path + optionsJSON:optionsJSON + completion:^(NSString *text, NSString *error) { + if (error) + mob_ai_send_error(pid, "recognize_text", error); + else + mob_ai_send_success(pid, "recognized_text", text); + }]; + + return enif_make_atom(env, "ok"); +} + +static ERL_NIF_TERM nif_ai_transcribe_audio(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + NSString *path = mob_ai_string_arg(env, argv[0]); + NSString *optionsJSON = mob_ai_string_arg(env, argv[1]); + if (!path || !optionsJSON) + return enif_make_badarg(env); + + ErlNifPid pid; + enif_self(env, &pid); + + [MobAI transcribeAudioAtPath:path + optionsJSON:optionsJSON + completion:^(NSString *text, NSString *error) { + if (error) + mob_ai_send_error(pid, "transcribe_audio", error); + else + mob_ai_send_success(pid, "transcribed_audio", text); + }]; + + return enif_make_atom(env, "ok"); +} + // ════════════════════════════════════════════════════════════════════════════ // Device capability NIFs // ════════════════════════════════════════════════════════════════════════════ @@ -5789,6 +5895,9 @@ static ERL_NIF_TERM nif_vendor_usb_close(ErlNifEnv *env, int argc, const ERL_NIF {"audio_play", 2, nif_audio_play, 0}, {"audio_stop_playback", 0, nif_audio_stop_playback, 0}, {"audio_set_volume", 1, nif_audio_set_volume, 0}, + {"ai_generate_text", 2, nif_ai_generate_text, 0}, + {"ai_recognize_text", 2, nif_ai_recognize_text, 0}, + {"ai_transcribe_audio", 2, nif_ai_transcribe_audio, 0}, {"motion_start", 2, nif_motion_start, 0}, {"motion_stop", 0, nif_motion_stop, 0}, {"scanner_scan", 1, nif_scanner_scan, 0}, diff --git a/lib/mob/ai.ex b/lib/mob/ai.ex new file mode 100644 index 00000000..99390bfb --- /dev/null +++ b/lib/mob/ai.ex @@ -0,0 +1,137 @@ +defmodule Mob.AI do + @moduledoc """ + Native AI capabilities exposed by the platform. + + These calls are asynchronous. The native side sends results back to the + calling process as `{:ai, event, payload}` messages. + + ## Text generation + + Mob.AI.generate_text(socket, "Summarize this note") + # -> handle_info({:ai, :generated_text, %{text: text}}, socket) + # -> handle_info({:ai, :error, %{operation: :generate_text, reason: reason}}, socket) + + On iOS this uses Apple's Foundation Models framework when available. The iOS + simulator does not provide the on-device system language model. + + ## Vision OCR + + Mob.AI.recognize_text(socket, "/path/to/image.png") + # -> handle_info({:ai, :recognized_text, %{text: text}}, socket) + + ## Speech transcription + + Mob.AI.transcribe_audio(socket, "/path/to/audio.m4a") + # -> handle_info({:ai, :transcribed_audio, %{text: text}}, socket) + + Speech transcription requires speech recognition permission on iOS. + """ + + @type generate_option :: + {:instructions, String.t()} + | {:temperature, number()} + | {:maximum_response_tokens, pos_integer()} + + @type ocr_option :: + {:recognition_level, :fast | :accurate} + | {:uses_language_correction, boolean()} + + @type speech_option :: + {:locale, String.t()} + | {:requires_on_device_recognition, boolean()} + + @doc """ + Generate text using the platform language model. + + Result arrives as: + + * `{:ai, :generated_text, %{text: text}}` + * `{:ai, :error, %{operation: :generate_text, reason: reason}}` + """ + @spec generate_text(Mob.Socket.t(), String.t(), [generate_option()]) :: Mob.Socket.t() + def generate_text(socket, prompt, opts \\ []) when is_binary(prompt) and is_list(opts) do + invoke( + :mob_nif.ai_generate_text(prompt, :json.encode(generate_text_opts(opts))), + :generate_text + ) + + socket + end + + @doc false + @spec generate_text_opts(keyword()) :: %{String.t() => term()} + def generate_text_opts(opts) do + %{ + "instructions" => Keyword.get(opts, :instructions, ""), + "temperature" => Keyword.get(opts, :temperature, 0.2) / 1, + "maximum_response_tokens" => Keyword.get(opts, :maximum_response_tokens, 256) + } + end + + @doc """ + Recognize text in an image file using platform OCR. + + Result arrives as: + + * `{:ai, :recognized_text, %{text: text}}` + * `{:ai, :error, %{operation: :recognize_text, reason: reason}}` + """ + @spec recognize_text(Mob.Socket.t(), String.t(), [ocr_option()]) :: Mob.Socket.t() + def recognize_text(socket, path, opts \\ []) when is_binary(path) and is_list(opts) do + invoke( + :mob_nif.ai_recognize_text(path, :json.encode(recognize_text_opts(opts))), + :recognize_text + ) + + socket + end + + @doc false + @spec recognize_text_opts(keyword()) :: %{String.t() => term()} + def recognize_text_opts(opts) do + %{ + "recognition_level" => Keyword.get(opts, :recognition_level, :accurate) |> Atom.to_string(), + "uses_language_correction" => Keyword.get(opts, :uses_language_correction, true) + } + end + + @doc """ + Transcribe an audio file using platform speech recognition. + + Result arrives as: + + * `{:ai, :transcribed_audio, %{text: text}}` + * `{:ai, :error, %{operation: :transcribe_audio, reason: reason}}` + """ + @spec transcribe_audio(Mob.Socket.t(), String.t(), [speech_option()]) :: Mob.Socket.t() + def transcribe_audio(socket, path, opts \\ []) when is_binary(path) and is_list(opts) do + invoke( + :mob_nif.ai_transcribe_audio(path, :json.encode(transcribe_audio_opts(opts))), + :transcribe_audio + ) + + socket + end + + @doc false + @spec transcribe_audio_opts(keyword()) :: %{String.t() => term()} + def transcribe_audio_opts(opts) do + %{ + "locale" => Keyword.get(opts, :locale, ""), + "requires_on_device_recognition" => + Keyword.get(opts, :requires_on_device_recognition, false) + } + end + + defp invoke(:ok, _operation), do: :ok + + defp invoke(:unsupported, operation) do + send( + self(), + {:ai, :error, + %{operation: operation, reason: "Native AI is not supported on this platform."}} + ) + + :ok + end +end diff --git a/src/mob_nif.erl b/src/mob_nif.erl index 722932d2..fecd66fa 100644 --- a/src/mob_nif.erl +++ b/src/mob_nif.erl @@ -41,6 +41,10 @@ audio_play/2, audio_stop_playback/0, audio_set_volume/1, + %% Native AI + ai_generate_text/2, + ai_recognize_text/2, + ai_transcribe_audio/2, %% Motion sensors motion_start/2, motion_stop/0, @@ -140,6 +144,9 @@ audio_play/2, audio_stop_playback/0, audio_set_volume/1, + ai_generate_text/2, + ai_recognize_text/2, + ai_transcribe_audio/2, motion_start/2, motion_stop/0, scanner_scan/1, @@ -237,6 +244,9 @@ audio_stop_recording() -> erlang:nif_error(not_loaded). audio_play(_Path, _OptsJson) -> erlang:nif_error(not_loaded). audio_stop_playback() -> erlang:nif_error(not_loaded). audio_set_volume(_Volume) -> erlang:nif_error(not_loaded). +ai_generate_text(_Prompt, _OptsJson) -> erlang:nif_error(not_loaded). +ai_recognize_text(_Path, _OptsJson) -> erlang:nif_error(not_loaded). +ai_transcribe_audio(_Path, _OptsJson) -> erlang:nif_error(not_loaded). motion_start(_Sensors, _Interval) -> erlang:nif_error(not_loaded). motion_stop() -> erlang:nif_error(not_loaded). scanner_scan(_FormatsJson) -> erlang:nif_error(not_loaded). diff --git a/test/mob/ai_test.exs b/test/mob/ai_test.exs new file mode 100644 index 00000000..fb6d9411 --- /dev/null +++ b/test/mob/ai_test.exs @@ -0,0 +1,59 @@ +defmodule Mob.AITest do + use ExUnit.Case, async: true + + alias Mob.AI + + describe "generate_text_opts/1" do + test "defaults are string keyed" do + assert AI.generate_text_opts([]) == %{ + "instructions" => "", + "temperature" => 0.2, + "maximum_response_tokens" => 256 + } + end + + test "custom values are passed through" do + assert AI.generate_text_opts( + instructions: "Be concise", + temperature: 0.7, + maximum_response_tokens: 64 + ) == %{ + "instructions" => "Be concise", + "temperature" => 0.7, + "maximum_response_tokens" => 64 + } + end + end + + describe "recognize_text_opts/1" do + test "defaults to accurate OCR with language correction" do + assert AI.recognize_text_opts([]) == %{ + "recognition_level" => "accurate", + "uses_language_correction" => true + } + end + + test "recognition level is encoded as a string" do + assert AI.recognize_text_opts(recognition_level: :fast) == %{ + "recognition_level" => "fast", + "uses_language_correction" => true + } + end + end + + describe "transcribe_audio_opts/1" do + test "defaults to platform locale and server-capable recognition" do + assert AI.transcribe_audio_opts([]) == %{ + "locale" => "", + "requires_on_device_recognition" => false + } + end + + test "custom locale and on-device setting are passed through" do + assert AI.transcribe_audio_opts(locale: "en-US", requires_on_device_recognition: true) == %{ + "locale" => "en-US", + "requires_on_device_recognition" => true + } + end + end +end From 4f04ef0fcb8b98b05752524643a258e0508d4b8f Mon Sep 17 00:00:00 2001 From: Yurko Hoshko Date: Thu, 14 May 2026 22:47:38 -0700 Subject: [PATCH 2/6] Clean up native intelligence APIs --- README.md | 3 +- android/jni/mob_nif.zig | 12 +- guides/device_capabilities.md | 72 +++++++++ guides/native_intelligence.md | 88 +++++++++++ ios/MobAI.swift | 235 ---------------------------- ios/MobFoundationModels.swift | 91 +++++++++++ ios/MobSpeech.swift | 120 ++++++++++++++ ios/MobVision.swift | 52 ++++++ ios/mob_nif.m | 100 ++++++------ lib/mob/ai.ex | 137 ---------------- lib/mob/foundation_models.ex | 66 ++++++++ lib/mob/speech.ex | 63 ++++++++ lib/mob/vision.ex | 58 +++++++ mix.exs | 4 + src/mob_nif.erl | 22 +-- test/mob/ai_test.exs | 59 ------- test/mob/foundation_models_test.exs | 27 ++++ test/mob/speech_test.exs | 22 +++ test/mob/vision_test.exs | 21 +++ 19 files changed, 759 insertions(+), 493 deletions(-) create mode 100644 guides/native_intelligence.md delete mode 100644 ios/MobAI.swift create mode 100644 ios/MobFoundationModels.swift create mode 100644 ios/MobSpeech.swift create mode 100644 ios/MobVision.swift delete mode 100644 lib/mob/ai.ex create mode 100644 lib/mob/foundation_models.ex create mode 100644 lib/mob/speech.ex create mode 100644 lib/mob/vision.ex delete mode 100644 test/mob/ai_test.exs create mode 100644 test/mob/foundation_models_test.exs create mode 100644 test/mob/speech_test.exs create mode 100644 test/mob/vision_test.exs diff --git a/README.md b/README.md index 534c8b77..f4eb1e73 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,7 @@ Mob.Notify.register_push(socket) def handle_info({:push_token, :ios, token}, socket), do: ... ``` -Also: `Mob.Clipboard`, `Mob.Share`, `Mob.Photos`, `Mob.Files`, `Mob.Audio`, `Mob.Motion`, `Mob.Biometric`, `Mob.Scanner`, `Mob.Permissions`. +Also: `Mob.Clipboard`, `Mob.Share`, `Mob.Photos`, `Mob.Files`, `Mob.Audio`, `Mob.FoundationModels`, `Mob.Vision`, `Mob.Speech`, `Mob.Motion`, `Mob.Biometric`, `Mob.Scanner`, `Mob.Permissions`. ## What's in the box @@ -156,6 +156,7 @@ The pre-built OTP runtime that ships with each app includes: Native APIs surfaced via `Mob.*` modules (above) cover camera, location, audio, files, biometrics, push, clipboard, share, scanner, +Foundation Models, Vision, Speech, motion sensors, permissions. The OTP runtime tarball is ~80 MB compressed; sliced per-arch by diff --git a/android/jni/mob_nif.zig b/android/jni/mob_nif.zig index 984dc6fa..ec8486a3 100644 --- a/android/jni/mob_nif.zig +++ b/android/jni/mob_nif.zig @@ -2433,7 +2433,7 @@ export fn nif_audio_set_volume( return erts.ok(env); } -export fn nif_ai_generate_text( +export fn nif_foundation_models_generate_text( env: ?*erts.ErlNifEnv, argc: c_int, argv: [*]const erts.ERL_NIF_TERM, @@ -2443,7 +2443,7 @@ export fn nif_ai_generate_text( return erts.atom(env, "unsupported"); } -export fn nif_ai_recognize_text( +export fn nif_vision_recognize_text( env: ?*erts.ErlNifEnv, argc: c_int, argv: [*]const erts.ERL_NIF_TERM, @@ -2453,7 +2453,7 @@ export fn nif_ai_recognize_text( return erts.atom(env, "unsupported"); } -export fn nif_ai_transcribe_audio( +export fn nif_speech_transcribe_audio( env: ?*erts.ErlNifEnv, argc: c_int, argv: [*]const erts.ERL_NIF_TERM, @@ -3303,9 +3303,9 @@ const nif_funcs = [_]erts.ErlNifFunc{ .{ .name = "audio_play", .arity = 2, .fptr = nif_audio_play, .flags = 0 }, .{ .name = "audio_stop_playback", .arity = 0, .fptr = nif_audio_stop_playback, .flags = 0 }, .{ .name = "audio_set_volume", .arity = 1, .fptr = nif_audio_set_volume, .flags = 0 }, - .{ .name = "ai_generate_text", .arity = 2, .fptr = nif_ai_generate_text, .flags = 0 }, - .{ .name = "ai_recognize_text", .arity = 2, .fptr = nif_ai_recognize_text, .flags = 0 }, - .{ .name = "ai_transcribe_audio", .arity = 2, .fptr = nif_ai_transcribe_audio, .flags = 0 }, + .{ .name = "foundation_models_generate_text", .arity = 2, .fptr = nif_foundation_models_generate_text, .flags = 0 }, + .{ .name = "vision_recognize_text", .arity = 2, .fptr = nif_vision_recognize_text, .flags = 0 }, + .{ .name = "speech_transcribe_audio", .arity = 2, .fptr = nif_speech_transcribe_audio, .flags = 0 }, .{ .name = "motion_start", .arity = 2, .fptr = nif_motion_start, .flags = 0 }, .{ .name = "motion_stop", .arity = 0, .fptr = nif_motion_stop, .flags = 0 }, .{ .name = "scanner_scan", .arity = 1, .fptr = nif_scanner_scan, .flags = 0 }, diff --git a/guides/device_capabilities.md b/guides/device_capabilities.md index dee72e73..180b8669 100644 --- a/guides/device_capabilities.md +++ b/guides/device_capabilities.md @@ -205,6 +205,78 @@ end iOS uses `AVAudioPlayer` / `AVPlayer`. Android uses `MediaPlayer`. +## Foundation Models + +iOS only. Generates text with Apple's on-device Foundation Models framework. +Requires an eligible physical device with Apple Intelligence enabled; the iOS +simulator reports this capability as unavailable. + +Apple docs: +[Foundation Models](https://developer.apple.com/documentation/foundationmodels) and +[Adding intelligent app features with generative models](https://developer.apple.com/documentation/foundationmodels/adding-intelligent-app-features-with-generative-models). + +```elixir +socket = + Mob.FoundationModels.generate_text(socket, "Turn this note into a short action list", + instructions: "Return compact plain text.", + temperature: 0.2, + maximum_response_tokens: 240 + ) + +def handle_info({:foundation_models, :generated_text, %{text: text}}, socket) do + {:noreply, Mob.Socket.assign(socket, :result, text)} +end + +def handle_info({:foundation_models, :error, %{reason: reason}}, socket) do + {:noreply, Mob.Socket.assign(socket, :error, reason)} +end +``` + +## Vision text recognition + +iOS only for now. Recognizes text in a local image file with Apple's Vision +framework. Combine with `Mob.Photos.pick/2` to OCR a user-selected image. + +Apple docs: +[VNRecognizeTextRequest](https://developer.apple.com/documentation/vision/vnrecognizetextrequest). + +```elixir +socket = + Mob.Vision.recognize_text(socket, image_path, + recognition_level: :accurate, + uses_language_correction: true + ) + +def handle_info({:vision, :recognized_text, %{text: text}}, socket) do + {:noreply, Mob.Socket.assign(socket, :ocr_text, text)} +end +``` + +## Speech transcription + +iOS only for now. Transcribes an existing audio file with Apple's Speech +framework. Use `Mob.Audio` to record microphone input first. + +Apple docs: +[SFSpeechRecognizer](https://developer.apple.com/documentation/speech/sfspeechrecognizer) and +[SFSpeechURLRecognitionRequest](https://developer.apple.com/documentation/speech/sfspeechurlrecognitionrequest). + +```elixir +socket = + Mob.Speech.transcribe_audio(socket, recording_path, + locale: "en-US", + requires_on_device_recognition: false + ) + +def handle_info({:speech, :transcribed_audio, %{text: text}}, socket) do + {:noreply, Mob.Socket.assign(socket, :transcript, text)} +end +``` + +Speech recognition requires iOS speech authorization. If +`requires_on_device_recognition` is true, iOS may reject locales that do not +support local recognition. + ## Location Requires `:location` permission. diff --git a/guides/native_intelligence.md b/guides/native_intelligence.md new file mode 100644 index 00000000..6529fc54 --- /dev/null +++ b/guides/native_intelligence.md @@ -0,0 +1,88 @@ +# Native Intelligence APIs + +Mob exposes a small, iOS-first bridge to Apple-native intelligence APIs: + +- `Mob.FoundationModels` for Foundation Models text generation. +- `Mob.Vision` for Vision text recognition. +- `Mob.Speech` for Speech framework file transcription. + +These APIs deliberately mirror Apple's framework boundaries instead of grouping +everything under a generic "AI" namespace. That keeps the Elixir surface close +to the native SDK names and leaves room for platform-specific capabilities to +grow without a catch-all module. + +Apple references: + +- [Foundation Models](https://developer.apple.com/documentation/foundationmodels) +- [Adding intelligent app features with generative models](https://developer.apple.com/documentation/foundationmodels/adding-intelligent-app-features-with-generative-models) +- [VNRecognizeTextRequest](https://developer.apple.com/documentation/vision/vnrecognizetextrequest) +- [SFSpeechRecognizer](https://developer.apple.com/documentation/speech/sfspeechrecognizer) +- [SFSpeechURLRecognitionRequest](https://developer.apple.com/documentation/speech/sfspeechurlrecognitionrequest) + +## Example Flow + +There is no checked-in sample app in the `mob` repository today. A complete app +can compose existing Mob features with the new native modules: + +```elixir +def handle_info({:photos, :picked, [%{path: path} | _]}, socket) do + {:noreply, Mob.Vision.recognize_text(socket, path)} +end + +def handle_info({:vision, :recognized_text, %{text: text}}, socket) do + prompt = "Summarize this OCR text as actions:\n\n#{text}" + {:noreply, Mob.FoundationModels.generate_text(socket, prompt)} +end + +def handle_info({:foundation_models, :generated_text, %{text: text}}, socket) do + {:noreply, Mob.Socket.assign(socket, :result, text)} +end +``` + +Speech uses the same pattern with `Mob.Audio`: + +```elixir +def handle_info({:audio, :recorded, %{path: path}}, socket) do + {:noreply, Mob.Speech.transcribe_audio(socket, path)} +end +``` + +## Current Scope + +Included: + +- Foundation Models plain text generation. +- Vision OCR from a local image path. +- Speech transcription from a local audio file. +- Android stubs that return `:unsupported` so apps can branch cleanly. + +Out of scope for this first bridge: + +- Foundation Models structured generation with `@Generable`. +- Streaming partial Foundation Models responses. +- Tool calling, multi-turn session persistence, or model transcript management. +- Vision requests beyond text recognition, such as barcode, face, object, and + document detection. +- Speech live microphone recognition, partial transcripts, custom language + models, and keyword spotting. +- Natural Language framework features such as language identification, + sentiment, tokenization, and embedding/classification APIs. +- Image generation or Private Cloud Compute-backed server features. +- Android ML Kit or platform-equivalent implementations. + +## Operational Notes + +Foundation Models is not available in the iOS simulator. On device it can still +be unavailable if Apple Intelligence is disabled, the device is not eligible, +the current locale is unsupported, or the model is not ready. + +Vision OCR needs a readable local file path. Photo-picker temporary files should +be copied if the app needs to keep them beyond the current workflow. + +Speech transcription requires iOS speech-recognition authorization. On-device +recognition is locale-dependent; setting `requires_on_device_recognition: true` +can make otherwise valid transcriptions fail. + +All three APIs send results back to the calling screen process. Treat them like +other Mob asynchronous device APIs: update screen state in `handle_info/2`, and +keep long-running UX cancellable at the app level. diff --git a/ios/MobAI.swift b/ios/MobAI.swift deleted file mode 100644 index 7c8f101d..00000000 --- a/ios/MobAI.swift +++ /dev/null @@ -1,235 +0,0 @@ -import Foundation -import Speech -import Vision - -#if canImport(FoundationModels) -import FoundationModels -#endif - -@objcMembers -public final class MobAI: NSObject { - private static var speechTasks: [SFSpeechRecognitionTask] = [] - - public static func generateText( - _ prompt: String, - optionsJSON: String, - completion: @escaping (String?, String?) -> Void - ) { - #if targetEnvironment(simulator) - completion(nil, "Foundation Models does not run in the iOS simulator.") - #else - guard #available(iOS 26.0, *) else { - completion(nil, "Foundation Models requires iOS 26.0 or newer.") - return - } - - #if canImport(FoundationModels) - let opts = decodeOptions(optionsJSON) - let instructions = opts["instructions"] as? String ?? "" - let temperature = opts["temperature"] as? Double ?? 0.2 - let maximumResponseTokens = opts["maximum_response_tokens"] as? Int ?? 256 - - Task { - let model = SystemLanguageModel.default - - guard model.supportsLocale(Locale.current) else { - completion(nil, "Foundation Models does not support the current locale: \(Locale.current.identifier).") - return - } - - switch model.availability { - case .available: - do { - let session = instructions.isEmpty - ? LanguageModelSession() - : LanguageModelSession(instructions: instructions) - let response = try await session.respond( - to: prompt, - options: GenerationOptions( - temperature: temperature, - maximumResponseTokens: maximumResponseTokens - ) - ) - completion(response.content, nil) - } catch { - completion(nil, "Foundation Models generation error: \(error.localizedDescription)") - } - - case .unavailable(let reason): - completion(nil, foundationAvailabilityMessage(reason)) - } - } - #else - completion(nil, "This build of Xcode does not expose the FoundationModels module.") - #endif - #endif - } - - public static func recognizeText( - atPath path: String, - optionsJSON: String, - completion: @escaping (String?, String?) -> Void - ) { - DispatchQueue.global(qos: .userInitiated).async { - guard FileManager.default.fileExists(atPath: path) else { - completion(nil, "Image file does not exist: \(path)") - return - } - - let opts = decodeOptions(optionsJSON) - let url = URL(fileURLWithPath: path) - let request = VNRecognizeTextRequest { request, error in - if let error { - completion(nil, "Vision OCR error: \(error.localizedDescription)") - return - } - - let observations = request.results as? [VNRecognizedTextObservation] ?? [] - let lines = observations.compactMap { $0.topCandidates(1).first?.string } - completion(lines.joined(separator: "\n"), nil) - } - - let level = opts["recognition_level"] as? String ?? "accurate" - request.recognitionLevel = level == "fast" ? .fast : .accurate - request.usesLanguageCorrection = opts["uses_language_correction"] as? Bool ?? true - request.automaticallyDetectsLanguage = true - - do { - try VNImageRequestHandler(url: url, options: [:]).perform([request]) - } catch { - completion(nil, "Vision OCR error: \(error.localizedDescription)") - } - } - } - - public static func transcribeAudio( - atPath path: String, - optionsJSON: String, - completion: @escaping (String?, String?) -> Void - ) { - guard FileManager.default.fileExists(atPath: path) else { - completion(nil, "Speech audio file does not exist: \(path)") - return - } - - let opts = decodeOptions(optionsJSON) - let localeIdentifier = opts["locale"] as? String ?? "" - let requiresOnDeviceRecognition = opts["requires_on_device_recognition"] as? Bool ?? false - let url = URL(fileURLWithPath: path) - - SFSpeechRecognizer.requestAuthorization { status in - guard status == .authorized else { - completion(nil, "Speech recognition authorization: \(speechAuthorizationName(status)).") - return - } - - guard let recognizer = makeSpeechRecognizer(localeIdentifier: localeIdentifier) else { - completion(nil, speechLocaleDiagnostic(localeIdentifier: localeIdentifier)) - return - } - - guard recognizer.isAvailable else { - completion(nil, "Speech recognizer for \(recognizer.locale.identifier) is not currently available.") - return - } - - let request = SFSpeechURLRecognitionRequest(url: url) - request.shouldReportPartialResults = false - request.requiresOnDeviceRecognition = requiresOnDeviceRecognition - - let task = recognizer.recognitionTask(with: request) { result, error in - if let result, result.isFinal { - completion(result.bestTranscription.formattedString, nil) - speechTasks.removeAll { $0.isFinishing || $0.isCancelled } - return - } - - if let error { - completion(nil, "Speech transcription error for \(recognizer.locale.identifier): \(error.localizedDescription)") - speechTasks.removeAll { $0.isFinishing || $0.isCancelled } - } - } - - speechTasks.append(task) - } - } - - private static func decodeOptions(_ optionsJSON: String) -> [String: Any] { - guard let data = optionsJSON.data(using: .utf8), - let object = try? JSONSerialization.jsonObject(with: data), - let dict = object as? [String: Any] else { - return [:] - } - - return dict - } - - #if canImport(FoundationModels) - @available(iOS 26.0, *) - private static func foundationAvailabilityMessage( - _ reason: SystemLanguageModel.Availability.UnavailableReason - ) -> String { - switch reason { - case .deviceNotEligible: - return "Foundation Models unavailable: this device is not eligible for Apple Intelligence." - case .appleIntelligenceNotEnabled: - return "Foundation Models unavailable: Apple Intelligence is not enabled in Settings." - case .modelNotReady: - return "Foundation Models unavailable: the on-device model is not ready yet." - @unknown default: - return "Foundation Models unavailable: \(String(describing: reason))." - } - } - #endif - - private static func makeSpeechRecognizer(localeIdentifier: String) -> SFSpeechRecognizer? { - if !localeIdentifier.isEmpty { - return SFSpeechRecognizer(locale: Locale(identifier: localeIdentifier)) - } - - if let recognizer = SFSpeechRecognizer() { - return recognizer - } - - let preferredIdentifiers = [ - Locale.current.identifier, - Locale.preferredLanguages.first ?? "", - "en-US", - "en_US" - ] - - for identifier in preferredIdentifiers where !identifier.isEmpty { - if let recognizer = SFSpeechRecognizer(locale: Locale(identifier: identifier)) { - return recognizer - } - } - - return nil - } - - private static func speechLocaleDiagnostic(localeIdentifier: String) -> String { - let requested = localeIdentifier.isEmpty ? "default locale" : localeIdentifier - let supported = SFSpeechRecognizer.supportedLocales() - .map(\.identifier) - .sorted() - .prefix(12) - .joined(separator: ", ") - - return "Failed to initialize speech recognizer for \(requested). Supported locales include: \(supported)." - } - - private static func speechAuthorizationName(_ status: SFSpeechRecognizerAuthorizationStatus) -> String { - switch status { - case .authorized: - return "authorized" - case .denied: - return "denied" - case .restricted: - return "restricted" - case .notDetermined: - return "not determined" - @unknown default: - return "unknown" - } - } -} diff --git a/ios/MobFoundationModels.swift b/ios/MobFoundationModels.swift new file mode 100644 index 00000000..d9e6f53a --- /dev/null +++ b/ios/MobFoundationModels.swift @@ -0,0 +1,91 @@ +import Foundation + +#if canImport(FoundationModels) +import FoundationModels +#endif + +@objcMembers +public final class MobFoundationModels: NSObject { + public static func generateText( + _ prompt: String, + optionsJSON: String, + completion: @escaping (String?, String?) -> Void + ) { + #if targetEnvironment(simulator) + completion(nil, "Foundation Models does not run in the iOS simulator.") + #else + guard #available(iOS 26.0, *) else { + completion(nil, "Foundation Models requires iOS 26.0 or newer.") + return + } + + #if canImport(FoundationModels) + let opts = decodeOptions(optionsJSON) + let instructions = opts["instructions"] as? String ?? "" + let temperature = opts["temperature"] as? Double ?? 0.2 + let maximumResponseTokens = opts["maximum_response_tokens"] as? Int ?? 256 + + Task { + let model = SystemLanguageModel.default + + guard model.supportsLocale(Locale.current) else { + completion(nil, "Foundation Models does not support the current locale: \(Locale.current.identifier).") + return + } + + switch model.availability { + case .available: + do { + let session = instructions.isEmpty + ? LanguageModelSession() + : LanguageModelSession(instructions: instructions) + let response = try await session.respond( + to: prompt, + options: GenerationOptions( + temperature: temperature, + maximumResponseTokens: maximumResponseTokens + ) + ) + completion(response.content, nil) + } catch { + completion(nil, "Foundation Models generation error: \(error.localizedDescription)") + } + + case .unavailable(let reason): + completion(nil, availabilityMessage(reason)) + } + } + #else + completion(nil, "This build of Xcode does not expose the FoundationModels module.") + #endif + #endif + } + + private static func decodeOptions(_ optionsJSON: String) -> [String: Any] { + guard let data = optionsJSON.data(using: .utf8), + let object = try? JSONSerialization.jsonObject(with: data), + let dict = object as? [String: Any] else { + return [:] + } + + return dict + } + + #if canImport(FoundationModels) + @available(iOS 26.0, *) + private static func availabilityMessage( + _ reason: SystemLanguageModel.Availability.UnavailableReason + ) -> String { + switch reason { + case .deviceNotEligible: + return "Foundation Models unavailable: this device is not eligible for Apple Intelligence." + case .appleIntelligenceNotEnabled: + return "Foundation Models unavailable: Apple Intelligence is not enabled in Settings." + case .modelNotReady: + return "Foundation Models unavailable: the on-device model is not ready yet." + @unknown default: + return "Foundation Models unavailable: \(String(describing: reason))." + } + } + #endif +} diff --git a/ios/MobSpeech.swift b/ios/MobSpeech.swift new file mode 100644 index 00000000..f1211147 --- /dev/null +++ b/ios/MobSpeech.swift @@ -0,0 +1,120 @@ +import Foundation +import Speech + +@objcMembers +public final class MobSpeech: NSObject { + private static var tasks: [SFSpeechRecognitionTask] = [] + + public static func transcribeAudio( + atPath path: String, + optionsJSON: String, + completion: @escaping (String?, String?) -> Void + ) { + guard FileManager.default.fileExists(atPath: path) else { + completion(nil, "Speech audio file does not exist: \(path)") + return + } + + let opts = decodeOptions(optionsJSON) + let localeIdentifier = opts["locale"] as? String ?? "" + let requiresOnDeviceRecognition = opts["requires_on_device_recognition"] as? Bool ?? false + let url = URL(fileURLWithPath: path) + + SFSpeechRecognizer.requestAuthorization { status in + guard status == .authorized else { + completion(nil, "Speech recognition authorization: \(authorizationName(status)).") + return + } + + guard let recognizer = makeRecognizer(localeIdentifier: localeIdentifier) else { + completion(nil, localeDiagnostic(localeIdentifier: localeIdentifier)) + return + } + + guard recognizer.isAvailable else { + completion(nil, "Speech recognizer for \(recognizer.locale.identifier) is not currently available.") + return + } + + let request = SFSpeechURLRecognitionRequest(url: url) + request.shouldReportPartialResults = false + request.requiresOnDeviceRecognition = requiresOnDeviceRecognition + + let task = recognizer.recognitionTask(with: request) { result, error in + if let result, result.isFinal { + completion(result.bestTranscription.formattedString, nil) + tasks.removeAll { $0.isFinishing || $0.isCancelled } + return + } + + if let error { + completion(nil, "Speech transcription error for \(recognizer.locale.identifier): \(error.localizedDescription)") + tasks.removeAll { $0.isFinishing || $0.isCancelled } + } + } + + tasks.append(task) + } + } + + private static func decodeOptions(_ optionsJSON: String) -> [String: Any] { + guard let data = optionsJSON.data(using: .utf8), + let object = try? JSONSerialization.jsonObject(with: data), + let dict = object as? [String: Any] else { + return [:] + } + + return dict + } + + private static func makeRecognizer(localeIdentifier: String) -> SFSpeechRecognizer? { + if !localeIdentifier.isEmpty { + return SFSpeechRecognizer(locale: Locale(identifier: localeIdentifier)) + } + + if let recognizer = SFSpeechRecognizer() { + return recognizer + } + + let preferredIdentifiers = [ + Locale.current.identifier, + Locale.preferredLanguages.first ?? "", + "en-US", + "en_US" + ] + + for identifier in preferredIdentifiers where !identifier.isEmpty { + if let recognizer = SFSpeechRecognizer(locale: Locale(identifier: identifier)) { + return recognizer + } + } + + return nil + } + + private static func localeDiagnostic(localeIdentifier: String) -> String { + let requested = localeIdentifier.isEmpty ? "default locale" : localeIdentifier + let supported = SFSpeechRecognizer.supportedLocales() + .map(\.identifier) + .sorted() + .prefix(12) + .joined(separator: ", ") + + return "Failed to initialize speech recognizer for \(requested). Supported locales include: \(supported)." + } + + private static func authorizationName(_ status: SFSpeechRecognizerAuthorizationStatus) -> String { + switch status { + case .authorized: + return "authorized" + case .denied: + return "denied" + case .restricted: + return "restricted" + case .notDetermined: + return "not determined" + @unknown default: + return "unknown" + } + } +} diff --git a/ios/MobVision.swift b/ios/MobVision.swift new file mode 100644 index 00000000..f000e54f --- /dev/null +++ b/ios/MobVision.swift @@ -0,0 +1,52 @@ +import Foundation +import Vision + +@objcMembers +public final class MobVision: NSObject { + public static func recognizeText( + atPath path: String, + optionsJSON: String, + completion: @escaping (String?, String?) -> Void + ) { + DispatchQueue.global(qos: .userInitiated).async { + guard FileManager.default.fileExists(atPath: path) else { + completion(nil, "Image file does not exist: \(path)") + return + } + + let opts = decodeOptions(optionsJSON) + let url = URL(fileURLWithPath: path) + let request = VNRecognizeTextRequest { request, error in + if let error { + completion(nil, "Vision OCR error: \(error.localizedDescription)") + return + } + + let observations = request.results as? [VNRecognizedTextObservation] ?? [] + let lines = observations.compactMap { $0.topCandidates(1).first?.string } + completion(lines.joined(separator: "\n"), nil) + } + + let level = opts["recognition_level"] as? String ?? "accurate" + request.recognitionLevel = level == "fast" ? .fast : .accurate + request.usesLanguageCorrection = opts["uses_language_correction"] as? Bool ?? true + request.automaticallyDetectsLanguage = true + + do { + try VNImageRequestHandler(url: url, options: [:]).perform([request]) + } catch { + completion(nil, "Vision OCR error: \(error.localizedDescription)") + } + } + } + + private static func decodeOptions(_ optionsJSON: String) -> [String: Any] { + guard let data = optionsJSON.data(using: .utf8), + let object = try? JSONSerialization.jsonObject(with: data), + let dict = object as? [String: Any] else { + return [:] + } + + return dict + } +} diff --git a/ios/mob_nif.m b/ios/mob_nif.m index 011b3d42..ab2d6bdc 100644 --- a/ios/mob_nif.m +++ b/ios/mob_nif.m @@ -1971,9 +1971,9 @@ static ERL_NIF_TERM nif_share_text(ErlNifEnv *env, int argc, const ERL_NIF_TERM return enif_make_atom(env, "ok"); } -// ── Native AI helpers ─────────────────────────────────────────────────────── +// ── Native text capability helpers ───────────────────────────────────────── -static NSString *mob_ai_string_arg(ErlNifEnv *env, ERL_NIF_TERM term) { +static NSString *mob_native_text_string_arg(ErlNifEnv *env, ERL_NIF_TERM term) { ErlNifBinary bin; if (!enif_inspect_binary(env, term, &bin) && !enif_inspect_iolist_as_binary(env, term, &bin)) @@ -1982,7 +1982,7 @@ static ERL_NIF_TERM nif_share_text(ErlNifEnv *env, int argc, const ERL_NIF_TERM return [[NSString alloc] initWithBytes:bin.data length:bin.size encoding:NSUTF8StringEncoding]; } -static ERL_NIF_TERM mob_ai_make_utf8_binary(ErlNifEnv *env, NSString *text) { +static ERL_NIF_TERM mob_native_text_make_utf8_binary(ErlNifEnv *env, NSString *text) { const char *utf8 = text ? text.UTF8String : ""; size_t len = utf8 ? strlen(utf8) : 0; ErlNifBinary bin; @@ -1992,87 +1992,97 @@ static ERL_NIF_TERM mob_ai_make_utf8_binary(ErlNifEnv *env, NSString *text) { return enif_make_binary(env, &bin); } -static void mob_ai_send_success(ErlNifPid pid, const char *event, NSString *text) { +static void mob_native_text_send_success(ErlNifPid pid, const char *family, const char *event, + NSString *text) { ErlNifEnv *e = enif_alloc_env(); ERL_NIF_TERM keys[1] = {enif_make_atom(e, "text")}; - ERL_NIF_TERM vals[1] = {mob_ai_make_utf8_binary(e, text)}; + ERL_NIF_TERM vals[1] = {mob_native_text_make_utf8_binary(e, text)}; ERL_NIF_TERM payload; enif_make_map_from_arrays(e, keys, vals, 1, &payload); - ERL_NIF_TERM msg = enif_make_tuple3(e, enif_make_atom(e, "ai"), enif_make_atom(e, event), payload); + ERL_NIF_TERM msg = enif_make_tuple3(e, enif_make_atom(e, family), enif_make_atom(e, event), payload); enif_send(NULL, &pid, e, msg); enif_free_env(e); } -static void mob_ai_send_error(ErlNifPid pid, const char *operation, NSString *reason) { +static void mob_native_text_send_error(ErlNifPid pid, const char *family, const char *operation, + NSString *reason) { ErlNifEnv *e = enif_alloc_env(); ERL_NIF_TERM keys[2] = {enif_make_atom(e, "operation"), enif_make_atom(e, "reason")}; - ERL_NIF_TERM vals[2] = {enif_make_atom(e, operation), mob_ai_make_utf8_binary(e, reason)}; + ERL_NIF_TERM vals[2] = {enif_make_atom(e, operation), mob_native_text_make_utf8_binary(e, reason)}; ERL_NIF_TERM payload; enif_make_map_from_arrays(e, keys, vals, 2, &payload); - ERL_NIF_TERM msg = enif_make_tuple3(e, enif_make_atom(e, "ai"), enif_make_atom(e, "error"), payload); + ERL_NIF_TERM msg = enif_make_tuple3(e, enif_make_atom(e, family), enif_make_atom(e, "error"), payload); enif_send(NULL, &pid, e, msg); enif_free_env(e); } -static ERL_NIF_TERM nif_ai_generate_text(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - NSString *prompt = mob_ai_string_arg(env, argv[0]); - NSString *optionsJSON = mob_ai_string_arg(env, argv[1]); +static ERL_NIF_TERM nif_foundation_models_generate_text(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + // Apple Foundation Models: + // https://developer.apple.com/documentation/foundationmodels + NSString *prompt = mob_native_text_string_arg(env, argv[0]); + NSString *optionsJSON = mob_native_text_string_arg(env, argv[1]); if (!prompt || !optionsJSON) return enif_make_badarg(env); ErlNifPid pid; enif_self(env, &pid); - [MobAI generateText:prompt - optionsJSON:optionsJSON - completion:^(NSString *text, NSString *error) { - if (error) - mob_ai_send_error(pid, "generate_text", error); - else - mob_ai_send_success(pid, "generated_text", text); - }]; + [MobFoundationModels generateText:prompt + optionsJSON:optionsJSON + completion:^(NSString *text, NSString *error) { + if (error) + mob_native_text_send_error(pid, "foundation_models", "generate_text", + error); + else + mob_native_text_send_success(pid, "foundation_models", + "generated_text", text); + }]; return enif_make_atom(env, "ok"); } -static ERL_NIF_TERM nif_ai_recognize_text(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - NSString *path = mob_ai_string_arg(env, argv[0]); - NSString *optionsJSON = mob_ai_string_arg(env, argv[1]); +static ERL_NIF_TERM nif_vision_recognize_text(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + // Apple Vision VNRecognizeTextRequest: + // https://developer.apple.com/documentation/vision/vnrecognizetextrequest + NSString *path = mob_native_text_string_arg(env, argv[0]); + NSString *optionsJSON = mob_native_text_string_arg(env, argv[1]); if (!path || !optionsJSON) return enif_make_badarg(env); ErlNifPid pid; enif_self(env, &pid); - [MobAI recognizeTextAtPath:path - optionsJSON:optionsJSON - completion:^(NSString *text, NSString *error) { - if (error) - mob_ai_send_error(pid, "recognize_text", error); - else - mob_ai_send_success(pid, "recognized_text", text); - }]; + [MobVision recognizeTextAtPath:path + optionsJSON:optionsJSON + completion:^(NSString *text, NSString *error) { + if (error) + mob_native_text_send_error(pid, "vision", "recognize_text", error); + else + mob_native_text_send_success(pid, "vision", "recognized_text", text); + }]; return enif_make_atom(env, "ok"); } -static ERL_NIF_TERM nif_ai_transcribe_audio(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - NSString *path = mob_ai_string_arg(env, argv[0]); - NSString *optionsJSON = mob_ai_string_arg(env, argv[1]); +static ERL_NIF_TERM nif_speech_transcribe_audio(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + // Apple Speech SFSpeechURLRecognitionRequest: + // https://developer.apple.com/documentation/speech/sfspeechurlrecognitionrequest + NSString *path = mob_native_text_string_arg(env, argv[0]); + NSString *optionsJSON = mob_native_text_string_arg(env, argv[1]); if (!path || !optionsJSON) return enif_make_badarg(env); ErlNifPid pid; enif_self(env, &pid); - [MobAI transcribeAudioAtPath:path - optionsJSON:optionsJSON - completion:^(NSString *text, NSString *error) { - if (error) - mob_ai_send_error(pid, "transcribe_audio", error); - else - mob_ai_send_success(pid, "transcribed_audio", text); - }]; + [MobSpeech transcribeAudioAtPath:path + optionsJSON:optionsJSON + completion:^(NSString *text, NSString *error) { + if (error) + mob_native_text_send_error(pid, "speech", "transcribe_audio", error); + else + mob_native_text_send_success(pid, "speech", "transcribed_audio", text); + }]; return enif_make_atom(env, "ok"); } @@ -5895,9 +5905,9 @@ static ERL_NIF_TERM nif_vendor_usb_close(ErlNifEnv *env, int argc, const ERL_NIF {"audio_play", 2, nif_audio_play, 0}, {"audio_stop_playback", 0, nif_audio_stop_playback, 0}, {"audio_set_volume", 1, nif_audio_set_volume, 0}, - {"ai_generate_text", 2, nif_ai_generate_text, 0}, - {"ai_recognize_text", 2, nif_ai_recognize_text, 0}, - {"ai_transcribe_audio", 2, nif_ai_transcribe_audio, 0}, + {"foundation_models_generate_text", 2, nif_foundation_models_generate_text, 0}, + {"vision_recognize_text", 2, nif_vision_recognize_text, 0}, + {"speech_transcribe_audio", 2, nif_speech_transcribe_audio, 0}, {"motion_start", 2, nif_motion_start, 0}, {"motion_stop", 0, nif_motion_stop, 0}, {"scanner_scan", 1, nif_scanner_scan, 0}, diff --git a/lib/mob/ai.ex b/lib/mob/ai.ex deleted file mode 100644 index 99390bfb..00000000 --- a/lib/mob/ai.ex +++ /dev/null @@ -1,137 +0,0 @@ -defmodule Mob.AI do - @moduledoc """ - Native AI capabilities exposed by the platform. - - These calls are asynchronous. The native side sends results back to the - calling process as `{:ai, event, payload}` messages. - - ## Text generation - - Mob.AI.generate_text(socket, "Summarize this note") - # -> handle_info({:ai, :generated_text, %{text: text}}, socket) - # -> handle_info({:ai, :error, %{operation: :generate_text, reason: reason}}, socket) - - On iOS this uses Apple's Foundation Models framework when available. The iOS - simulator does not provide the on-device system language model. - - ## Vision OCR - - Mob.AI.recognize_text(socket, "/path/to/image.png") - # -> handle_info({:ai, :recognized_text, %{text: text}}, socket) - - ## Speech transcription - - Mob.AI.transcribe_audio(socket, "/path/to/audio.m4a") - # -> handle_info({:ai, :transcribed_audio, %{text: text}}, socket) - - Speech transcription requires speech recognition permission on iOS. - """ - - @type generate_option :: - {:instructions, String.t()} - | {:temperature, number()} - | {:maximum_response_tokens, pos_integer()} - - @type ocr_option :: - {:recognition_level, :fast | :accurate} - | {:uses_language_correction, boolean()} - - @type speech_option :: - {:locale, String.t()} - | {:requires_on_device_recognition, boolean()} - - @doc """ - Generate text using the platform language model. - - Result arrives as: - - * `{:ai, :generated_text, %{text: text}}` - * `{:ai, :error, %{operation: :generate_text, reason: reason}}` - """ - @spec generate_text(Mob.Socket.t(), String.t(), [generate_option()]) :: Mob.Socket.t() - def generate_text(socket, prompt, opts \\ []) when is_binary(prompt) and is_list(opts) do - invoke( - :mob_nif.ai_generate_text(prompt, :json.encode(generate_text_opts(opts))), - :generate_text - ) - - socket - end - - @doc false - @spec generate_text_opts(keyword()) :: %{String.t() => term()} - def generate_text_opts(opts) do - %{ - "instructions" => Keyword.get(opts, :instructions, ""), - "temperature" => Keyword.get(opts, :temperature, 0.2) / 1, - "maximum_response_tokens" => Keyword.get(opts, :maximum_response_tokens, 256) - } - end - - @doc """ - Recognize text in an image file using platform OCR. - - Result arrives as: - - * `{:ai, :recognized_text, %{text: text}}` - * `{:ai, :error, %{operation: :recognize_text, reason: reason}}` - """ - @spec recognize_text(Mob.Socket.t(), String.t(), [ocr_option()]) :: Mob.Socket.t() - def recognize_text(socket, path, opts \\ []) when is_binary(path) and is_list(opts) do - invoke( - :mob_nif.ai_recognize_text(path, :json.encode(recognize_text_opts(opts))), - :recognize_text - ) - - socket - end - - @doc false - @spec recognize_text_opts(keyword()) :: %{String.t() => term()} - def recognize_text_opts(opts) do - %{ - "recognition_level" => Keyword.get(opts, :recognition_level, :accurate) |> Atom.to_string(), - "uses_language_correction" => Keyword.get(opts, :uses_language_correction, true) - } - end - - @doc """ - Transcribe an audio file using platform speech recognition. - - Result arrives as: - - * `{:ai, :transcribed_audio, %{text: text}}` - * `{:ai, :error, %{operation: :transcribe_audio, reason: reason}}` - """ - @spec transcribe_audio(Mob.Socket.t(), String.t(), [speech_option()]) :: Mob.Socket.t() - def transcribe_audio(socket, path, opts \\ []) when is_binary(path) and is_list(opts) do - invoke( - :mob_nif.ai_transcribe_audio(path, :json.encode(transcribe_audio_opts(opts))), - :transcribe_audio - ) - - socket - end - - @doc false - @spec transcribe_audio_opts(keyword()) :: %{String.t() => term()} - def transcribe_audio_opts(opts) do - %{ - "locale" => Keyword.get(opts, :locale, ""), - "requires_on_device_recognition" => - Keyword.get(opts, :requires_on_device_recognition, false) - } - end - - defp invoke(:ok, _operation), do: :ok - - defp invoke(:unsupported, operation) do - send( - self(), - {:ai, :error, - %{operation: operation, reason: "Native AI is not supported on this platform."}} - ) - - :ok - end -end diff --git a/lib/mob/foundation_models.ex b/lib/mob/foundation_models.ex new file mode 100644 index 00000000..52e99112 --- /dev/null +++ b/lib/mob/foundation_models.ex @@ -0,0 +1,66 @@ +defmodule Mob.FoundationModels do + @moduledoc """ + iOS Foundation Models text generation. + + This wraps Apple's Foundation Models framework, specifically + `SystemLanguageModel`, `LanguageModelSession`, and `GenerationOptions`. + See Apple's guide: + https://developer.apple.com/documentation/foundationmodels/adding-intelligent-app-features-with-generative-models + + Calls are asynchronous. Results are delivered to the calling process: + + Mob.FoundationModels.generate_text(socket, "Summarize this note") + + def handle_info({:foundation_models, :generated_text, %{text: text}}, socket) do + {:noreply, Mob.Socket.assign(socket, :summary, text)} + end + + def handle_info({:foundation_models, :error, %{operation: :generate_text, reason: reason}}, socket) do + {:noreply, Mob.Socket.assign(socket, :error, reason)} + end + + Foundation Models requires an eligible physical iOS device with Apple + Intelligence enabled. The iOS simulator does not provide the on-device system + language model. + """ + + @type generate_option :: + {:instructions, String.t()} + | {:temperature, number()} + | {:maximum_response_tokens, pos_integer()} + + @doc """ + Generate text using Apple's on-device Foundation Models framework. + """ + @spec generate_text(Mob.Socket.t(), String.t(), [generate_option()]) :: Mob.Socket.t() + def generate_text(socket, prompt, opts \\ []) when is_binary(prompt) and is_list(opts) do + invoke( + :mob_nif.foundation_models_generate_text(prompt, :json.encode(generate_text_opts(opts))), + :generate_text + ) + + socket + end + + @doc false + @spec generate_text_opts(keyword()) :: %{String.t() => term()} + def generate_text_opts(opts) do + %{ + "instructions" => Keyword.get(opts, :instructions, ""), + "temperature" => Keyword.get(opts, :temperature, 0.2) / 1, + "maximum_response_tokens" => Keyword.get(opts, :maximum_response_tokens, 256) + } + end + + defp invoke(:ok, _operation), do: :ok + + defp invoke(:unsupported, operation) do + send( + self(), + {:foundation_models, :error, + %{operation: operation, reason: "Foundation Models is not supported on this platform."}} + ) + + :ok + end +end diff --git a/lib/mob/speech.ex b/lib/mob/speech.ex new file mode 100644 index 00000000..f4cde0ae --- /dev/null +++ b/lib/mob/speech.ex @@ -0,0 +1,63 @@ +defmodule Mob.Speech do + @moduledoc """ + iOS Speech framework transcription. + + File transcription wraps Apple's `SFSpeechRecognizer` and + `SFSpeechURLRecognitionRequest`. + See Apple's API documentation: + https://developer.apple.com/documentation/speech/sfspeechrecognizer + https://developer.apple.com/documentation/speech/sfspeechurlrecognitionrequest + + Calls are asynchronous. Results are delivered to the calling process: + + Mob.Speech.transcribe_audio(socket, "/path/to/audio.m4a") + + def handle_info({:speech, :transcribed_audio, %{text: text}}, socket) do + {:noreply, Mob.Socket.assign(socket, :transcript, text)} + end + + On iOS, speech transcription requires speech recognition authorization. Audio + recording still uses `Mob.Audio`; this module only transcribes an existing + audio file. + """ + + @type transcribe_audio_option :: + {:locale, String.t()} + | {:requires_on_device_recognition, boolean()} + + @doc """ + Transcribe an audio file using Apple's Speech framework. + """ + @spec transcribe_audio(Mob.Socket.t(), String.t(), [transcribe_audio_option()]) :: + Mob.Socket.t() + def transcribe_audio(socket, path, opts \\ []) when is_binary(path) and is_list(opts) do + invoke( + :mob_nif.speech_transcribe_audio(path, :json.encode(transcribe_audio_opts(opts))), + :transcribe_audio + ) + + socket + end + + @doc false + @spec transcribe_audio_opts(keyword()) :: %{String.t() => term()} + def transcribe_audio_opts(opts) do + %{ + "locale" => Keyword.get(opts, :locale, ""), + "requires_on_device_recognition" => + Keyword.get(opts, :requires_on_device_recognition, false) + } + end + + defp invoke(:ok, _operation), do: :ok + + defp invoke(:unsupported, operation) do + send( + self(), + {:speech, :error, + %{operation: operation, reason: "Speech transcription is not supported on this platform."}} + ) + + :ok + end +end diff --git a/lib/mob/vision.ex b/lib/mob/vision.ex new file mode 100644 index 00000000..ad7156dd --- /dev/null +++ b/lib/mob/vision.ex @@ -0,0 +1,58 @@ +defmodule Mob.Vision do + @moduledoc """ + iOS Vision framework capabilities. + + Text recognition wraps Apple's `VNRecognizeTextRequest`. + See Apple's API documentation: + https://developer.apple.com/documentation/vision/vnrecognizetextrequest + + Calls are asynchronous. Results are delivered to the calling process: + + Mob.Vision.recognize_text(socket, "/path/to/image.png") + + def handle_info({:vision, :recognized_text, %{text: text}}, socket) do + {:noreply, Mob.Socket.assign(socket, :ocr_text, text)} + end + """ + + @type recognize_text_option :: + {:recognition_level, :fast | :accurate} + | {:uses_language_correction, boolean()} + + @doc """ + Recognize text in an image file using Apple's Vision framework. + """ + @spec recognize_text(Mob.Socket.t(), String.t(), [recognize_text_option()]) :: Mob.Socket.t() + def recognize_text(socket, path, opts \\ []) when is_binary(path) and is_list(opts) do + invoke( + :mob_nif.vision_recognize_text(path, :json.encode(recognize_text_opts(opts))), + :recognize_text + ) + + socket + end + + @doc false + @spec recognize_text_opts(keyword()) :: %{String.t() => term()} + def recognize_text_opts(opts) do + %{ + "recognition_level" => Keyword.get(opts, :recognition_level, :accurate) |> Atom.to_string(), + "uses_language_correction" => Keyword.get(opts, :uses_language_correction, true) + } + end + + defp invoke(:ok, _operation), do: :ok + + defp invoke(:unsupported, operation) do + send( + self(), + {:vision, :error, + %{ + operation: operation, + reason: "Vision text recognition is not supported on this platform." + }} + ) + + :ok + end +end diff --git a/mix.exs b/mix.exs index 24939d65..b80a4859 100644 --- a/mix.exs +++ b/mix.exs @@ -75,6 +75,7 @@ defmodule Mob.MixProject do "guides/theming.md": [title: "Theming"], "guides/navigation.md": [title: "Navigation"], "guides/device_capabilities.md": [title: "Device Capabilities"], + "guides/native_intelligence.md": [title: "Native Intelligence APIs"], "guides/native_extensions.md": [title: "Native Extensions (NIFs, features)"], "guides/dns_on_ios.md": [title: "DNS on iOS"], "guides/push_notifications.md": [title: "Push Notifications"], @@ -111,6 +112,9 @@ defmodule Mob.MixProject do Mob.Photos, Mob.Files, Mob.Audio, + Mob.FoundationModels, + Mob.Vision, + Mob.Speech, Mob.Motion, Mob.Scanner, Mob.Notify diff --git a/src/mob_nif.erl b/src/mob_nif.erl index fecd66fa..216ea254 100644 --- a/src/mob_nif.erl +++ b/src/mob_nif.erl @@ -41,10 +41,12 @@ audio_play/2, audio_stop_playback/0, audio_set_volume/1, - %% Native AI - ai_generate_text/2, - ai_recognize_text/2, - ai_transcribe_audio/2, + %% Foundation Models + foundation_models_generate_text/2, + %% Vision + vision_recognize_text/2, + %% Speech + speech_transcribe_audio/2, %% Motion sensors motion_start/2, motion_stop/0, @@ -144,9 +146,9 @@ audio_play/2, audio_stop_playback/0, audio_set_volume/1, - ai_generate_text/2, - ai_recognize_text/2, - ai_transcribe_audio/2, + foundation_models_generate_text/2, + vision_recognize_text/2, + speech_transcribe_audio/2, motion_start/2, motion_stop/0, scanner_scan/1, @@ -244,9 +246,9 @@ audio_stop_recording() -> erlang:nif_error(not_loaded). audio_play(_Path, _OptsJson) -> erlang:nif_error(not_loaded). audio_stop_playback() -> erlang:nif_error(not_loaded). audio_set_volume(_Volume) -> erlang:nif_error(not_loaded). -ai_generate_text(_Prompt, _OptsJson) -> erlang:nif_error(not_loaded). -ai_recognize_text(_Path, _OptsJson) -> erlang:nif_error(not_loaded). -ai_transcribe_audio(_Path, _OptsJson) -> erlang:nif_error(not_loaded). +foundation_models_generate_text(_Prompt, _OptsJson) -> erlang:nif_error(not_loaded). +vision_recognize_text(_Path, _OptsJson) -> erlang:nif_error(not_loaded). +speech_transcribe_audio(_Path, _OptsJson) -> erlang:nif_error(not_loaded). motion_start(_Sensors, _Interval) -> erlang:nif_error(not_loaded). motion_stop() -> erlang:nif_error(not_loaded). scanner_scan(_FormatsJson) -> erlang:nif_error(not_loaded). diff --git a/test/mob/ai_test.exs b/test/mob/ai_test.exs deleted file mode 100644 index fb6d9411..00000000 --- a/test/mob/ai_test.exs +++ /dev/null @@ -1,59 +0,0 @@ -defmodule Mob.AITest do - use ExUnit.Case, async: true - - alias Mob.AI - - describe "generate_text_opts/1" do - test "defaults are string keyed" do - assert AI.generate_text_opts([]) == %{ - "instructions" => "", - "temperature" => 0.2, - "maximum_response_tokens" => 256 - } - end - - test "custom values are passed through" do - assert AI.generate_text_opts( - instructions: "Be concise", - temperature: 0.7, - maximum_response_tokens: 64 - ) == %{ - "instructions" => "Be concise", - "temperature" => 0.7, - "maximum_response_tokens" => 64 - } - end - end - - describe "recognize_text_opts/1" do - test "defaults to accurate OCR with language correction" do - assert AI.recognize_text_opts([]) == %{ - "recognition_level" => "accurate", - "uses_language_correction" => true - } - end - - test "recognition level is encoded as a string" do - assert AI.recognize_text_opts(recognition_level: :fast) == %{ - "recognition_level" => "fast", - "uses_language_correction" => true - } - end - end - - describe "transcribe_audio_opts/1" do - test "defaults to platform locale and server-capable recognition" do - assert AI.transcribe_audio_opts([]) == %{ - "locale" => "", - "requires_on_device_recognition" => false - } - end - - test "custom locale and on-device setting are passed through" do - assert AI.transcribe_audio_opts(locale: "en-US", requires_on_device_recognition: true) == %{ - "locale" => "en-US", - "requires_on_device_recognition" => true - } - end - end -end diff --git a/test/mob/foundation_models_test.exs b/test/mob/foundation_models_test.exs new file mode 100644 index 00000000..85396fe3 --- /dev/null +++ b/test/mob/foundation_models_test.exs @@ -0,0 +1,27 @@ +defmodule Mob.FoundationModelsTest do + use ExUnit.Case, async: true + + alias Mob.FoundationModels + + describe "generate_text_opts/1" do + test "defaults are string keyed" do + assert FoundationModels.generate_text_opts([]) == %{ + "instructions" => "", + "temperature" => 0.2, + "maximum_response_tokens" => 256 + } + end + + test "custom values are passed through" do + assert FoundationModels.generate_text_opts( + instructions: "Be concise", + temperature: 0.7, + maximum_response_tokens: 64 + ) == %{ + "instructions" => "Be concise", + "temperature" => 0.7, + "maximum_response_tokens" => 64 + } + end + end +end diff --git a/test/mob/speech_test.exs b/test/mob/speech_test.exs new file mode 100644 index 00000000..17aa5a5d --- /dev/null +++ b/test/mob/speech_test.exs @@ -0,0 +1,22 @@ +defmodule Mob.SpeechTest do + use ExUnit.Case, async: true + + alias Mob.Speech + + describe "transcribe_audio_opts/1" do + test "defaults to platform locale and server-capable recognition" do + assert Speech.transcribe_audio_opts([]) == %{ + "locale" => "", + "requires_on_device_recognition" => false + } + end + + test "custom locale and on-device setting are passed through" do + assert Speech.transcribe_audio_opts(locale: "en-US", requires_on_device_recognition: true) == + %{ + "locale" => "en-US", + "requires_on_device_recognition" => true + } + end + end +end diff --git a/test/mob/vision_test.exs b/test/mob/vision_test.exs new file mode 100644 index 00000000..8ed5a266 --- /dev/null +++ b/test/mob/vision_test.exs @@ -0,0 +1,21 @@ +defmodule Mob.VisionTest do + use ExUnit.Case, async: true + + alias Mob.Vision + + describe "recognize_text_opts/1" do + test "defaults to accurate OCR with language correction" do + assert Vision.recognize_text_opts([]) == %{ + "recognition_level" => "accurate", + "uses_language_correction" => true + } + end + + test "recognition level is encoded as a string" do + assert Vision.recognize_text_opts(recognition_level: :fast) == %{ + "recognition_level" => "fast", + "uses_language_correction" => true + } + end + end +end From d174b02b0cd307ff228f2a74c2333ecec4803b01 Mon Sep 17 00:00:00 2001 From: Yurko Hoshko Date: Sat, 16 May 2026 22:53:12 -0700 Subject: [PATCH 3/6] Refine iOS native intelligence APIs --- README.md | 2 +- guides/device_capabilities.md | 52 ++++++++--- guides/native_intelligence.md | 88 ------------------- lib/mob/{ => ios}/foundation_models.ex | 6 +- lib/mob/{ => ios}/speech.ex | 4 +- lib/mob/{ => ios}/vision.ex | 4 +- mix.exs | 7 +- test/mob/{ => ios}/foundation_models_test.exs | 4 +- test/mob/{ => ios}/speech_test.exs | 4 +- test/mob/{ => ios}/vision_test.exs | 4 +- 10 files changed, 58 insertions(+), 117 deletions(-) delete mode 100644 guides/native_intelligence.md rename lib/mob/{ => ios}/foundation_models.ex (90%) rename lib/mob/{ => ios}/speech.ex (95%) rename lib/mob/{ => ios}/vision.ex (94%) rename test/mob/{ => ios}/foundation_models_test.exs (90%) rename test/mob/{ => ios}/speech_test.exs (91%) rename test/mob/{ => ios}/vision_test.exs (91%) diff --git a/README.md b/README.md index f4eb1e73..f8da86be 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,7 @@ Mob.Notify.register_push(socket) def handle_info({:push_token, :ios, token}, socket), do: ... ``` -Also: `Mob.Clipboard`, `Mob.Share`, `Mob.Photos`, `Mob.Files`, `Mob.Audio`, `Mob.FoundationModels`, `Mob.Vision`, `Mob.Speech`, `Mob.Motion`, `Mob.Biometric`, `Mob.Scanner`, `Mob.Permissions`. +Also: `Mob.Clipboard`, `Mob.Share`, `Mob.Photos`, `Mob.Files`, `Mob.Audio`, `Mob.IOS.FoundationModels`, `Mob.IOS.Vision`, `Mob.IOS.Speech`, `Mob.Motion`, `Mob.Biometric`, `Mob.Scanner`, `Mob.Permissions`. ## What's in the box diff --git a/guides/device_capabilities.md b/guides/device_capabilities.md index 180b8669..582ed1e2 100644 --- a/guides/device_capabilities.md +++ b/guides/device_capabilities.md @@ -205,9 +205,9 @@ end iOS uses `AVAudioPlayer` / `AVPlayer`. Android uses `MediaPlayer`. -## Foundation Models +## iOS Foundation Models -iOS only. Generates text with Apple's on-device Foundation Models framework. +Generates text with Apple's on-device Foundation Models framework. Requires an eligible physical device with Apple Intelligence enabled; the iOS simulator reports this capability as unavailable. @@ -217,7 +217,7 @@ Apple docs: ```elixir socket = - Mob.FoundationModels.generate_text(socket, "Turn this note into a short action list", + Mob.IOS.FoundationModels.generate_text(socket, "Turn this note into a short action list", instructions: "Return compact plain text.", temperature: 0.2, maximum_response_tokens: 240 @@ -232,17 +232,17 @@ def handle_info({:foundation_models, :error, %{reason: reason}}, socket) do end ``` -## Vision text recognition +## iOS Vision text recognition -iOS only for now. Recognizes text in a local image file with Apple's Vision -framework. Combine with `Mob.Photos.pick/2` to OCR a user-selected image. +Recognizes text in a local image file with Apple's Vision framework. Combine +with `Mob.Photos.pick/2` to OCR a user-selected image. Apple docs: [VNRecognizeTextRequest](https://developer.apple.com/documentation/vision/vnrecognizetextrequest). ```elixir socket = - Mob.Vision.recognize_text(socket, image_path, + Mob.IOS.Vision.recognize_text(socket, image_path, recognition_level: :accurate, uses_language_correction: true ) @@ -252,10 +252,10 @@ def handle_info({:vision, :recognized_text, %{text: text}}, socket) do end ``` -## Speech transcription +## iOS Speech transcription -iOS only for now. Transcribes an existing audio file with Apple's Speech -framework. Use `Mob.Audio` to record microphone input first. +Transcribes an existing audio file with Apple's Speech framework. Use +`Mob.Audio` to record microphone input first. Apple docs: [SFSpeechRecognizer](https://developer.apple.com/documentation/speech/sfspeechrecognizer) and @@ -263,7 +263,7 @@ Apple docs: ```elixir socket = - Mob.Speech.transcribe_audio(socket, recording_path, + Mob.IOS.Speech.transcribe_audio(socket, recording_path, locale: "en-US", requires_on_device_recognition: false ) @@ -277,6 +277,36 @@ Speech recognition requires iOS speech authorization. If `requires_on_device_recognition` is true, iOS may reject locales that do not support local recognition. +### iOS native intelligence testing + +| Capability | iOS simulator | Physical iPhone | +|---|---:|---:| +| `Mob.IOS.FoundationModels.generate_text/3` | No. The simulator does not provide the on-device system language model. | Yes, on Apple Intelligence-capable devices with Apple Intelligence enabled and the model ready. | +| `Mob.IOS.Vision.recognize_text/3` | Yes. Pass a readable image path in the simulator app container or pick a simulator photo. | Yes. | +| `Mob.IOS.Speech.transcribe_audio/3` | Usually yes for file transcription, subject to simulator Speech authorization and runtime locale/service availability. | Yes, subject to Speech authorization and locale support. | + +The lightest simulator smoke test is: + +1. Build a Mob app that exposes a screen with a text field for an image path and + calls `Mob.IOS.Vision.recognize_text/3`. +2. Copy an image with readable text into the simulator app's Documents + directory, then run OCR against that path. +3. Record audio with `Mob.Audio.start_recording/2` and pass the resulting path + to `Mob.IOS.Speech.transcribe_audio/3`. +4. Confirm Foundation Models returns the expected simulator-unavailable error. + +### Scope and follow-up ideas + +This first bridge includes plain Foundation Models text generation, Vision OCR +from a local image path, and Speech transcription from a local audio file. + +Not included yet: Foundation Models structured generation with `@Generable`, +streaming partial Foundation Models responses, tool calling, multi-turn session +persistence, Vision requests beyond text recognition, live Speech recognition, +custom Speech language models, Natural Language framework features, image +generation, Private Cloud Compute-backed server features, and Android ML Kit or +platform-equivalent implementations. + ## Location Requires `:location` permission. diff --git a/guides/native_intelligence.md b/guides/native_intelligence.md deleted file mode 100644 index 6529fc54..00000000 --- a/guides/native_intelligence.md +++ /dev/null @@ -1,88 +0,0 @@ -# Native Intelligence APIs - -Mob exposes a small, iOS-first bridge to Apple-native intelligence APIs: - -- `Mob.FoundationModels` for Foundation Models text generation. -- `Mob.Vision` for Vision text recognition. -- `Mob.Speech` for Speech framework file transcription. - -These APIs deliberately mirror Apple's framework boundaries instead of grouping -everything under a generic "AI" namespace. That keeps the Elixir surface close -to the native SDK names and leaves room for platform-specific capabilities to -grow without a catch-all module. - -Apple references: - -- [Foundation Models](https://developer.apple.com/documentation/foundationmodels) -- [Adding intelligent app features with generative models](https://developer.apple.com/documentation/foundationmodels/adding-intelligent-app-features-with-generative-models) -- [VNRecognizeTextRequest](https://developer.apple.com/documentation/vision/vnrecognizetextrequest) -- [SFSpeechRecognizer](https://developer.apple.com/documentation/speech/sfspeechrecognizer) -- [SFSpeechURLRecognitionRequest](https://developer.apple.com/documentation/speech/sfspeechurlrecognitionrequest) - -## Example Flow - -There is no checked-in sample app in the `mob` repository today. A complete app -can compose existing Mob features with the new native modules: - -```elixir -def handle_info({:photos, :picked, [%{path: path} | _]}, socket) do - {:noreply, Mob.Vision.recognize_text(socket, path)} -end - -def handle_info({:vision, :recognized_text, %{text: text}}, socket) do - prompt = "Summarize this OCR text as actions:\n\n#{text}" - {:noreply, Mob.FoundationModels.generate_text(socket, prompt)} -end - -def handle_info({:foundation_models, :generated_text, %{text: text}}, socket) do - {:noreply, Mob.Socket.assign(socket, :result, text)} -end -``` - -Speech uses the same pattern with `Mob.Audio`: - -```elixir -def handle_info({:audio, :recorded, %{path: path}}, socket) do - {:noreply, Mob.Speech.transcribe_audio(socket, path)} -end -``` - -## Current Scope - -Included: - -- Foundation Models plain text generation. -- Vision OCR from a local image path. -- Speech transcription from a local audio file. -- Android stubs that return `:unsupported` so apps can branch cleanly. - -Out of scope for this first bridge: - -- Foundation Models structured generation with `@Generable`. -- Streaming partial Foundation Models responses. -- Tool calling, multi-turn session persistence, or model transcript management. -- Vision requests beyond text recognition, such as barcode, face, object, and - document detection. -- Speech live microphone recognition, partial transcripts, custom language - models, and keyword spotting. -- Natural Language framework features such as language identification, - sentiment, tokenization, and embedding/classification APIs. -- Image generation or Private Cloud Compute-backed server features. -- Android ML Kit or platform-equivalent implementations. - -## Operational Notes - -Foundation Models is not available in the iOS simulator. On device it can still -be unavailable if Apple Intelligence is disabled, the device is not eligible, -the current locale is unsupported, or the model is not ready. - -Vision OCR needs a readable local file path. Photo-picker temporary files should -be copied if the app needs to keep them beyond the current workflow. - -Speech transcription requires iOS speech-recognition authorization. On-device -recognition is locale-dependent; setting `requires_on_device_recognition: true` -can make otherwise valid transcriptions fail. - -All three APIs send results back to the calling screen process. Treat them like -other Mob asynchronous device APIs: update screen state in `handle_info/2`, and -keep long-running UX cancellable at the app level. diff --git a/lib/mob/foundation_models.ex b/lib/mob/ios/foundation_models.ex similarity index 90% rename from lib/mob/foundation_models.ex rename to lib/mob/ios/foundation_models.ex index 52e99112..bcdd16c0 100644 --- a/lib/mob/foundation_models.ex +++ b/lib/mob/ios/foundation_models.ex @@ -1,4 +1,4 @@ -defmodule Mob.FoundationModels do +defmodule Mob.IOS.FoundationModels do @moduledoc """ iOS Foundation Models text generation. @@ -9,13 +9,13 @@ defmodule Mob.FoundationModels do Calls are asynchronous. Results are delivered to the calling process: - Mob.FoundationModels.generate_text(socket, "Summarize this note") + Mob.IOS.FoundationModels.generate_text(socket, "Summarize this note") def handle_info({:foundation_models, :generated_text, %{text: text}}, socket) do {:noreply, Mob.Socket.assign(socket, :summary, text)} end - def handle_info({:foundation_models, :error, %{operation: :generate_text, reason: reason}}, socket) do + def handle_info({:foundation_models, :error, %{reason: reason}}, socket) do {:noreply, Mob.Socket.assign(socket, :error, reason)} end diff --git a/lib/mob/speech.ex b/lib/mob/ios/speech.ex similarity index 95% rename from lib/mob/speech.ex rename to lib/mob/ios/speech.ex index f4cde0ae..9d5cc2f1 100644 --- a/lib/mob/speech.ex +++ b/lib/mob/ios/speech.ex @@ -1,4 +1,4 @@ -defmodule Mob.Speech do +defmodule Mob.IOS.Speech do @moduledoc """ iOS Speech framework transcription. @@ -10,7 +10,7 @@ defmodule Mob.Speech do Calls are asynchronous. Results are delivered to the calling process: - Mob.Speech.transcribe_audio(socket, "/path/to/audio.m4a") + Mob.IOS.Speech.transcribe_audio(socket, "/path/to/audio.m4a") def handle_info({:speech, :transcribed_audio, %{text: text}}, socket) do {:noreply, Mob.Socket.assign(socket, :transcript, text)} diff --git a/lib/mob/vision.ex b/lib/mob/ios/vision.ex similarity index 94% rename from lib/mob/vision.ex rename to lib/mob/ios/vision.ex index ad7156dd..00cc29cd 100644 --- a/lib/mob/vision.ex +++ b/lib/mob/ios/vision.ex @@ -1,4 +1,4 @@ -defmodule Mob.Vision do +defmodule Mob.IOS.Vision do @moduledoc """ iOS Vision framework capabilities. @@ -8,7 +8,7 @@ defmodule Mob.Vision do Calls are asynchronous. Results are delivered to the calling process: - Mob.Vision.recognize_text(socket, "/path/to/image.png") + Mob.IOS.Vision.recognize_text(socket, "/path/to/image.png") def handle_info({:vision, :recognized_text, %{text: text}}, socket) do {:noreply, Mob.Socket.assign(socket, :ocr_text, text)} diff --git a/mix.exs b/mix.exs index b80a4859..cc3aa31d 100644 --- a/mix.exs +++ b/mix.exs @@ -75,7 +75,6 @@ defmodule Mob.MixProject do "guides/theming.md": [title: "Theming"], "guides/navigation.md": [title: "Navigation"], "guides/device_capabilities.md": [title: "Device Capabilities"], - "guides/native_intelligence.md": [title: "Native Intelligence APIs"], "guides/native_extensions.md": [title: "Native Extensions (NIFs, features)"], "guides/dns_on_ios.md": [title: "DNS on iOS"], "guides/push_notifications.md": [title: "Push Notifications"], @@ -112,9 +111,9 @@ defmodule Mob.MixProject do Mob.Photos, Mob.Files, Mob.Audio, - Mob.FoundationModels, - Mob.Vision, - Mob.Speech, + Mob.IOS.FoundationModels, + Mob.IOS.Vision, + Mob.IOS.Speech, Mob.Motion, Mob.Scanner, Mob.Notify diff --git a/test/mob/foundation_models_test.exs b/test/mob/ios/foundation_models_test.exs similarity index 90% rename from test/mob/foundation_models_test.exs rename to test/mob/ios/foundation_models_test.exs index 85396fe3..3f2638c0 100644 --- a/test/mob/foundation_models_test.exs +++ b/test/mob/ios/foundation_models_test.exs @@ -1,7 +1,7 @@ -defmodule Mob.FoundationModelsTest do +defmodule Mob.IOS.FoundationModelsTest do use ExUnit.Case, async: true - alias Mob.FoundationModels + alias Mob.IOS.FoundationModels describe "generate_text_opts/1" do test "defaults are string keyed" do diff --git a/test/mob/speech_test.exs b/test/mob/ios/speech_test.exs similarity index 91% rename from test/mob/speech_test.exs rename to test/mob/ios/speech_test.exs index 17aa5a5d..c009428f 100644 --- a/test/mob/speech_test.exs +++ b/test/mob/ios/speech_test.exs @@ -1,7 +1,7 @@ -defmodule Mob.SpeechTest do +defmodule Mob.IOS.SpeechTest do use ExUnit.Case, async: true - alias Mob.Speech + alias Mob.IOS.Speech describe "transcribe_audio_opts/1" do test "defaults to platform locale and server-capable recognition" do diff --git a/test/mob/vision_test.exs b/test/mob/ios/vision_test.exs similarity index 91% rename from test/mob/vision_test.exs rename to test/mob/ios/vision_test.exs index 8ed5a266..2af04917 100644 --- a/test/mob/vision_test.exs +++ b/test/mob/ios/vision_test.exs @@ -1,7 +1,7 @@ -defmodule Mob.VisionTest do +defmodule Mob.IOS.VisionTest do use ExUnit.Case, async: true - alias Mob.Vision + alias Mob.IOS.Vision describe "recognize_text_opts/1" do test "defaults to accurate OCR with language correction" do From 37fd8076bc81e432c490534a38a3b0b74855c631 Mon Sep 17 00:00:00 2001 From: Yurko Hoshko Date: Sat, 16 May 2026 23:00:26 -0700 Subject: [PATCH 4/6] Document simulator smoke testing --- guides/device_capabilities.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/guides/device_capabilities.md b/guides/device_capabilities.md index 582ed1e2..d679abf6 100644 --- a/guides/device_capabilities.md +++ b/guides/device_capabilities.md @@ -295,6 +295,34 @@ The lightest simulator smoke test is: to `Mob.IOS.Speech.transcribe_audio/3`. 4. Confirm Foundation Models returns the expected simulator-unavailable error. +For a simulator app that is already installed, the useful setup commands are: + +```sh +SIM_ID="booted" +BUNDLE_ID="com.example.my_mob_app" +CONTAINER="$(xcrun simctl get_app_container "$SIM_ID" "$BUNDLE_ID" data)" + +cp ./ocr_fixture.png "$CONTAINER/Documents/ocr_fixture.png" +``` + +Then pass the copied path to the app: + +```elixir +image_path = "/path/from/xcrun/simctl/get_app_container/Documents/ocr_fixture.png" +Mob.IOS.Vision.recognize_text(socket, image_path) +``` + +For Speech, recording inside the Mob app is the most representative smoke test: + +```elixir +socket = Mob.Audio.start_recording(socket) +socket = Mob.Audio.stop_recording(socket) + +def handle_info({:audio, :recorded, %{path: path}}, socket) do + {:noreply, Mob.IOS.Speech.transcribe_audio(socket, path)} +end +``` + ### Scope and follow-up ideas This first bridge includes plain Foundation Models text generation, Vision OCR From 841740a1083818ee9c37af152ffa78bfe3a93744 Mon Sep 17 00:00:00 2001 From: Yurko Hoshko Date: Sat, 16 May 2026 23:59:07 -0700 Subject: [PATCH 5/6] Document picker-based AI smoke tests --- guides/device_capabilities.md | 49 +++++++++++++++++++++++------------ lib/mob/ios/speech.ex | 10 +++++-- lib/mob/ios/vision.ex | 10 +++++-- 3 files changed, 49 insertions(+), 20 deletions(-) diff --git a/guides/device_capabilities.md b/guides/device_capabilities.md index d679abf6..13931267 100644 --- a/guides/device_capabilities.md +++ b/guides/device_capabilities.md @@ -242,10 +242,17 @@ Apple docs: ```elixir socket = - Mob.IOS.Vision.recognize_text(socket, image_path, - recognition_level: :accurate, - uses_language_correction: true - ) + Mob.Photos.pick(socket, max: 1, types: [:image]) + +def handle_info({:photos, :picked, [%{path: path} | _]}, socket) do + socket = + Mob.IOS.Vision.recognize_text(socket, path, + recognition_level: :accurate, + uses_language_correction: true + ) + + {:noreply, socket} +end def handle_info({:vision, :recognized_text, %{text: text}}, socket) do {:noreply, Mob.Socket.assign(socket, :ocr_text, text)} @@ -255,7 +262,8 @@ end ## iOS Speech transcription Transcribes an existing audio file with Apple's Speech framework. Use -`Mob.Audio` to record microphone input first. +`Mob.Files.pick/2` for user-selected audio or `Mob.Audio` to record microphone +input first. Apple docs: [SFSpeechRecognizer](https://developer.apple.com/documentation/speech/sfspeechrecognizer) and @@ -263,10 +271,17 @@ Apple docs: ```elixir socket = - Mob.IOS.Speech.transcribe_audio(socket, recording_path, - locale: "en-US", - requires_on_device_recognition: false - ) + Mob.Files.pick(socket, types: ["public.audio", "public.mpeg-4-audio", "audio/*"]) + +def handle_info({:files, :picked, [%{path: path} | _]}, socket) do + socket = + Mob.IOS.Speech.transcribe_audio(socket, path, + locale: "en-US", + requires_on_device_recognition: false + ) + + {:noreply, socket} +end def handle_info({:speech, :transcribed_audio, %{text: text}}, socket) do {:noreply, Mob.Socket.assign(socket, :transcript, text)} @@ -287,15 +302,17 @@ support local recognition. The lightest simulator smoke test is: -1. Build a Mob app that exposes a screen with a text field for an image path and - calls `Mob.IOS.Vision.recognize_text/3`. -2. Copy an image with readable text into the simulator app's Documents - directory, then run OCR against that path. -3. Record audio with `Mob.Audio.start_recording/2` and pass the resulting path - to `Mob.IOS.Speech.transcribe_audio/3`. +1. Build a Mob app that exposes a screen with `Mob.Photos.pick/2` and calls + `Mob.IOS.Vision.recognize_text/3` with the selected photo path. +2. Add a photo with readable text to the simulator photo library, select it in + the picker, and confirm OCR returns the expected text. +3. Build a second action with `Mob.Files.pick/2`, select an audio file from the + native document picker, and pass the selected path to + `Mob.IOS.Speech.transcribe_audio/3`. 4. Confirm Foundation Models returns the expected simulator-unavailable error. -For a simulator app that is already installed, the useful setup commands are: +For path-based debugging in a simulator app that is already installed, the +useful setup commands are: ```sh SIM_ID="booted" diff --git a/lib/mob/ios/speech.ex b/lib/mob/ios/speech.ex index 9d5cc2f1..ab498a2c 100644 --- a/lib/mob/ios/speech.ex +++ b/lib/mob/ios/speech.ex @@ -8,9 +8,15 @@ defmodule Mob.IOS.Speech do https://developer.apple.com/documentation/speech/sfspeechrecognizer https://developer.apple.com/documentation/speech/sfspeechurlrecognitionrequest - Calls are asynchronous. Results are delivered to the calling process: + Calls are asynchronous. Results are delivered to the calling process. Pair + this with `Mob.Files.pick/2` when the audio should come from the native + document picker: - Mob.IOS.Speech.transcribe_audio(socket, "/path/to/audio.m4a") + socket = Mob.Files.pick(socket, types: ["public.audio", "audio/*"]) + + def handle_info({:files, :picked, [%{path: path} | _]}, socket) do + {:noreply, Mob.IOS.Speech.transcribe_audio(socket, path)} + end def handle_info({:speech, :transcribed_audio, %{text: text}}, socket) do {:noreply, Mob.Socket.assign(socket, :transcript, text)} diff --git a/lib/mob/ios/vision.ex b/lib/mob/ios/vision.ex index 00cc29cd..15fbe32f 100644 --- a/lib/mob/ios/vision.ex +++ b/lib/mob/ios/vision.ex @@ -6,9 +6,15 @@ defmodule Mob.IOS.Vision do See Apple's API documentation: https://developer.apple.com/documentation/vision/vnrecognizetextrequest - Calls are asynchronous. Results are delivered to the calling process: + Calls are asynchronous. Results are delivered to the calling process. Pair + this with `Mob.Photos.pick/2` when the image should come from the user's + photo library: - Mob.IOS.Vision.recognize_text(socket, "/path/to/image.png") + socket = Mob.Photos.pick(socket, max: 1, types: [:image]) + + def handle_info({:photos, :picked, [%{path: path} | _]}, socket) do + {:noreply, Mob.IOS.Vision.recognize_text(socket, path)} + end def handle_info({:vision, :recognized_text, %{text: text}}, socket) do {:noreply, Mob.Socket.assign(socket, :ocr_text, text)} From 2093c503aeed46cf4392d2c99d6816f182da6cbd Mon Sep 17 00:00:00 2001 From: Yurko Hoshko Date: Sun, 17 May 2026 00:21:56 -0700 Subject: [PATCH 6/6] Document recording-based Speech smoke test --- guides/device_capabilities.md | 16 ++++++++-------- lib/mob/ios/speech.ex | 9 +++++---- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/guides/device_capabilities.md b/guides/device_capabilities.md index 13931267..6bc021de 100644 --- a/guides/device_capabilities.md +++ b/guides/device_capabilities.md @@ -262,18 +262,18 @@ end ## iOS Speech transcription Transcribes an existing audio file with Apple's Speech framework. Use -`Mob.Files.pick/2` for user-selected audio or `Mob.Audio` to record microphone -input first. +`Mob.Audio` to record microphone input first, then transcribe the saved +recording path. Apple docs: [SFSpeechRecognizer](https://developer.apple.com/documentation/speech/sfspeechrecognizer) and [SFSpeechURLRecognitionRequest](https://developer.apple.com/documentation/speech/sfspeechurlrecognitionrequest). ```elixir -socket = - Mob.Files.pick(socket, types: ["public.audio", "public.mpeg-4-audio", "audio/*"]) +socket = Mob.Audio.start_recording(socket, format: :aac, quality: :high) +socket = Mob.Audio.stop_recording(socket) -def handle_info({:files, :picked, [%{path: path} | _]}, socket) do +def handle_info({:audio, :recorded, %{path: path}}, socket) do socket = Mob.IOS.Speech.transcribe_audio(socket, path, locale: "en-US", @@ -306,9 +306,9 @@ The lightest simulator smoke test is: `Mob.IOS.Vision.recognize_text/3` with the selected photo path. 2. Add a photo with readable text to the simulator photo library, select it in the picker, and confirm OCR returns the expected text. -3. Build a second action with `Mob.Files.pick/2`, select an audio file from the - native document picker, and pass the selected path to - `Mob.IOS.Speech.transcribe_audio/3`. +3. Build two audio actions: one calls `Mob.Audio.start_recording/2`; the other + calls `Mob.Audio.stop_recording/1`. Pass the recorded file path from + `{:audio, :recorded, %{path: path}}` to `Mob.IOS.Speech.transcribe_audio/3`. 4. Confirm Foundation Models returns the expected simulator-unavailable error. For path-based debugging in a simulator app that is already installed, the diff --git a/lib/mob/ios/speech.ex b/lib/mob/ios/speech.ex index ab498a2c..eb9a3e02 100644 --- a/lib/mob/ios/speech.ex +++ b/lib/mob/ios/speech.ex @@ -9,12 +9,13 @@ defmodule Mob.IOS.Speech do https://developer.apple.com/documentation/speech/sfspeechurlrecognitionrequest Calls are asynchronous. Results are delivered to the calling process. Pair - this with `Mob.Files.pick/2` when the audio should come from the native - document picker: + this with `Mob.Audio.start_recording/2` and `Mob.Audio.stop_recording/1` when + the audio should come from the microphone: - socket = Mob.Files.pick(socket, types: ["public.audio", "audio/*"]) + socket = Mob.Audio.start_recording(socket, format: :aac, quality: :high) + socket = Mob.Audio.stop_recording(socket) - def handle_info({:files, :picked, [%{path: path} | _]}, socket) do + def handle_info({:audio, :recorded, %{path: path}}, socket) do {:noreply, Mob.IOS.Speech.transcribe_audio(socket, path)} end