tests

Author: sanchitmonga22

Playground/YapRun/mac_plan.md

@@ -0,0 +1,139 @@
+Now I have comprehensive details. Let me compile the complete UX walkthrough.
+
+Here's the full Wispr Flow macOS user experience — end to end — so you can replicate it for RunAnywhere.
+## Download & Install Experience
+The entire install takes under 2 minutes: [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+
+1. Go to **wisprflow.ai** → click **"Download for macOS"**
+2. A `.dmg` file downloads (~100 MB) [reddit](https://www.reddit.com/r/macapps/comments/1nzi52c/wispr_flow_managed_to_get_their_shit_together/)
+3. Open the DMG → **drag the Flow icon into Applications** [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+4. Launch Wispr Flow from Applications [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+5. **Sign in** (Apple Sign-In or email) [roadmap.wisprflow](https://roadmap.wisprflow.ai/changelog/pointup2-new-flow-bar-flow-pro)
+6. Follow the on-screen setup prompts
+## First-Launch Permissions
+On first launch, macOS asks for two critical permissions: [docs.wisprflow](https://docs.wisprflow.ai/articles/6409258247-starting-your-first-dictation)
+
+1. **Microphone** → `System Settings → Privacy & Security → Microphone` — allow Wispr Flow
+2. **Accessibility** → `System Settings → Privacy & Security → Accessibility` — allow Wispr Flow (needed for global hotkey capture, reading active app context, and simulating paste)
+
+No other configuration is needed. The app is immediately ready to use. [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+## What Appears On Screen
+After setup, two things are visible:
+### The Flow Bar (Floating Pill)
+A small **rounded pill/bubble at the bottom center of the screen**. It: [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+
+- Sits **always on top** of all windows, on every desktop/space
+- Is semi-transparent and unobtrusive when idle
+- **Left-click** it → starts hands-free dictation [roadmap.wisprflow](https://roadmap.wisprflow.ai/changelog/pointup2-new-flow-bar-flow-pro)
+- **Right-click** it → shows quick settings (language, microphone selection) [roadmap.wisprflow](https://roadmap.wisprflow.ai/changelog/pointup2-new-flow-bar-flow-pro)
+- Can be toggled off in Settings → "Show flow bar at all times" [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+### Menu Bar Icon
+A small icon in the macOS menu bar (top-right area) for quick access to the Hub (main settings window). [willowvoice](https://willowvoice.com/blog/super-whisper-vs-wispr-flow-comparison-reviews-and-alternatives-in-2025)
+## Core Dictation UX (Push-to-Talk)
+This is the primary experience — **hold a key, speak, release, text appears**: [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+
+1. **Click into any text field** in any app (Slack, Gmail, Notes, Notion, Cursor, browser, etc.) [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+2. **Hold the Fn key** (default hotkey) [docs.wisprflow](https://docs.wisprflow.ai/articles/6409258247-starting-your-first-dictation)
+3. You hear an **audio ping** and see the Flow Bar animate with **white moving bars** (waveform) — this means the mic is live [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+4. **Speak naturally** — ramble, use filler words, change your mind mid-sentence [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+5. **Release the Fn key** [docs.wisprflow](https://docs.wisprflow.ai/articles/6409258247-starting-your-first-dictation)
+6. Flow processes your speech (200ms start, 450ms paste in Instant Mode) [wisprflow](https://wisprflow.ai/post/top-10-dictation-tools-december-2025)
+7. **Formatted, clean text appears** exactly where your cursor was — filler words removed, punctuation added, paragraphs created as needed [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+
+If you just **tap** the hotkey instead of holding it, a reminder prompt appears telling you to hold it down. Press **Esc** while dictating to cancel — nothing pastes, but the transcript is saved in Recent Activity. [docs.wisprflow](https://docs.wisprflow.ai/articles/6409258247-starting-your-first-dictation)
+## Hands-Free Dictation
+For longer dictation without holding a key: [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+
+1. Press **Fn + Space** → hands-free mode activates
+2. An **X icon** and a **stop button (circle with square)** appear on the Flow Bar [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+3. The waveform animates to show the mic is active
+4. **Speak freely** without holding any key
+5. Press **Fn** once to stop and paste the transcript [docs.wisprflow](https://docs.wisprflow.ai/articles/6409258247-starting-your-first-dictation)
+6. Or **click the stop button** on the Flow Bar [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+## Flow Bar Click Dictation
+For people who prefer mouse over keyboard: [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+
+1. **Click** the Flow Bar bubble → recording starts
+2. Hover shows tooltip instructions
+3. **Click again** → stops recording and pastes text
+4. Same behavior as push-to-talk, just mouse-driven
+## What the Text Looks Like
+Flow doesn't just dump raw speech. Examples from the tutorial: [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+
+| What you say | What Flow types |
+|-------------|----------------|
+| "Let's meet tomorrow. Actually, let's do Friday instead." | "Let's meet Friday." (one clean sentence) |
+| "Grocery list, apples, pears, bananas, strawberries" | A bulleted list with colon header and one bullet per item |
+| "um so like I was thinking maybe we could uh do that thing" | Clean sentence with filler removed |
+| Mid-sentence corrections | Reshapes the sentence so it reads clean  [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ) |
+
+Flow also **adapts tone per app** — formal in Gmail, casual in iMessage, technical in Cursor. [wisprflow](https://wisprflow.ai/post/wispr-flow-for-seamless-communication)
+## The Hub (Main App Window)
+Opened from the menu bar icon. Contains these sections: [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+### Home
+- **Today** — recent dictation history with full transcripts [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+- **Stats** — total words dictated, words per minute (WPM) [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+- Quick tips (hands-free shortcut reminder)
+### Dictionary
+- Add custom words, names, jargon [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+- **Misspelling correction**: e.g., you say "CJ" but want it typed as "C-J" with a dash — add the correction once, it's fixed forever [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+- **Auto-add to Dictionary** option — Flow watches your corrections and learns automatically [ppl-ai-file-upload.s3.amazonaws](https://ppl-ai-file-upload.s3.amazonaws.com/web/direct-files/attachments/images/160267873/a090ff18-8e94-4931-a7cb-616097500514/image.jpg)
+- Up to 800 words/phrases [wisprflow](https://wisprflow.ai/post/top-10-dictation-tools-december-2025)
+- Shareable with your team [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+### Snippets
+- Saved text shortcuts expanded by voice [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+- Say a trigger phrase → Flow expands it to the full saved text
+- Great for: email signatures, links, addresses, bios, boilerplate [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+### Styles
+- Control how Flow writes and formats your words [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+- Set different tones for different apps/tasks
+- Keeps writing style consistent across tools
+### Notes
+- Built-in quick notes — press hotkey, speak a note, it's saved [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+- Recent notes shown at bottom
+- Notes sync across desktop and mobile [ppl-ai-file-upload.s3.amazonaws](https://ppl-ai-file-upload.s3.amazonaws.com/web/direct-files/attachments/images/160267873/a090ff18-8e94-4931-a7cb-616097500514/image.jpg)
+### Settings
+Organized into tabs: [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+
+| Tab | Options |
+|-----|---------|
+| **General** | Change push-to-talk hotkey, hands-free shortcut, command mode shortcut, paste-last-transcript shortcut. Reset to defaults button  [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ) |
+| **System** | Launch at login (on/off), Show flow bar at all times, Show app in Dock, Dictation sound effects (audio ping), **Mute music while dictating** (ducks music, restores after)  [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ) |
+| **Vibe** | Writing style/tone preferences  [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ) |
+| **Coding** | Developer-specific settings  [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ) |
+| **Experiment** | Auto-add to dictionary, smart formatting, email signature, creator mode  [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ) |
+| **Account** | Login, plan management, reset app  [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ) |
+## Command Mode (Pro Feature)
+For editing existing text with your voice: [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+
+- Activate with a separate hotkey
+- Speak instructions like: "make this more formal", "use lowercase in iMessage", "break into paragraphs" [wisprflow](https://wisprflow.ai/post/wispr-flow-for-seamless-communication)
+- Flow reads the surrounding text via the Accessibility API and applies the edit [ppl-ai-file-upload.s3.amazonaws](https://ppl-ai-file-upload.s3.amazonaws.com/web/direct-files/attachments/images/160267873/a090ff18-8e94-4931-a7cb-616097500514/image.jpg)
+## How It Stays Running
+The app persistence model for your replication: [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+
+- **Agent app** — runs without a Dock icon by default (togglable) [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+- **Launch at login** — auto-starts with macOS [youtube](https://www.youtube.com/watch?v=qlg3p5HdXRQ)
+- **Global hotkey listener** — always monitoring for Fn press via `CGEvent` tap, even when the app has no window [docs.wisprflow](https://docs.wisprflow.ai/articles/6409258247-starting-your-first-dictation)
+- **Near-zero CPU when idle** — only activates on hotkey press [reddit](https://www.reddit.com/r/macapps/comments/1nzi52c/wispr_flow_managed_to_get_their_shit_together/)
+- **No background audio recording** — mic only activates when hotkey is held or hands-free is active [reddit](https://www.reddit.com/r/macapps/comments/1nzi52c/wispr_flow_managed_to_get_their_shit_together/)
+- **Flow Bar** is an always-on-top overlay that shows the app is ready [roadmap.wisprflow](https://roadmap.wisprflow.ai/changelog/pointup2-new-flow-bar-flow-pro)
+## Technical Implementation Summary for RunAnywhere macOS
+| UX Element | Implementation |
+|-----------|---------------|
+| **Flow Bar** | `NSPanel` with `.floating` level, `.borderless` style, `.canJoinAllSpaces` collection behavior |
+| **Global Fn hotkey** | `CGEvent.tapCreate` with `kCGEventKeyDown`/`kCGEventKeyUp` for Fn key monitoring |
+| **Audio ping on start** | `AVAudioPlayer` playing a short sound file on keyDown |
+| **Waveform animation** | Core Animation or SwiftUI animation on the Flow Bar showing audio levels from `AVAudioEngine.inputNode.installTap` |
+| **Text insertion** | Copy to `NSPasteboard.general`, then simulate `Cmd+V` via `CGEvent(keyboardEventSource:virtualKey:keyDown:)` |
+| **Context awareness** | `AXUIElementCopyAttributeValue` on the focused element to read surrounding text |
+| **App name detection** | `NSWorkspace.shared.frontmostApplication?.localizedName` |
+| **Mute music** | Use `MediaRemote` private framework or AppleScript to pause/play iTunes/Spotify |
+| **Launch at login** | `SMAppService.mainApp.register()` (modern) or `SMLoginItemSetEnabled` (legacy) |
+| **Agent mode (no Dock)** | `LSUIElement = YES` in Info.plist |
+| **Menu bar** | `NSStatusBar.system.statusItem(withLength:)` |
+| **Settings/Hub** | Standard `NSWindow` shown on status item click |
+| **Recent Activity** | Local SQLite or Core Data store of all transcriptions |
+| **Dictionary/Snippets** | JSON or plist in App Group container, synced via CloudKit if needed |
+
+The key difference for your build: instead of streaming audio to cloud, you run **Sherpa Whisper on-device** in the same process. On your M3 Max, Whisper Tiny should return results in <500ms, which is competitive with Wispr's cloud latency. [wisprflow](https://wisprflow.ai/post/top-10-dictation-tools-december-2025)

examples/ios/RunAnywhereAI/RunAnywhereAI/Features/VoiceKeyboard/DictationActivityAttributes.swift

@@ -9,11 +9,15 @@
 //  The extension gets this file via the manual pbxproj wiring (same pattern as SharedConstants.swift).
 //
 
+#if os(iOS)
 import ActivityKit
+#endif
 import Foundation
 
 /// The static attributes for a dictation flow session.
 /// These do not change after the activity is started.
+#if os(iOS)
+@available(iOS 16.1, *)
 struct DictationActivityAttributes: ActivityAttributes {
 
     /// The dynamic / live state updated throughout the session.
@@ -31,3 +35,4 @@ struct DictationActivityAttributes: ActivityAttributes {
     /// Session identifier — set once at start
     var sessionId: String
 }
+#endif

sdk/runanywhere-commons/cmake/FetchONNXRuntime.cmake

@@ -69,12 +69,10 @@ elseif(IOS OR CMAKE_SYSTEM_NAME STREQUAL "iOS")
     add_library(onnxruntime STATIC IMPORTED GLOBAL)
 
     # Determine architecture-specific library path
-    if(CMAKE_OSX_SYSROOT MATCHES "simulator")
-        if(CMAKE_OSX_ARCHITECTURES MATCHES "arm64")
-            set(ONNX_FRAMEWORK_ARCH "ios-arm64_x86_64-simulator")
-        else()
-            set(ONNX_FRAMEWORK_ARCH "ios-arm64_x86_64-simulator")
-        endif()
+    # Check both CMAKE_OSX_SYSROOT (case-insensitive) and IOS_PLATFORM from ios.toolchain.cmake
+    string(TOLOWER "${CMAKE_OSX_SYSROOT}" _sysroot_lower)
+    if(_sysroot_lower MATCHES "simulator" OR (DEFINED IOS_PLATFORM AND IOS_PLATFORM MATCHES "SIMULATOR"))
+        set(ONNX_FRAMEWORK_ARCH "ios-arm64_x86_64-simulator")
     else()
         set(ONNX_FRAMEWORK_ARCH "ios-arm64")
     endif()
@@ -138,6 +136,41 @@ elseif(ANDROID)
         )
         target_include_directories(onnxruntime INTERFACE "${ONNX_HEADER_PATH}")
 
+        # Sherpa-ONNX Android prebuilts only ship the C API header.
+        # The ONNX C++ API headers (onnxruntime_cxx_api.h etc.) are header-only
+        # wrappers needed by wakeword_onnx.cpp.  Download them if missing.
+        if(NOT EXISTS "${ONNX_HEADER_PATH}/onnxruntime_cxx_api.h")
+            set(ONNX_CXX_HEADER_DIR "${CMAKE_BINARY_DIR}/_deps/onnxruntime-cxx-headers")
+            file(MAKE_DIRECTORY "${ONNX_CXX_HEADER_DIR}")
+
+            set(ONNX_HEADER_BASE_URL "https://raw.githubusercontent.com/microsoft/onnxruntime/v${ONNX_VERSION_ANDROID}/include/onnxruntime/core/session")
+            set(ONNX_CXX_HEADERS
+                onnxruntime_cxx_api.h
+                onnxruntime_cxx_inline.h
+                onnxruntime_float16.h
+                onnxruntime_session_options_config_keys.h
+                onnxruntime_run_options_config_keys.h
+            )
+
+            foreach(header ${ONNX_CXX_HEADERS})
+                if(NOT EXISTS "${ONNX_CXX_HEADER_DIR}/${header}")
+                    message(STATUS "Downloading ONNX C++ header: ${header}")
+                    file(DOWNLOAD
+                        "${ONNX_HEADER_BASE_URL}/${header}"
+                        "${ONNX_CXX_HEADER_DIR}/${header}"
+                        STATUS download_status
+                    )
+                    list(GET download_status 0 download_code)
+                    if(NOT download_code EQUAL 0)
+                        message(WARNING "Failed to download ${header} (status: ${download_status})")
+                    endif()
+                endif()
+            endforeach()
+
+            target_include_directories(onnxruntime INTERFACE "${ONNX_CXX_HEADER_DIR}")
+            message(STATUS "ONNX Runtime C++ headers: ${ONNX_CXX_HEADER_DIR}")
+        endif()
+
         message(STATUS "ONNX Runtime Android library: ${ONNX_LIB_PATH}")
         message(STATUS "ONNX Runtime Android headers: ${ONNX_HEADER_PATH}")
     else()

sdk/runanywhere-commons/src/backends/onnx/CMakeLists.txt

@@ -45,7 +45,8 @@ if(RAC_PLATFORM_IOS)
         message(STATUS "Found Sherpa-ONNX xcframework at: ${SHERPA_ONNX_ROOT}")
         set(SHERPA_ONNX_AVAILABLE ON)
 
-        if(CMAKE_OSX_SYSROOT MATCHES "simulator")
+        string(TOLOWER "${CMAKE_OSX_SYSROOT}" _sherpa_sysroot_lower)
+        if(_sherpa_sysroot_lower MATCHES "simulator" OR (DEFINED IOS_PLATFORM AND IOS_PLATFORM MATCHES "SIMULATOR"))
             set(SHERPA_ARCH "ios-arm64_x86_64-simulator")
         else()
             set(SHERPA_ARCH "ios-arm64")

sdk/runanywhere-commons/src/backends/onnx/rac_backend_onnx_register.cpp

@@ -243,7 +243,8 @@ rac_bool_t onnx_stt_can_handle(const rac_service_request_t* request, void* user_
 
     if (strstr(path, "whisper") != nullptr || strstr(path, "zipformer") != nullptr ||
         strstr(path, "paraformer") != nullptr || strstr(path, "parakeet") != nullptr ||
-        strstr(path, "nemo") != nullptr || strstr(path, ".onnx") != nullptr) {
+        strstr(path, "nemo") != nullptr || strstr(path, "moonshine") != nullptr ||
+        strstr(path, ".onnx") != nullptr) {
         RAC_LOG_INFO(LOG_CAT, "onnx_stt_can_handle: path matches -> TRUE");
         return RAC_TRUE;
     }

sdk/runanywhere-commons/src/features/stt/rac_stt_service.cpp

@@ -146,6 +146,20 @@ void rac_stt_result_free(rac_stt_result_t* result) {
         free(result->text);
         result->text = nullptr;
     }
+    if (result->detected_language) {
+        free(result->detected_language);
+        result->detected_language = nullptr;
+    }
+    if (result->words) {
+        for (size_t i = 0; i < result->num_words; i++) {
+            if (result->words[i].text) {
+                free(const_cast<char*>(result->words[i].text));
+            }
+        }
+        free(result->words);
+        result->words = nullptr;
+        result->num_words = 0;
+    }
 }
 
 }  // extern "C"