feat(listener): enhance loading state events with context data and frontend wiring

- Add AudioMode enum for serializing ChannelMode
- Enhance AudioReady event with mode and device fields
- Add AudioError event for mic/device failures
- Enhance Connected event with adapter field
- Replace retry_count with is_retryable in ConnectionError
- Wire up all new events in frontend general.ts
- Add loadingPhase and lastError fields to live state

Co-Authored-By: yujonglee <[email protected]>

Author: devin-ai-integration[bot]

apps/desktop/src/store/zustand/listener/general.ts

@@ -34,16 +34,25 @@ const hasSessionId = (
 ): payload is SessionEvent & { session_id: string } =>
   "session_id" in payload && typeof payload.session_id === "string";
 
+export type LoadingPhase =
+  | "idle"
+  | "initializing_audio"
+  | "audio_ready"
+  | "connecting"
+  | "connected";
+
 export type GeneralState = {
   live: {
     sessionEventUnlisten?: () => void;
     loading: boolean;
+    loadingPhase: LoadingPhase;
     status: LiveSessionStatus;
     amplitude: { mic: number; speaker: number };
     seconds: number;
     intervalId?: NodeJS.Timeout;
     sessionId: string | null;
     muted: boolean;
+    lastError: string | null;
   };
 };
 
@@ -65,10 +74,12 @@ const initialState: GeneralState = {
   live: {
     status: "inactive",
     loading: false,
+    loadingPhase: "idle",
     amplitude: { mic: 0, speaker: 0 },
     seconds: 0,
     sessionId: null,
     muted: false,
+    lastError: null,
   },
 };
 
@@ -155,6 +166,7 @@ export const createGeneralSlice = <
           mutate(state, (draft) => {
             draft.live.status = "running_active";
             draft.live.loading = false;
+            draft.live.loadingPhase = "idle";
             draft.live.seconds = 0;
             draft.live.intervalId = intervalId;
             draft.live.sessionId = targetSessionId;
@@ -181,15 +193,55 @@ export const createGeneralSlice = <
           mutate(state, (draft) => {
             draft.live.status = "inactive";
             draft.live.loading = false;
+            draft.live.loadingPhase = "idle";
             draft.live.sessionId = null;
             draft.live.sessionEventUnlisten = undefined;
+            draft.live.lastError = null;
           }),
         );
 
         get().resetTranscript();
       } else if (payload.type === "streamResponse") {
         const response = payload.response;
         get().handleTranscriptResponse(response as unknown as StreamResponse);
+      } else if (payload.type === "initializingAudio") {
+        set((state) =>
+          mutate(state, (draft) => {
+            draft.live.loadingPhase = "initializing_audio";
+            draft.live.lastError = null;
+          }),
+        );
+      } else if (payload.type === "audioReady") {
+        set((state) =>
+          mutate(state, (draft) => {
+            draft.live.loadingPhase = "audio_ready";
+          }),
+        );
+      } else if (payload.type === "audioError") {
+        set((state) =>
+          mutate(state, (draft) => {
+            draft.live.lastError = payload.error;
+            draft.live.loading = false;
+          }),
+        );
+      } else if (payload.type === "connecting") {
+        set((state) =>
+          mutate(state, (draft) => {
+            draft.live.loadingPhase = "connecting";
+          }),
+        );
+      } else if (payload.type === "connected") {
+        set((state) =>
+          mutate(state, (draft) => {
+            draft.live.loadingPhase = "connected";
+          }),
+        );
+      } else if (payload.type === "connectionError") {
+        set((state) =>
+          mutate(state, (draft) => {
+            draft.live.lastError = payload.error;
+          }),
+        );
       }
     };
 
@@ -259,11 +311,13 @@ export const createGeneralSlice = <
 
               draft.live.sessionEventUnlisten = undefined;
               draft.live.loading = false;
+              draft.live.loadingPhase = "idle";
               draft.live.status = "inactive";
               draft.live.amplitude = { mic: 0, speaker: 0 };
               draft.live.seconds = 0;
               draft.live.sessionId = null;
               draft.live.muted = initialState.live.muted;
+              draft.live.lastError = null;
             }),
           );
         },

plugins/listener/src/actors/listener.rs

@@ -96,10 +96,11 @@ impl Actor for ListenerActor {
             tracing::error!(?error, "failed_to_emit_connecting");
         }
 
-        let (tx, rx_task, shutdown_tx) = spawn_rx_task(args.clone(), myself).await?;
+        let (tx, rx_task, shutdown_tx, adapter_name) = spawn_rx_task(args.clone(), myself).await?;
 
         if let Err(error) = (SessionEvent::Connected {
             session_id: args.session_id.clone(),
+            adapter: adapter_name,
         })
         .emit(&args.app)
         {
@@ -213,13 +214,24 @@ async fn spawn_rx_task(
         ChannelSender,
         tokio::task::JoinHandle<()>,
         tokio::sync::oneshot::Sender<()>,
+        String, // adapter name
     ),
     ActorProcessingErr,
 > {
     let adapter_kind = AdapterKind::from_url_and_languages(&args.base_url, &args.languages);
     let is_dual = matches!(args.mode, crate::actors::ChannelMode::MicAndSpeaker);
 
-    match (adapter_kind, is_dual) {
+    let adapter_name = match adapter_kind {
+        AdapterKind::Argmax => "Argmax",
+        AdapterKind::Soniox => "Soniox",
+        AdapterKind::Fireworks => "Fireworks",
+        AdapterKind::Deepgram => "Deepgram",
+        AdapterKind::AssemblyAI => "AssemblyAI",
+        AdapterKind::OpenAI => "OpenAI",
+        AdapterKind::Gladia => "Gladia",
+    };
+
+    let result = match (adapter_kind, is_dual) {
         (AdapterKind::Argmax, false) => {
             spawn_rx_task_single_with_adapter::<ArgmaxAdapter>(args, myself).await
         }
@@ -262,7 +274,9 @@ async fn spawn_rx_task(
         (AdapterKind::Gladia, true) => {
             spawn_rx_task_dual_with_adapter::<GladiaAdapter>(args, myself).await
         }
-    }
+    }?;
+
+    Ok((result.0, result.1, result.2, adapter_name.to_string()))
 }
 
 fn build_listen_params(args: &ListenerArgs) -> owhisper_interface::ListenParams {
@@ -330,7 +344,7 @@ async fn spawn_rx_task_single_with_adapter<A: RealtimeSttAdapter>(
             let _ = (SessionEvent::ConnectionError {
                 session_id: args.session_id.clone(),
                 error: "listen_ws_connect_timeout".to_string(),
-                retry_count: 0,
+                is_retryable: true,
             })
             .emit(&args.app);
             return Err(actor_error("listen_ws_connect_timeout"));
@@ -340,7 +354,7 @@ async fn spawn_rx_task_single_with_adapter<A: RealtimeSttAdapter>(
             let _ = (SessionEvent::ConnectionError {
                 session_id: args.session_id.clone(),
                 error: format!("listen_ws_connect_failed: {:?}", e),
-                retry_count: 0,
+                is_retryable: true,
             })
             .emit(&args.app);
             return Err(actor_error(format!("listen_ws_connect_failed: {:?}", e)));
@@ -402,7 +416,7 @@ async fn spawn_rx_task_dual_with_adapter<A: RealtimeSttAdapter>(
             let _ = (SessionEvent::ConnectionError {
                 session_id: args.session_id.clone(),
                 error: "listen_ws_connect_timeout".to_string(),
-                retry_count: 0,
+                is_retryable: true,
             })
             .emit(&args.app);
             return Err(actor_error("listen_ws_connect_timeout"));
@@ -412,7 +426,7 @@ async fn spawn_rx_task_dual_with_adapter<A: RealtimeSttAdapter>(
             let _ = (SessionEvent::ConnectionError {
                 session_id: args.session_id.clone(),
                 error: format!("listen_ws_connect_failed: {:?}", e),
-                retry_count: 0,
+                is_retryable: true,
             })
             .emit(&args.app);
             return Err(actor_error(format!("listen_ws_connect_failed: {:?}", e)));

plugins/listener/src/actors/source.rs

@@ -195,6 +195,12 @@ impl Actor for SourceActor {
             }
             SourceMsg::StreamFailed(reason) => {
                 tracing::error!(%reason, "source_stream_failed_stopping");
+                let _ = (SessionEvent::AudioError {
+                    session_id: st.session_id.clone(),
+                    error: reason.clone(),
+                    device: st.mic_device.clone(),
+                })
+                .emit(&st.app);
                 myself.stop(Some(reason));
             }
         }
@@ -240,6 +246,8 @@ async fn start_source_loop(
     if result.is_ok() {
         if let Err(error) = (SessionEvent::AudioReady {
             session_id: st.session_id.clone(),
+            mode: st.current_mode.into(),
+            device: st.mic_device.clone(),
         })
         .emit(&st.app)
         {

plugins/listener/src/events.rs

@@ -8,6 +8,25 @@ macro_rules! common_event_derives {
     };
 }
 
+/// Audio channel mode for the session
+#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, specta::Type)]
+#[serde(rename_all = "snake_case")]
+pub enum AudioMode {
+    MicOnly,
+    SpeakerOnly,
+    MicAndSpeaker,
+}
+
+impl From<crate::actors::ChannelMode> for AudioMode {
+    fn from(mode: crate::actors::ChannelMode) -> Self {
+        match mode {
+            crate::actors::ChannelMode::MicOnly => AudioMode::MicOnly,
+            crate::actors::ChannelMode::SpeakerOnly => AudioMode::SpeakerOnly,
+            crate::actors::ChannelMode::MicAndSpeaker => AudioMode::MicAndSpeaker,
+        }
+    }
+}
+
 common_event_derives! {
     #[serde(tag = "type")]
     pub enum SessionEvent {
@@ -33,16 +52,29 @@ common_event_derives! {
         #[serde(rename = "initializingAudio")]
         InitializingAudio { session_id: String },
         #[serde(rename = "audioReady")]
-        AudioReady { session_id: String },
+        AudioReady {
+            session_id: String,
+            mode: AudioMode,
+            device: Option<String>,
+        },
+        #[serde(rename = "audioError")]
+        AudioError {
+            session_id: String,
+            error: String,
+            device: Option<String>,
+        },
         #[serde(rename = "connecting")]
         Connecting { session_id: String },
         #[serde(rename = "connected")]
-        Connected { session_id: String },
+        Connected {
+            session_id: String,
+            adapter: String,
+        },
         #[serde(rename = "connectionError")]
         ConnectionError {
             session_id: String,
             error: String,
-            retry_count: u32,
+            is_retryable: bool,
         },
         ExitRequested
     }