-
Notifications
You must be signed in to change notification settings - Fork 0
feat(gui): wire partial/final transcript commands to STT pipeline seam #386
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -113,6 +113,88 @@ impl OverlayModel { | |
| self.snapshot() | ||
| } | ||
|
|
||
| /// Apply a live partial transcript update from the STT pipeline. | ||
| /// Keeps the shell in Listening state and updates the provisional text. | ||
| pub fn apply_partial_transcript( | ||
| &mut self, | ||
| text: &str, | ||
| status_detail: Option<&str>, | ||
| ) -> OverlaySnapshot { | ||
| self.snapshot.partial_transcript = text.to_string(); | ||
| self.snapshot.status = OverlayStatus::Listening; | ||
| if let Some(detail) = status_detail { | ||
| self.snapshot.status_detail = detail.to_string(); | ||
| } else { | ||
| self.snapshot.status_detail = | ||
| "Streaming partial words from the STT pipeline.".to_string(); | ||
| } | ||
| self.snapshot.error_message = None; | ||
| self.snapshot.expanded = true; | ||
| self.snapshot() | ||
| } | ||
|
|
||
| /// Promote the current partial transcript to final and transition to Ready. | ||
| /// Called when the STT pipeline commits an utterance. | ||
| pub fn apply_final_transcript( | ||
| &mut self, | ||
| text: &str, | ||
| status_detail: Option<&str>, | ||
| ) -> OverlaySnapshot { | ||
| self.snapshot.partial_transcript.clear(); | ||
| self.snapshot.final_transcript = text.to_string(); | ||
| self.snapshot.status = OverlayStatus::Ready; | ||
| if let Some(detail) = status_detail { | ||
| self.snapshot.status_detail = detail.to_string(); | ||
| } else { | ||
| self.snapshot.status_detail = | ||
| "Final transcript staged. Real injection wiring lands in a later tranche." | ||
| .to_string(); | ||
| } | ||
| self.snapshot.error_message = None; | ||
| self.snapshot.expanded = true; | ||
| self.snapshot() | ||
| } | ||
|
|
||
| /// Transition to Processing state (STT pipeline is finalizing the utterance). | ||
| pub fn apply_processing_state(&mut self, status_detail: Option<&str>) -> OverlaySnapshot { | ||
| self.snapshot.status = OverlayStatus::Processing; | ||
| if let Some(detail) = status_detail { | ||
| self.snapshot.status_detail = detail.to_string(); | ||
| } else { | ||
| self.snapshot.status_detail = | ||
| "Processing the utterance into a committed transcript.".to_string(); | ||
| } | ||
| self.snapshot.expanded = true; | ||
| self.snapshot() | ||
| } | ||
|
|
||
| /// Transition to Listening state (new utterance started). | ||
| pub fn apply_listening_state(&mut self, status_detail: Option<&str>) -> OverlaySnapshot { | ||
| self.snapshot.status = OverlayStatus::Listening; | ||
| self.snapshot.partial_transcript.clear(); | ||
| self.snapshot.final_transcript.clear(); | ||
|
Comment on lines
+176
to
+178
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Useful? React with 👍 / 👎. |
||
| if let Some(detail) = status_detail { | ||
| self.snapshot.status_detail = detail.to_string(); | ||
| } else { | ||
| self.snapshot.status_detail = "Listening for speech.".to_string(); | ||
| } | ||
| self.snapshot.expanded = true; | ||
| self.snapshot() | ||
|
Comment on lines
+174
to
+186
|
||
| } | ||
|
|
||
| /// Stop capture and return to Idle, clearing all transcript state. | ||
| /// Unlike `stop()` which increments the demo token, this is used by the real pipeline. | ||
| pub fn stop_capture(&mut self) -> OverlaySnapshot { | ||
| self.snapshot.status = OverlayStatus::Idle; | ||
| self.snapshot.paused = false; | ||
| self.snapshot.partial_transcript.clear(); | ||
| self.snapshot.final_transcript.clear(); | ||
| self.snapshot.status_detail = | ||
| "Capture stopped. The seam is ready for the next session.".to_string(); | ||
| self.snapshot.error_message = None; | ||
| self.snapshot() | ||
| } | ||
qodo-code-review[bot] marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| fn reject_command(&mut self, message: &str, detail: &str) -> OverlaySnapshot { | ||
| self.demo_token += 1; | ||
| self.snapshot.expanded = true; | ||
|
|
@@ -183,4 +265,44 @@ mod tests { | |
| assert_eq!(resumed.status, OverlayStatus::Listening); | ||
| assert!(!resumed.paused); | ||
| } | ||
|
|
||
| #[test] | ||
| fn apply_partial_transcript_updates_text_and_keeps_listening() { | ||
| let mut model = OverlayModel::default(); | ||
| model.apply_listening_state(None); | ||
|
|
||
| let snap1 = model.apply_partial_transcript("hello", None); | ||
| assert_eq!(snap1.partial_transcript, "hello"); | ||
| assert_eq!(snap1.status, OverlayStatus::Listening); | ||
| assert!(snap1.final_transcript.is_empty()); | ||
|
|
||
| let snap2 = model.apply_partial_transcript("hello world", None); | ||
| assert_eq!(snap2.partial_transcript, "hello world"); | ||
| assert_eq!(snap2.status, OverlayStatus::Listening); | ||
| } | ||
|
|
||
| #[test] | ||
| fn apply_final_transcript_moves_partial_to_final_and_transitions_to_ready() { | ||
| let mut model = OverlayModel::default(); | ||
| model.apply_listening_state(None); | ||
| model.apply_partial_transcript("hello world", None); | ||
|
|
||
| let snap = model.apply_final_transcript("hello world", None); | ||
| assert!(snap.partial_transcript.is_empty()); | ||
| assert_eq!(snap.final_transcript, "hello world"); | ||
| assert_eq!(snap.status, OverlayStatus::Ready); | ||
| } | ||
|
|
||
| #[test] | ||
| fn stop_capture_clears_all_transcript_state() { | ||
| let mut model = OverlayModel::default(); | ||
| model.apply_listening_state(None); | ||
| model.apply_partial_transcript("partial text", None); | ||
|
|
||
| let snap = model.stop_capture(); | ||
| assert_eq!(snap.status, OverlayStatus::Idle); | ||
| assert!(snap.partial_transcript.is_empty()); | ||
| assert!(snap.final_transcript.is_empty()); | ||
| assert!(snap.error_message.is_none()); | ||
| } | ||
|
Comment on lines
+274
to
+312
|
||
| } | ||
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -1,4 +1,4 @@ | ||||||
| import { useCallback, useEffect, useState } from "react"; | ||||||
| import { useCallback, useEffect, useState, useRef } from "react"; | ||||||
| import { | ||||||
| DEFAULT_SNAPSHOT, | ||||||
| type OverlaySnapshot, | ||||||
|
|
@@ -12,6 +12,11 @@ import { | |||||
| stopDemoDriver, | ||||||
| subscribeToOverlayEvents, | ||||||
| togglePauseState, | ||||||
| updatePartialTranscript, | ||||||
| updateFinalTranscript, | ||||||
| setOverlayProcessing, | ||||||
| setOverlayListening, | ||||||
| stopOverlayCapture, | ||||||
| } from "../lib/overlayBridge"; | ||||||
|
|
||||||
| function messageFromError(error: unknown): string { | ||||||
|
|
@@ -82,6 +87,48 @@ export function useOverlayShell() { | |||||
| [], | ||||||
| ); | ||||||
|
|
||||||
| // Debounce-flush partial transcript updates to avoid flooding the shell on rapid STT output. | ||||||
| const pendingPartialRef = useRef<string | null>(null); | ||||||
| const flushTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null); | ||||||
|
|
||||||
| const flushPartial = useCallback(() => { | ||||||
| const text = pendingPartialRef.current; | ||||||
| if (text === null) return; | ||||||
| pendingPartialRef.current = null; | ||||||
| if (flushTimerRef.current !== null) { | ||||||
| clearTimeout(flushTimerRef.current); | ||||||
| flushTimerRef.current = null; | ||||||
| } | ||||||
| void runCommand(() => updatePartialTranscript(text)); | ||||||
| }, [runCommand]); | ||||||
|
|
||||||
| const queuePartialTranscript = useCallback( | ||||||
| (text: string) => { | ||||||
| pendingPartialRef.current = text; | ||||||
| if (flushTimerRef.current !== null) { | ||||||
| clearTimeout(flushTimerRef.current); | ||||||
| } | ||||||
| // Flush after 80 ms of no new partials — balances latency vs. reduce repaints. | ||||||
|
||||||
| // Flush after 80 ms of no new partials — balances latency vs. reduce repaints. | |
| // Flush after 80 ms of no new partials — balancing latency vs. reducing repaints. |
Copilot
AI
Apr 2, 2026
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The new debounce/flush behavior for partial transcripts isn’t covered by tests in this file. Please add a test that verifies rapid queuePartialTranscript calls only invoke updatePartialTranscript once after ~80ms (with the latest text), and that unmount cleanup cancels a pending flush.
Outdated
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Cancel pending partial flush before finalizing transcript
queuePartialTranscript schedules a delayed updatePartialTranscript call, but updateFinalTranscript is invoked directly without clearing that timer. If STT emits a final result within 80ms of the last partial (a common path), the queued partial is sent after the final and can overwrite the snapshot back to listening with stale partial text. This creates out-of-order state transitions and can hide the committed final transcript.
Useful? React with 👍 / 👎.
Outdated
Copilot
AI
Apr 2, 2026
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The return block comment says “flushPartial sends immediately”, but flushPartial isn’t returned from the hook, so callers can’t actually flush immediately. Either export flushPartial (and/or a cancel function) alongside queuePartialTranscript, or adjust the comment/API so they match.
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -43,3 +43,28 @@ export function subscribeToOverlayEvents( | |||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| onEvent(event.payload); | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| }); | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| /// Feed a live partial transcript update from the STT pipeline. | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| export function updatePartialTranscript(text: string): Promise<OverlaySnapshot> { | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| return invoke<OverlaySnapshot>("update_partial_transcript", { text }); | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| /// Feed a final transcript from the STT pipeline. | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| export function updateFinalTranscript(text: string): Promise<OverlaySnapshot> { | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| return invoke<OverlaySnapshot>("update_final_transcript", { text }); | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| /// Transition the overlay to Processing state. | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| export function setOverlayProcessing(): Promise<OverlaySnapshot> { | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| return invoke<OverlaySnapshot>("set_overlay_processing"); | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| /// Transition the overlay to Listening state. | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| export function setOverlayListening(): Promise<OverlaySnapshot> { | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| return invoke<OverlaySnapshot>("set_overlay_listening"); | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| /// Stop real capture and return to Idle. | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
Comment on lines
+47
to
+67
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| /// Feed a live partial transcript update from the STT pipeline. | |
| export function updatePartialTranscript(text: string): Promise<OverlaySnapshot> { | |
| return invoke<OverlaySnapshot>("update_partial_transcript", { text }); | |
| } | |
| /// Feed a final transcript from the STT pipeline. | |
| export function updateFinalTranscript(text: string): Promise<OverlaySnapshot> { | |
| return invoke<OverlaySnapshot>("update_final_transcript", { text }); | |
| } | |
| /// Transition the overlay to Processing state. | |
| export function setOverlayProcessing(): Promise<OverlaySnapshot> { | |
| return invoke<OverlaySnapshot>("set_overlay_processing"); | |
| } | |
| /// Transition the overlay to Listening state. | |
| export function setOverlayListening(): Promise<OverlaySnapshot> { | |
| return invoke<OverlaySnapshot>("set_overlay_listening"); | |
| } | |
| /// Stop real capture and return to Idle. | |
| // Feed a live partial transcript update from the STT pipeline. | |
| export function updatePartialTranscript(text: string): Promise<OverlaySnapshot> { | |
| return invoke<OverlaySnapshot>("update_partial_transcript", { text }); | |
| } | |
| // Feed a final transcript from the STT pipeline. | |
| export function updateFinalTranscript(text: string): Promise<OverlaySnapshot> { | |
| return invoke<OverlaySnapshot>("update_final_transcript", { text }); | |
| } | |
| // Transition the overlay to Processing state. | |
| export function setOverlayProcessing(): Promise<OverlaySnapshot> { | |
| return invoke<OverlaySnapshot>("set_overlay_processing"); | |
| } | |
| // Transition the overlay to Listening state. | |
| export function setOverlayListening(): Promise<OverlaySnapshot> { | |
| return invoke<OverlaySnapshot>("set_overlay_listening"); | |
| } | |
| // Stop real capture and return to Idle. |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
apply_processing_statesetsstatus/status_detailbut doesn’t clearerror_messageor resetpaused. If the model was previously in Error or paused demo state, the UI can incorrectly keep showing a stale error/paused flag after transitioning to Processing. Recommend clearingerror_messageand forcingpaused = falsein this transition (similar to other state updates).