Support more input transcription parameters for openai realtime (#1637)

Author: adambenali

.changeset/fresh-foxes-remember.md

@@ -0,0 +1,5 @@
+---
+"livekit-plugins-openai": patch
+---
+
+Support more input transcription parameters for openai realtime

livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/api_proto.py

@@ -69,6 +69,8 @@ class ContentPart(TypedDict):
 
 class InputAudioTranscription(TypedDict):
     model: InputTranscriptionModel | str
+    language: NotRequired[str]
+    prompt: NotRequired[str]
 
 
 class ServerVad(TypedDict):

livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py

@@ -149,6 +149,8 @@ class ServerVadOptions:
 @dataclass
 class InputTranscriptionOptions:
     model: api_proto.InputTranscriptionModel | str
+    language: str | None = None
+    prompt: str | None = None
 
 
 @dataclass
@@ -976,6 +978,14 @@ def session_update(
             input_audio_transcription_opts = {
                 "model": self._opts.input_audio_transcription.model,
             }
+            if self._opts.input_audio_transcription.language is not None:
+                input_audio_transcription_opts["language"] = (
+                    self._opts.input_audio_transcription.language
+                )
+            if self._opts.input_audio_transcription.prompt is not None:
+                input_audio_transcription_opts["prompt"] = (
+                    self._opts.input_audio_transcription.prompt
+                )
 
         session_data: api_proto.ClientEvent.SessionUpdateData = {
             "modalities": self._opts.modalities,
@@ -1296,6 +1306,8 @@ def _handle_session_updated(
         else:
             input_audio_transcription = InputTranscriptionOptions(
                 model=session["input_audio_transcription"]["model"],
+                language=session["input_audio_transcription"].get("language"),
+                prompt=session["input_audio_transcription"].get("prompt"),
             )
 
         self.emit(

livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/stt.py

@@ -39,6 +39,7 @@ class _STTOptions:
     language: str
     detect_language: bool
     model: WhisperModels | str
+    prompt: str | None = None
 
 
 class STT(stt.STT):
@@ -48,6 +49,7 @@ def __init__(
         language: str = "en",
         detect_language: bool = False,
         model: WhisperModels | str = "whisper-1",
+        prompt: str | None = None,
         base_url: str | None = None,
         api_key: str | None = None,
         client: openai.AsyncClient | None = None,
@@ -69,6 +71,7 @@ def __init__(
             language=language,
             detect_language=detect_language,
             model=model,
+            prompt=prompt,
         )
 
         self._client = client or openai.AsyncClient(
@@ -91,9 +94,11 @@ def update_options(
         *,
         model: WhisperModels | GroqAudioModels | None = None,
         language: str | None = None,
+        prompt: str | None = None,
     ) -> None:
         self._opts.model = model or self._opts.model
         self._opts.language = language or self._opts.language
+        self._opts.prompt = prompt or self._opts.prompt
 
     @staticmethod
     def with_groq(
@@ -103,6 +108,7 @@ def with_groq(
         base_url: str | None = "https://api.groq.com/openai/v1",
         client: openai.AsyncClient | None = None,
         language: str = "en",
+        prompt: str | None = None,
         detect_language: bool = False,
     ) -> STT:
         """
@@ -123,6 +129,7 @@ def with_groq(
             client=client,
             language=language,
             detect_language=detect_language,
+            prompt=prompt,
         )
 
     def _sanitize_options(self, *, language: str | None = None) -> _STTOptions:
@@ -140,6 +147,9 @@ async def _recognize_impl(
         try:
             config = self._sanitize_options(language=language)
             data = rtc.combine_audio_frames(buffer).to_wav_bytes()
+            prompt = (
+                self._opts.prompt if self._opts.prompt is not None else openai.NOT_GIVEN
+            )
             resp = await self._client.audio.transcriptions.create(
                 file=(
                     "file.wav",
@@ -148,6 +158,7 @@ async def _recognize_impl(
                 ),
                 model=self._opts.model,
                 language=config.language,
+                prompt=prompt,
                 # verbose_json returns language and other details
                 response_format="verbose_json",
                 timeout=httpx.Timeout(30, connect=conn_options.timeout),