Add LiveAPI Transcription

Author: David Motsonashvili

firebase-ai/src/main/kotlin/com/google/firebase/ai/LiveGenerativeModel.kt

@@ -111,7 +111,9 @@ internal constructor(
           modelName,
           config?.toInternal(),
           tools?.map { it.toInternal() },
-          systemInstruction?.toInternal()
+          systemInstruction?.toInternal(),
+          config?.inputAudioTranscription?.toInternal(),
+          config?.outputAudioTranscription?.toInternal()
         )
         .toInternal()
     val data: String = Json.encodeToString(clientMessage)
@@ -135,7 +137,7 @@ internal constructor(
     } catch (e: ClosedReceiveChannelException) {
       val reason = webSession?.closeReason?.await()
       val message =
-        "Channel was closed by the server.${if(reason!=null) " Details: ${reason.message}" else "" }"
+        "Channel was closed by the server.${if (reason != null) " Details: ${reason.message}" else ""}"
       throw ServiceConnectionHandshakeFailedException(message, e)
     }
   }

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/AudioTranscriptionConfig.kt

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.firebase.ai.type
+
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.json.JsonObject
+
+/**
+ * The audio transcription configuration.
+ * @property enable If true, the server will use Gemini to transcribe the audio.
+ * @property prefixPrompt Prefix prompt for the audio transcription op. This is useful to override
+ * the default prefix prompt that only asks the model to transcribe the audio. Overriding can be
+ * useful to provide additional context to the model such as what language is expected to be spoken
+ * in the audio.
+ */
+public class AudioTranscriptionConfig(
+  internal val enable: Boolean? = null,
+  internal val prefixPrompt: String? = null
+) {
+
+  internal fun toInternal() = JsonObject(emptyMap())
+
+}

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveClientSetupMessage.kt

@@ -18,6 +18,7 @@ package com.google.firebase.ai.type
 
 import kotlinx.serialization.ExperimentalSerializationApi
 import kotlinx.serialization.Serializable
+import kotlinx.serialization.json.JsonObject
 
 /**
  * First message in a live session.
@@ -32,7 +33,9 @@ internal class LiveClientSetupMessage(
   // needs its own config class
   val generationConfig: LiveGenerationConfig.Internal?,
   val tools: List<Tool.Internal>?,
-  val systemInstruction: Content.Internal?
+  val systemInstruction: Content.Internal?,
+  val inputAudioTranscription: JsonObject?,
+  val outputAudioTranscription: JsonObject?,
 ) {
   @Serializable
   internal class Internal(val setup: LiveClientSetup) {
@@ -41,10 +44,21 @@ internal class LiveClientSetupMessage(
       val model: String,
       val generationConfig: LiveGenerationConfig.Internal?,
       val tools: List<Tool.Internal>?,
-      val systemInstruction: Content.Internal?
+      val systemInstruction: Content.Internal?,
+      val inputAudioTranscription: JsonObject?,
+      val outputAudioTranscription: JsonObject?,
     )
   }
 
   fun toInternal() =
-    Internal(Internal.LiveClientSetup(model, generationConfig, tools, systemInstruction))
+    Internal(
+      Internal.LiveClientSetup(
+        model,
+        generationConfig,
+        tools,
+        systemInstruction,
+        inputAudioTranscription,
+        outputAudioTranscription
+      )
+    )
 }

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveGenerationConfig.kt

@@ -53,6 +53,11 @@ import kotlinx.serialization.Serializable
  *
  * @property speechConfig Specifies the voice configuration of the audio response from the server.
  *
+ * @property inputAudioTranscription Specifies the configuration for transcribing input audio.
+ *
+ * @property outputAudioTranscription Specifies the configuration for transcribing output audio from
+ * the model.
+ *
  * Refer to the
  * [Control generated output](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/control-generated-output)
  * guide for more details.
@@ -67,7 +72,9 @@ private constructor(
   internal val presencePenalty: Float?,
   internal val frequencyPenalty: Float?,
   internal val responseModality: ResponseModality?,
-  internal val speechConfig: SpeechConfig?
+  internal val speechConfig: SpeechConfig?,
+  internal val inputAudioTranscription: AudioTranscriptionConfig?,
+  internal val outputAudioTranscription: AudioTranscriptionConfig?,
 ) {
 
   /**
@@ -91,6 +98,10 @@ private constructor(
    * @property responseModality See [LiveGenerationConfig.responseModality]
    *
    * @property speechConfig See [LiveGenerationConfig.speechConfig]
+   *
+   * @property inputAudioTranscription see [LiveGenerationConfig.inputAudioTranscription]
+   *
+   * @property outputAudioTranscription see [LiveGenerationConfig.outputAudioTranscription]
    */
   public class Builder {
     @JvmField public var temperature: Float? = null
@@ -101,6 +112,8 @@ private constructor(
     @JvmField public var frequencyPenalty: Float? = null
     @JvmField public var responseModality: ResponseModality? = null
     @JvmField public var speechConfig: SpeechConfig? = null
+    @JvmField public var inputAudioTranscription: AudioTranscriptionConfig? = null
+    @JvmField public var outputAudioTranscription: AudioTranscriptionConfig? = null
 
     public fun setTemperature(temperature: Float?): Builder = apply {
       this.temperature = temperature
@@ -123,6 +136,14 @@ private constructor(
       this.speechConfig = speechConfig
     }
 
+    public fun setInputAudioTranscript(config: AudioTranscriptionConfig?): Builder = apply {
+      this.inputAudioTranscription = config
+    }
+
+    public fun setOutputAudioTranscript(config: AudioTranscriptionConfig?): Builder = apply {
+      this.outputAudioTranscription = config
+    }
+
     /** Create a new [LiveGenerationConfig] with the attached arguments. */
     public fun build(): LiveGenerationConfig =
       LiveGenerationConfig(
@@ -133,7 +154,9 @@ private constructor(
         presencePenalty = presencePenalty,
         frequencyPenalty = frequencyPenalty,
         speechConfig = speechConfig,
-        responseModality = responseModality
+        responseModality = responseModality,
+        inputAudioTranscription = inputAudioTranscription,
+        outputAudioTranscription = outputAudioTranscription,
       )
   }
 

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/LiveServerMessage.kt

@@ -82,24 +82,40 @@ public class LiveServerContent(
    * [interrupted] -> [turnComplete].
    */
   public val generationComplete: Boolean,
+
+  /**
+   * The input transcription. The transcription is independent to the model turn which means it
+   * doesn't imply any ordering between transcription and model turn.
+   */
+  public val inputTranscription: Transcription?,
+
+  /**
+   * The output transcription. The transcription is independent to the model turn which means it
+   * doesn't imply any ordering between transcription and model turn.
+   */
+  public val outputTranscription: Transcription?
 ) : LiveServerMessage {
   @OptIn(ExperimentalSerializationApi::class)
   @Serializable
   internal data class Internal(
-    val modelTurn: Content.Internal? = null,
-    val interrupted: Boolean = false,
-    val turnComplete: Boolean = false,
-    val generationComplete: Boolean = false
+    val modelTurn: Content.Internal?,
+    val interrupted: Boolean?,
+    val turnComplete: Boolean?,
+    val generationComplete: Boolean?,
+    val inputTranscription: Transcription.Internal?,
+    val outputTranscription: Transcription.Internal?
   )
   @Serializable
   internal data class InternalWrapper(val serverContent: Internal) : InternalLiveServerMessage {
     @OptIn(ExperimentalSerializationApi::class)
     override fun toPublic() =
       LiveServerContent(
         serverContent.modelTurn?.toPublic(),
-        serverContent.interrupted,
-        serverContent.turnComplete,
-        serverContent.generationComplete
+        serverContent.interrupted ?: false,
+        serverContent.turnComplete?: false,
+        serverContent.generationComplete?: false,
+        serverContent.inputTranscription?.toPublic(),
+        serverContent.outputTranscription?.toPublic()
       )
   }
 }

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Transcription.kt

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.firebase.ai.type
+
+import kotlinx.serialization.Serializable
+
+/**
+ * Audio transcription message.
+ * @property text Transcription text
+ */
+public class Transcription(public val text: String?) {
+
+  @Serializable
+  internal data class Internal(val text: String?) {
+    fun toPublic(): Transcription {
+      return Transcription(text)
+    }
+  }
+}