Session API changes (#802)

* Expose server version

* expose semver

* agent type helpers

* Allow for multiple listeners on audio switch handler

* Fix crash when cleaning up local participant

* Deprecate Room.withPreconnectAudio

* Change TokenSource.fetch methods to return Result<TokenSourceResponse> to explicitly handle exceptions

* add test utilities

* Fix transcription conversion error

* spotless

Author: davidliu

.changeset/brown-cheetahs-argue.md

@@ -0,0 +1,5 @@
+---
+"client-sdk-android": minor
+---
+
+Change TokenSource.fetch methods to return Result<TokenSourceResponse> to explicitly handle exceptions

.changeset/cold-days-speak.md

@@ -0,0 +1,5 @@
+---
+"client-sdk-android": minor
+---
+
+Add support for multiple listeners on AudioSwitchHandler

.changeset/cool-meals-fetch.md

@@ -0,0 +1,5 @@
+---
+"client-sdk-android": minor
+---
+
+Rename AgentState to AgentSdkState

.changeset/curvy-otters-move.md

@@ -0,0 +1,7 @@
+---
+"client-sdk-android": minor
+---
+
+Deprecate Room.withPreconnectAudio method.
+
+- Set AudioTrackPublishDefaults.preconnect = true on the RoomOptions instead to use the preconnect buffer.

.changeset/gorgeous-carrots-share.md

@@ -0,0 +1,5 @@
+---
+"client-sdk-android": patch
+---
+
+Fix crash when cleaning up local participant

.changeset/spicy-planes-sip.md

@@ -0,0 +1,5 @@
+---
+"client-sdk-android": minor
+---
+
+Expose agentAttributes as a value on Participant

.changeset/spotty-fishes-hang.md

@@ -0,0 +1,5 @@
+---
+"client-sdk-android": minor
+---
+
+Expose the server info of the currently connected server on Room

livekit-android-sdk/build.gradle

@@ -161,7 +161,7 @@ dependencies {
     kapt libs.dagger.compiler
 
     implementation libs.timber
-    implementation libs.semver4j
+    api libs.semver4j
 
     lintChecks project(':livekit-lint')
     lintPublish project(':livekit-lint')

livekit-android-sdk/src/main/java/io/livekit/android/audio/AudioSwitchHandler.kt

@@ -30,6 +30,7 @@ import com.twilio.audioswitch.AudioSwitch
 import com.twilio.audioswitch.LegacyAudioSwitch
 import io.livekit.android.room.Room
 import io.livekit.android.util.LKLog
+import java.util.Collections
 import javax.inject.Inject
 import javax.inject.Singleton
 
@@ -54,15 +55,47 @@ constructor(private val context: Context) : AudioHandler {
      *
      * @see AudioDeviceChangeListener
      */
+    @Deprecated("Use registerAudioDeviceChangeListener.")
     var audioDeviceChangeListener: AudioDeviceChangeListener? = null
 
+    private val audioDeviceChangeListeners = Collections.synchronizedSet(mutableSetOf<AudioDeviceChangeListener>())
+
+    private val audioDeviceChangeDispatcher by lazy(LazyThreadSafetyMode.NONE) {
+        object : AudioDeviceChangeListener {
+            override fun invoke(audioDevices: List<AudioDevice>, selectedAudioDevice: AudioDevice?) {
+                @Suppress("DEPRECATION")
+                audioDeviceChangeListener?.invoke(audioDevices, selectedAudioDevice)
+                synchronized(audioDeviceChangeListeners) {
+                    for (listener in audioDeviceChangeListeners) {
+                        listener.invoke(audioDevices, selectedAudioDevice)
+                    }
+                }
+            }
+        }
+    }
+
     /**
      * Listen to changes in audio focus.
      *
      * @see AudioManager.OnAudioFocusChangeListener
      */
+    @Deprecated("Use registerOnAudioFocusChangeListener.")
     var onAudioFocusChangeListener: AudioManager.OnAudioFocusChangeListener? = null
 
+    private val onAudioFocusChangeListeners = Collections.synchronizedSet(mutableSetOf<AudioManager.OnAudioFocusChangeListener>())
+
+    private val onAudioFocusChangeDispatcher by lazy(LazyThreadSafetyMode.NONE) {
+        AudioManager.OnAudioFocusChangeListener { focusChange ->
+            @Suppress("DEPRECATION")
+            onAudioFocusChangeListener?.onAudioFocusChange(focusChange)
+            synchronized(onAudioFocusChangeListeners) {
+                for (listener in onAudioFocusChangeListeners) {
+                    listener.onAudioFocusChange(focusChange)
+                }
+            }
+        }
+    }
+
     /**
      * The preferred priority of audio devices to use. The first available audio device will be used.
      *
@@ -170,14 +203,14 @@ constructor(private val context: Context) : AudioHandler {
                         AudioSwitch(
                             context = context,
                             loggingEnabled = loggingEnabled,
-                            audioFocusChangeListener = onAudioFocusChangeListener ?: defaultOnAudioFocusChangeListener,
+                            audioFocusChangeListener = onAudioFocusChangeDispatcher,
                             preferredDeviceList = preferredDeviceList ?: defaultPreferredDeviceList,
                         )
                     } else {
                         LegacyAudioSwitch(
                             context = context,
                             loggingEnabled = loggingEnabled,
-                            audioFocusChangeListener = onAudioFocusChangeListener ?: defaultOnAudioFocusChangeListener,
+                            audioFocusChangeListener = onAudioFocusChangeDispatcher,
                             preferredDeviceList = preferredDeviceList ?: defaultPreferredDeviceList,
                         )
                     }
@@ -190,7 +223,7 @@ constructor(private val context: Context) : AudioHandler {
                 switch.forceHandleAudioRouting = forceHandleAudioRouting
 
                 audioSwitch = switch
-                switch.start(audioDeviceChangeListener ?: defaultAudioDeviceChangeListener)
+                switch.start(audioDeviceChangeDispatcher)
                 switch.activate()
             }
         }
@@ -235,16 +268,43 @@ constructor(private val context: Context) : AudioHandler {
         }
     }
 
+    /**
+     * Listen to changes in the available and active audio devices.
+     * @see unregisterAudioDeviceChangeListener
+     */
+    fun registerAudioDeviceChangeListener(listener: AudioDeviceChangeListener) {
+        audioDeviceChangeListeners.add(listener)
+    }
+
+    /**
+     * Remove a previously registered audio device change listener.
+     * @see registerAudioDeviceChangeListener
+     */
+    fun unregisterAudioDeviceChangeListener(listener: AudioDeviceChangeListener) {
+        audioDeviceChangeListeners.remove(listener)
+    }
+
+    /**
+     * Listen to changes in audio focus.
+     *
+     * @see AudioManager.OnAudioFocusChangeListener
+     * @see unregisterOnAudioFocusChangeListener
+     */
+    fun registerOnAudioFocusChangeListener(listener: AudioManager.OnAudioFocusChangeListener) {
+        onAudioFocusChangeListeners.add(listener)
+    }
+
+    /**
+     * Remove a previously registered focus change listener.
+     *
+     * @see AudioManager.OnAudioFocusChangeListener
+     * @see registerOnAudioFocusChangeListener
+     */
+    fun unregisterOnAudioFocusChangeListener(listener: AudioManager.OnAudioFocusChangeListener) {
+        onAudioFocusChangeListeners.remove(listener)
+    }
+
     companion object {
-        private val defaultOnAudioFocusChangeListener by lazy(LazyThreadSafetyMode.NONE) {
-            AudioManager.OnAudioFocusChangeListener { }
-        }
-        private val defaultAudioDeviceChangeListener by lazy(LazyThreadSafetyMode.NONE) {
-            object : AudioDeviceChangeListener {
-                override fun invoke(audioDevices: List<AudioDevice>, selectedAudioDevice: AudioDevice?) {
-                }
-            }
-        }
         private val defaultPreferredDeviceList by lazy(LazyThreadSafetyMode.NONE) {
             listOf(
                 AudioDevice.BluetoothHeadset::class.java,

livekit-android-sdk/src/main/java/io/livekit/android/audio/PreconnectAudioBuffer.kt

@@ -27,9 +27,12 @@ import io.livekit.android.room.datastream.StreamBytesOptions
 import io.livekit.android.room.participant.Participant
 import io.livekit.android.util.LKLog
 import io.livekit.android.util.flow
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Job
 import kotlinx.coroutines.cancel
 import kotlinx.coroutines.coroutineScope
 import kotlinx.coroutines.delay
+import kotlinx.coroutines.ensureActive
 import kotlinx.coroutines.flow.collect
 import kotlinx.coroutines.flow.takeWhile
 import kotlinx.coroutines.launch
@@ -195,8 +198,8 @@ internal constructor(timeout: Duration) : AudioTrackSink {
  * use with LiveKit Agents.
  * @param onError The error handler to call when an error occurs while sending the audio buffer.
  * @param operation The connection lambda to call with the pre-connect audio.
- *
  */
+@Deprecated("Set AudioTrackPublishDefaults.preconnect = true on the RoomOptions instead.")
 suspend fun <T> Room.withPreconnectAudio(
     timeout: Duration = TIMEOUT,
     topic: String = DEFAULT_TOPIC,
@@ -298,3 +301,104 @@ suspend fun <T> Room.withPreconnectAudio(
 
     return@coroutineScope retValue
 }
+
+internal suspend fun Room.startPreconnectAudioJob(
+    roomScope: CoroutineScope,
+    timeout: Duration = TIMEOUT,
+    topic: String = DEFAULT_TOPIC
+): () -> Unit {
+    isPrerecording = true
+    val audioTrack = localParticipant.getOrCreateDefaultAudioTrack()
+    val preconnectAudioBuffer = PreconnectAudioBuffer(timeout)
+
+    LKLog.v { "Starting preconnect audio buffer" }
+    preconnectAudioBuffer.startRecording()
+    audioTrack.addSink(preconnectAudioBuffer)
+    audioTrack.prewarm()
+
+    val jobs = mutableListOf<Job>()
+    fun stopRecording() {
+        if (!isPrerecording) {
+            return
+        }
+
+        LKLog.v { "Stopping preconnect audio buffer" }
+        audioTrack.removeSink(preconnectAudioBuffer)
+        preconnectAudioBuffer.stopRecording()
+        isPrerecording = false
+    }
+
+    // Clear the preconnect audio buffer after the timeout to free memory.
+    roomScope.launch {
+        delay(TIMEOUT)
+        preconnectAudioBuffer.clear()
+    }
+
+    val sentIdentities = mutableSetOf<Participant.Identity>()
+    roomScope.launch {
+        suspend fun handleSendIfNeeded(participant: Participant) {
+            coroutineScope inner@{
+                engine::connectionState.flow
+                    .takeWhile { it != ConnectionState.CONNECTED }
+                    .collect()
+
+                ensureActive()
+                val kind = participant.kind
+                val state = participant.state
+                val identity = participant.identity
+                if (sentIdentities.contains(identity) || kind != Participant.Kind.AGENT || state != Participant.State.ACTIVE || identity == null) {
+                    return@inner
+                }
+
+                stopRecording()
+                launch {
+                    try {
+                        preconnectAudioBuffer.sendAudioData(
+                            room = this@startPreconnectAudioJob,
+                            trackSid = audioTrack.sid,
+                            agentIdentities = listOf(identity),
+                            topic = topic,
+                        )
+                        sentIdentities.add(identity)
+                    } catch (e: Exception) {
+                        LKLog.w(e) { "Error occurred while sending the audio preconnect data." }
+                    }
+                }
+            }
+        }
+
+        events.collect { event ->
+            when (event) {
+                is RoomEvent.LocalTrackSubscribed -> {
+                    LKLog.i { "Local audio track has been subscribed to, stopping preconnect audio recording." }
+                    stopRecording()
+                }
+
+                is RoomEvent.ParticipantConnected -> {
+                    // agents may connect with ACTIVE state and not trigger a participant state changed.
+                    handleSendIfNeeded(event.participant)
+                }
+
+                is RoomEvent.ParticipantStateChanged -> {
+                    handleSendIfNeeded(event.participant)
+                }
+
+                is RoomEvent.Disconnected -> {
+                    cancel()
+                }
+
+                else -> {
+                    // Intentionally blank.
+                }
+            }
+        }
+    }
+
+    return cancelPrerecord@{
+        if (!isPrerecording) {
+            return@cancelPrerecord
+        }
+        jobs.forEach { it.cancel() }
+        preconnectAudioBuffer.clear()
+    }
+}