Detect screenshare rotation (#552)

davidliu · web-flow · commit dcbb36b25a31 · 2024-12-10T13:08:30.000+09:00
* Update gradle and build tools

* Update Kotlin to 1.9.25

* Add separate default capture/publish options for screenshare tracks

* Detect rotation for screenshare tracks

* spotless

* Update robolectric to 4.14.1
diff --git a/.changeset/nice-clocks-refuse.md b/.changeset/nice-clocks-refuse.md
@@ -0,0 +1,5 @@
+---
+"client-sdk-android": minor
+---
+
+Detect rotation for screenshare tracks
diff --git a/.changeset/stale-ways-bathe.md b/.changeset/stale-ways-bathe.md
@@ -0,0 +1,5 @@
+---
+"client-sdk-android": patch
+---
+
+Update Kotlin dependency to 1.9.25
diff --git a/.changeset/thirty-readers-lie.md b/.changeset/thirty-readers-lie.md
@@ -0,0 +1,5 @@
+---
+"client-sdk-android": minor
+---
+
+Add separate default capture/publish options for screenshare tracks
diff --git a/.idea/kotlinc.xml b/.idea/kotlinc.xml
diff --git a/deps.gradle b/deps.gradle
@@ -1,13 +1,13 @@
 ext {
-    android_build_tools_version = '8.2.2'
+    android_build_tools_version = '8.7.2'
     compose_version = '1.2.1'
-    compose_compiler_version = '1.4.5'
-    kotlin_version = '1.8.20'
+    compose_compiler_version = '1.5.15'
+    kotlin_version = '1.9.25'
     java_version = JavaVersion.VERSION_1_8
-    dokka_version = '1.8.20'
+    dokka_version = '1.9.20'
     androidSdk = [
-        compileVersion: 34,
-        targetVersion : 34,
+        compileVersion: 35,
+        targetVersion : 35,
         minVersion    : 21,
     ]
     generated = [
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -92,7 +92,7 @@ mockito-kotlin = { module = "org.mockito.kotlin:mockito-kotlin", version = "4.1.
 #noinspection GradleDependency
 mockito-inline = { module = "org.mockito:mockito-inline", version = "4.11.0" }
 
-robolectric = { module = "org.robolectric:robolectric", version = "4.11.1" }
+robolectric = { module = "org.robolectric:robolectric", version = "4.14.1" }
 turbine = { module = "app.cash.turbine:turbine", version = "1.0.0" }
 appcompat = { group = "androidx.appcompat", name = "appcompat", version.ref = "appcompat" }
 material = { group = "com.google.android.material", name = "material", version.ref = "material" }
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
@@ -1,6 +1,6 @@
 #Mon May 01 22:58:53 JST 2023
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
diff --git a/livekit-android-sdk/build.gradle b/livekit-android-sdk/build.gradle
@@ -52,8 +52,9 @@ android {
         targetCompatibility java_version
     }
     packagingOptions {
-        // Exclude our protos from being included in the final aar.
-        exclude "**/*.proto"
+        resources {
+            excludes += ['**/*.proto']
+        }
     }
 
     buildFeatures {
diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/RoomOptions.kt b/livekit-android-sdk/src/main/java/io/livekit/android/RoomOptions.kt
@@ -43,4 +43,6 @@ data class RoomOptions(
     val videoTrackCaptureDefaults: LocalVideoTrackOptions? = null,
     val audioTrackPublishDefaults: AudioTrackPublishDefaults? = null,
     val videoTrackPublishDefaults: VideoTrackPublishDefaults? = null,
+    val screenShareTrackCaptureDefaults: LocalVideoTrackOptions? = null,
+    val screenShareTrackPublishDefaults: VideoTrackPublishDefaults? = null,
 )
diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/DefaultsManager.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/DefaultsManager.kt
@@ -1,5 +1,5 @@
 /*
- * Copyright 2023 LiveKit, Inc.
+ * Copyright 2023-2024 LiveKit, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ import io.livekit.android.room.participant.AudioTrackPublishDefaults
 import io.livekit.android.room.participant.VideoTrackPublishDefaults
 import io.livekit.android.room.track.LocalAudioTrackOptions
 import io.livekit.android.room.track.LocalVideoTrackOptions
+import io.livekit.android.room.track.ScreenSharePresets
 import javax.inject.Inject
 import javax.inject.Singleton
 
@@ -31,4 +32,6 @@ constructor() {
     var audioTrackPublishDefaults: AudioTrackPublishDefaults = AudioTrackPublishDefaults()
     var videoTrackCaptureDefaults: LocalVideoTrackOptions = LocalVideoTrackOptions()
     var videoTrackPublishDefaults: VideoTrackPublishDefaults = VideoTrackPublishDefaults()
+    var screenShareTrackCaptureDefaults: LocalVideoTrackOptions = LocalVideoTrackOptions(isScreencast = true, captureParams = ScreenSharePresets.ORIGINAL.capture)
+    var screenShareTrackPublishDefaults: VideoTrackPublishDefaults = VideoTrackPublishDefaults(videoEncoding = ScreenSharePresets.ORIGINAL.encoding)
 }
diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/Room.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/Room.kt
@@ -251,6 +251,16 @@ constructor(
      */
     var videoTrackPublishDefaults: VideoTrackPublishDefaults by defaultsManager::videoTrackPublishDefaults
 
+    /**
+     * Default options to use when creating a screen share track.
+     */
+    var screenShareTrackCaptureDefaults: LocalVideoTrackOptions by defaultsManager::screenShareTrackCaptureDefaults
+
+    /**
+     * Default options to use when publishing a screen share track.
+     */
+    var screenShareTrackPublishDefaults: VideoTrackPublishDefaults by defaultsManager::screenShareTrackPublishDefaults
+
     val localParticipant: LocalParticipant = localParticipantFactory.create(dynacast = false).apply {
         internalListener = this@Room
     }
@@ -285,11 +295,13 @@ constructor(
         RoomOptions(
             adaptiveStream = adaptiveStream,
             dynacast = dynacast,
+            e2eeOptions = e2eeOptions,
             audioTrackCaptureDefaults = audioTrackCaptureDefaults,
             videoTrackCaptureDefaults = videoTrackCaptureDefaults,
             audioTrackPublishDefaults = audioTrackPublishDefaults,
             videoTrackPublishDefaults = videoTrackPublishDefaults,
-            e2eeOptions = e2eeOptions,
+            screenShareTrackCaptureDefaults = screenShareTrackCaptureDefaults,
+            screenShareTrackPublishDefaults = screenShareTrackPublishDefaults,
         )
 
     /**
@@ -502,6 +514,12 @@ constructor(
         options.videoTrackPublishDefaults?.let {
             videoTrackPublishDefaults = it
         }
+        options.screenShareTrackCaptureDefaults?.let {
+            screenShareTrackCaptureDefaults = it
+        }
+        options.screenShareTrackPublishDefaults?.let {
+            screenShareTrackPublishDefaults = it
+        }
         adaptiveStream = options.adaptiveStream
         dynacast = options.dynacast
         e2eeOptions = options.e2eeOptions
diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt
@@ -93,6 +93,8 @@ internal constructor(
     var audioTrackPublishDefaults: AudioTrackPublishDefaults by defaultsManager::audioTrackPublishDefaults
     var videoTrackCaptureDefaults: LocalVideoTrackOptions by defaultsManager::videoTrackCaptureDefaults
     var videoTrackPublishDefaults: VideoTrackPublishDefaults by defaultsManager::videoTrackPublishDefaults
+    var screenShareTrackCaptureDefaults: LocalVideoTrackOptions by defaultsManager::screenShareTrackCaptureDefaults
+    var screenShareTrackPublishDefaults: VideoTrackPublishDefaults by defaultsManager::screenShareTrackPublishDefaults
 
     private var republishes: List<LocalTrackPublication>? = null
     private val localTrackPublications
@@ -181,13 +183,13 @@ internal constructor(
      * @param name The name of the track.
      * @param mediaProjectionPermissionResultData The resultData returned from launching
      * [MediaProjectionManager.createScreenCaptureIntent()](https://developer.android.com/reference/android/media/projection/MediaProjectionManager#createScreenCaptureIntent()).
-     * @param options The capture options to use for this track, or [Room.videoTrackCaptureDefaults] if none is passed.
+     * @param options The capture options to use for this track, or [Room.screenShareTrackCaptureDefaults] if none is passed.
      * @param videoProcessor A video processor to attach to this track that can modify the frames before publishing.
      */
     fun createScreencastTrack(
         name: String = "",
         mediaProjectionPermissionResultData: Intent,
-        options: LocalVideoTrackOptions = videoTrackCaptureDefaults.copy(),
+        options: LocalVideoTrackOptions = screenShareTrackCaptureDefaults.copy(),
         videoProcessor: VideoProcessor? = null,
     ): LocalScreencastVideoTrack {
         val screencastOptions = options.copy(isScreencast = true)
@@ -249,8 +251,8 @@ internal constructor(
      * @param mediaProjectionPermissionResultData The resultData returned from launching
      * [MediaProjectionManager.createScreenCaptureIntent()](https://developer.android.com/reference/android/media/projection/MediaProjectionManager#createScreenCaptureIntent()).
      * @throws IllegalArgumentException if attempting to enable screenshare without [mediaProjectionPermissionResultData]
-     * @see Room.videoTrackCaptureDefaults
-     * @see Room.videoTrackPublishDefaults
+     * @see Room.screenShareTrackCaptureDefaults
+     * @see Room.screenShareTrackPublishDefaults
      */
     suspend fun setScreenShareEnabled(
         enabled: Boolean,
@@ -294,7 +296,7 @@ internal constructor(
                                 createScreencastTrack(mediaProjectionPermissionResultData = mediaProjectionPermissionResultData)
                             track.startForegroundService(null, null)
                             track.startCapture()
-                            publishVideoTrack(track)
+                            publishVideoTrack(track, options = VideoTrackPublishOptions(null, screenShareTrackPublishDefaults))
                         }
 
                         else -> {
diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalScreencastVideoTrack.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalScreencastVideoTrack.kt
@@ -22,15 +22,35 @@ import android.app.NotificationManager
 import android.content.Context
 import android.content.Intent
 import android.media.projection.MediaProjection
+import android.util.DisplayMetrics
+import android.view.OrientationEventListener
+import android.view.WindowManager
 import dagger.assisted.Assisted
 import dagger.assisted.AssistedFactory
 import dagger.assisted.AssistedInject
 import io.livekit.android.room.DefaultsManager
+import io.livekit.android.room.participant.LocalParticipant
 import io.livekit.android.room.track.screencapture.ScreenCaptureConnection
 import io.livekit.android.room.track.screencapture.ScreenCaptureService
-import livekit.org.webrtc.*
-import java.util.*
+import io.livekit.android.util.LKLog
+import livekit.org.webrtc.EglBase
+import livekit.org.webrtc.PeerConnectionFactory
+import livekit.org.webrtc.ScreenCapturerAndroid
+import livekit.org.webrtc.SurfaceTextureHelper
+import livekit.org.webrtc.VideoCapturer
+import livekit.org.webrtc.VideoProcessor
+import livekit.org.webrtc.VideoSource
+import java.util.UUID
 
+/**
+ * A video track that captures the screen for publishing.
+ *
+ * Note: A foreground service is generally required for use. Use [startForegroundService] or start
+ * your own foreground service before starting the video track.
+ *
+ * @see LocalParticipant.createScreencastTrack
+ * @see LocalScreencastVideoTrack.startForegroundService
+ */
 class LocalScreencastVideoTrack
 @AssistedInject
 constructor(
@@ -58,6 +78,76 @@ constructor(
     videoTrackFactory,
 ) {
 
+    private var prevDisplayWidth = 0
+    private var prevDisplayHeight = 0
+    private val displayMetrics = DisplayMetrics()
+    private val windowManager = context.getSystemService(Context.WINDOW_SERVICE) as WindowManager
+    private val orientationEventListener = object : OrientationEventListener(context) {
+        override fun onOrientationChanged(orientation: Int) {
+            if (isDisposed) {
+                this.disable()
+                return
+            }
+            updateCaptureFormatIfNeeded()
+        }
+    }
+
+    private fun getCaptureDimensions(displayWidth: Int, displayHeight: Int): Pair<Int, Int> {
+        val captureWidth: Int
+        val captureHeight: Int
+
+        if (options.captureParams.width == 0 && options.captureParams.height == 0) {
+            // Use raw display size
+            captureWidth = displayWidth
+            captureHeight = displayHeight
+        } else {
+            // Use captureParams.width as longest side and captureParams.height as shortest side.
+            if (displayWidth > displayHeight) {
+                captureWidth = options.captureParams.width
+                captureHeight = options.captureParams.height
+            } else {
+                captureWidth = options.captureParams.height
+                captureHeight = options.captureParams.width
+            }
+        }
+
+        return captureWidth to captureHeight
+    }
+
+    private fun updateCaptureFormatIfNeeded() {
+        windowManager.defaultDisplay.getRealMetrics(displayMetrics)
+        val displayWidth = displayMetrics.widthPixels
+        val displayHeight = displayMetrics.heightPixels
+
+        // Updates whenever the display rotates
+        if (displayWidth != prevDisplayWidth || displayHeight != prevDisplayHeight) {
+            prevDisplayWidth = displayWidth
+            prevDisplayHeight = displayHeight
+
+            val (captureWidth, captureHeight) = getCaptureDimensions(displayWidth, displayHeight)
+
+            try {
+                capturer.changeCaptureFormat(captureWidth, captureHeight, options.captureParams.maxFps)
+            } catch (e: Exception) {
+                LKLog.w(e) { "Exception when changing capture format of the screen share track." }
+            }
+        }
+    }
+
+    override fun startCapture() {
+        // Don't use super.startCapture, must calculate correct dimensions
+        windowManager.defaultDisplay.getRealMetrics(displayMetrics)
+        val displayWidth = displayMetrics.widthPixels
+        val displayHeight = displayMetrics.heightPixels
+        val (captureWidth, captureHeight) = getCaptureDimensions(displayWidth, displayHeight)
+
+        capturer.startCapture(captureWidth, captureHeight, options.captureParams.maxFps)
+
+        if (orientationEventListener.canDetectOrientation()) {
+            orientationEventListener.enable()
+        }
+    }
+
     private val serviceConnection = ScreenCaptureConnection(context)
 
     init {
@@ -95,6 +185,7 @@ constructor(
     override fun stop() {
         super.stop()
         serviceConnection.stop()
+        orientationEventListener.disable()
     }
 
     @AssistedFactory
@@ -129,7 +220,7 @@ constructor(
             options: LocalVideoTrackOptions,
             rootEglBase: EglBase,
             screencastVideoTrackFactory: Factory,
-            videoProcessor: VideoProcessor?
+            videoProcessor: VideoProcessor?,
         ): LocalScreencastVideoTrack {
             val source = peerConnectionFactory.createVideoSource(options.isScreencast)
             source.setVideoProcessor(videoProcessor)
diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalVideoTrackOptions.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalVideoTrackOptions.kt
@@ -172,3 +172,48 @@ enum class VideoPreset43(
         VideoEncoding(3_800_000, 30),
     ),
 }
+
+/**
+ * 16:9 Video presets along with suggested bitrates.
+ */
+enum class ScreenSharePresets(
+    override val capture: VideoCaptureParameter,
+    override val encoding: VideoEncoding,
+) : VideoPreset {
+    H360_FPS3(
+        VideoCaptureParameter(640, 360, 3),
+        VideoEncoding(200_000, 3),
+    ),
+    H360_FPS15(
+        VideoCaptureParameter(640, 360, 15),
+        VideoEncoding(400_000, 15),
+    ),
+    H720_FPS5(
+        VideoCaptureParameter(1280, 720, 5),
+        VideoEncoding(800_000, 5),
+    ),
+    H720_FPS15(
+        VideoCaptureParameter(1280, 720, 15),
+        VideoEncoding(1_500_000, 15),
+    ),
+    H720_FPS30(
+        VideoCaptureParameter(1280, 720, 30),
+        VideoEncoding(2_000_000, 30),
+    ),
+    H1080_FPS15(
+        VideoCaptureParameter(1920, 1080, 15),
+        VideoEncoding(2_500_000, 15),
+    ),
+    H1080_FPS30(
+        VideoCaptureParameter(1920, 1080, 30),
+        VideoEncoding(5_000_000, 30),
+    ),
+
+    /**
+     * Uses the original resolution without resizing.
+     */
+    ORIGINAL(
+        VideoCaptureParameter(0, 0, 30),
+        VideoEncoding(7_000_000, 30),
+    )
+}
diff --git a/livekit-android-test/build.gradle b/livekit-android-test/build.gradle
@@ -17,9 +17,6 @@ android {
         consumerProguardFiles "consumer-rules.pro"
     }
 
-    lintOptions {
-        disable 'VisibleForTests'
-    }
     buildTypes {
         release {
             minifyEnabled false
@@ -39,6 +36,9 @@ android {
             includeAndroidResources = true
         }
     }
+    lint {
+        disable 'VisibleForTests'
+    }
 }
 
 dokkaHtml {
diff --git a/livekit-android-test/src/main/java/io/livekit/android/test/mock/MockAudioProcessingController.kt b/livekit-android-test/src/main/java/io/livekit/android/test/mock/MockAudioProcessingController.kt

-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +---
 +"client-sdk-android": minor
 +---
++
 +Detect rotation for screenshare tracks
Original file line number	Diff line number	Diff line change
`@@ -52,8 +52,9 @@ android {`
`52`	`52`	`targetCompatibility java_version`
`53`	`53`	`}`
`54`	`54`	`packagingOptions {`
`55`		`- // Exclude our protos from being included in the final aar.`
`56`		`- exclude "*/.proto"`
	`55`	`+ resources {`
	`56`	`+ excludes += ['*/.proto']`
	`57`	`+ }`
`57`	`58`	`}`
`58`	`59`
`59`	`60`	`buildFeatures {`
Original file line number	Diff line number	Diff line change
`@@ -43,4 +43,6 @@ data class RoomOptions(`
`43`	`43`	`val videoTrackCaptureDefaults: LocalVideoTrackOptions? = null,`
`44`	`44`	`val audioTrackPublishDefaults: AudioTrackPublishDefaults? = null,`
`45`	`45`	`val videoTrackPublishDefaults: VideoTrackPublishDefaults? = null,`
	`46`	`+ val screenShareTrackCaptureDefaults: LocalVideoTrackOptions? = null,`
	`47`	`+ val screenShareTrackPublishDefaults: VideoTrackPublishDefaults? = null,`
`46`	`48`	`)`