Skip to content

Commit df652cf

Browse files
davidliudavidliu
authored
Default to scaling and cropping camera output to fit desired dimensions (#558)
1 parent d5464db commit df652cf

File tree

5 files changed

+207
-4
lines changed

5 files changed

+207
-4
lines changed

‎.changeset/odd-zoos-dance.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
"client-sdk-android": minor
3+
---
4+
5+
Default to scaling and cropping camera output to fit desired dimensions
6+
7+
* This behavior may be turned off through the `VideoCaptureParams.adaptOutputToDimensions`

‎livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalVideoTrack.kt‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import io.livekit.android.room.track.video.CameraCapturerUtils.findCamera
3232
import io.livekit.android.room.track.video.CameraCapturerUtils.getCameraPosition
3333
import io.livekit.android.room.track.video.CameraCapturerWithSize
3434
import io.livekit.android.room.track.video.CaptureDispatchObserver
35+
import io.livekit.android.room.track.video.ScaleCropVideoProcessor
3536
import io.livekit.android.room.track.video.VideoCapturerWithSize
3637
import io.livekit.android.room.util.EncodingUtils
3738
import io.livekit.android.util.FlowObservable
@@ -473,11 +474,22 @@ constructor(
473474
videoProcessor: VideoProcessor? = null,
474475
): LocalVideoTrack {
475476
val source = peerConnectionFactory.createVideoSource(options.isScreencast)
476-
source.setVideoProcessor(videoProcessor)
477+
478+
val finalVideoProcessor = if (options.captureParams.adaptOutputToDimensions) {
479+
ScaleCropVideoProcessor(
480+
targetWidth = options.captureParams.width,
481+
targetHeight = options.captureParams.height,
482+
).apply {
483+
childVideoProcessor = videoProcessor
484+
}
485+
} else {
486+
videoProcessor
487+
}
488+
source.setVideoProcessor(finalVideoProcessor)
477489

478490
val surfaceTextureHelper = SurfaceTextureHelper.create("VideoCaptureThread", rootEglBase.eglBaseContext)
479491

480-
// Dispatch raw frames to local renderer only if not using a VideoProcessor.
492+
// Dispatch raw frames to local renderer only if not using a user-provided VideoProcessor.
481493
val dispatchObserver = if (videoProcessor == null) {
482494
CaptureDispatchObserver().apply {
483495
registerObserver(source.capturerObserver)

‎livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalVideoTrackOptions.kt‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,27 @@ data class LocalVideoTrackOptions(
2929
val captureParams: VideoCaptureParameter = VideoPreset169.H720.capture,
3030
)
3131

32-
data class VideoCaptureParameter(
32+
data class VideoCaptureParameter
33+
@JvmOverloads
34+
constructor(
35+
/**
36+
* Desired width.
37+
*/
3338
val width: Int,
39+
/**
40+
* Desired height.
41+
*/
3442
val height: Int,
43+
/**
44+
* Capture frame rate.
45+
*/
3546
val maxFps: Int,
47+
/**
48+
* Sometimes the capturer may not support the exact desired dimensions requested.
49+
* If this is enabled, it will scale down and crop the captured frames to the
50+
* same aspect ratio as [width]:[height].
51+
*/
52+
val adaptOutputToDimensions: Boolean = true,
3653
)
3754

3855
data class VideoEncoding(
@@ -213,7 +230,7 @@ enum class ScreenSharePresets(
213230
* Uses the original resolution without resizing.
214231
*/
215232
ORIGINAL(
216-
VideoCaptureParameter(0, 0, 30),
233+
VideoCaptureParameter(0, 0, 30, adaptOutputToDimensions = false),
217234
VideoEncoding(7_000_000, 30),
218235
)
219236
}

‎livekit-android-sdk/src/main/java/io/livekit/android/room/track/video/ChainVideoProcessor.kt‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
/*
2+
* Copyright 2024 LiveKit, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package io.livekit.android.room.track.video
18+
19+
import androidx.annotation.CallSuper
20+
import livekit.org.webrtc.VideoFrame
21+
import livekit.org.webrtc.VideoProcessor
22+
import livekit.org.webrtc.VideoSink
23+
24+
/**
25+
* A VideoProcessor that can be chained together.
26+
*
27+
* Child classes should propagate frames down to the
28+
* next link through [continueChain].
29+
*/
30+
abstract class ChainVideoProcessor : VideoProcessor {
31+
32+
/**
33+
* The video sink where frames that have been completely processed are sent.
34+
*/
35+
var videoSink: VideoSink? = null
36+
private set
37+
38+
/**
39+
* The next link in the chain to feed frames to.
40+
*
41+
* Setting [childVideoProcessor] to null will mean that this is object
42+
* the end of the chain, and processed frames are ready to be published.
43+
*/
44+
var childVideoProcessor: VideoProcessor? = null
45+
set(value) {
46+
value?.setSink(videoSink)
47+
field = value
48+
}
49+
50+
@CallSuper
51+
override fun onCapturerStarted(started: Boolean) {
52+
childVideoProcessor?.onCapturerStarted(started)
53+
}
54+
55+
@CallSuper
56+
override fun onCapturerStopped() {
57+
childVideoProcessor?.onCapturerStopped()
58+
}
59+
60+
final override fun setSink(videoSink: VideoSink?) {
61+
childVideoProcessor?.setSink(videoSink)
62+
this.videoSink = videoSink
63+
}
64+
65+
/**
66+
* A utility method to pass the frame down to the next link in the chain.
67+
*/
68+
protected fun continueChain(frame: VideoFrame) {
69+
childVideoProcessor?.onFrameCaptured(frame) ?: videoSink?.onFrame(frame)
70+
}
71+
}

‎livekit-android-sdk/src/main/java/io/livekit/android/room/track/video/ScaleCropVideoProcessor.kt‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
/*
2+
* Copyright 2024 LiveKit, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package io.livekit.android.room.track.video
18+
19+
import livekit.org.webrtc.VideoFrame
20+
import kotlin.math.max
21+
import kotlin.math.roundToInt
22+
23+
/**
24+
* A video processor that scales down and crops to match
25+
* the target dimensions and aspect ratio.
26+
*
27+
* If the frames are smaller than the target dimensions,
28+
* upscaling will not occur, instead only cropping to match
29+
* the aspect ratio.
30+
*/
31+
class ScaleCropVideoProcessor(
32+
var targetWidth: Int,
33+
var targetHeight: Int,
34+
) : ChainVideoProcessor() {
35+
36+
override fun onFrameCaptured(frame: VideoFrame) {
37+
if (frame.rotatedWidth == targetWidth && frame.rotatedHeight == targetHeight) {
38+
// already the perfect size, just pass along the frame.
39+
continueChain(frame)
40+
return
41+
}
42+
43+
val width = frame.buffer.width
44+
val height = frame.buffer.height
45+
// Ensure target dimensions don't exceed source dimensions
46+
val scaleWidth: Int
47+
val scaleHeight: Int
48+
49+
if (targetWidth > width || targetHeight > height) {
50+
// Calculate scale factor to fit within source dimensions
51+
val widthScale = targetWidth.toDouble() / width
52+
val heightScale = targetHeight.toDouble() / height
53+
val scale = max(widthScale, heightScale)
54+
55+
// Apply scale to target dimensions
56+
scaleWidth = (targetWidth / scale).roundToInt()
57+
scaleHeight = (targetHeight / scale).roundToInt()
58+
} else {
59+
scaleWidth = targetWidth
60+
scaleHeight = targetHeight
61+
}
62+
63+
val sourceRatio = width.toDouble() / height
64+
val targetRatio = scaleWidth.toDouble() / scaleHeight
65+
66+
val cropWidth: Int
67+
val cropHeight: Int
68+
69+
// Calculate crop dimension
70+
if (sourceRatio > targetRatio) {
71+
// source is wider, crop height
72+
cropHeight = height
73+
cropWidth = (height * targetRatio).roundToInt()
74+
} else {
75+
// source is taller, crop width
76+
cropWidth = width
77+
cropHeight = (width / targetRatio).roundToInt()
78+
}
79+
80+
// Calculate center offsets
81+
val offsetX = (width - cropWidth) / 2
82+
val offsetY = (height - cropHeight) / 2
83+
val newBuffer = frame.buffer.cropAndScale(
84+
offsetX,
85+
offsetY,
86+
cropWidth,
87+
cropHeight,
88+
scaleWidth,
89+
scaleHeight,
90+
)
91+
92+
val croppedFrame = VideoFrame(newBuffer, frame.rotation, frame.timestampNs)
93+
continueChain(croppedFrame)
94+
croppedFrame.release()
95+
}
96+
}

0 commit comments

Comments
 (0)