Replace video transcoder with Media3 Transformer (#5018)

Author: jmartinesp

gradle/libs.versions.toml

@@ -100,6 +100,9 @@ androidx_lifecycle_process = { module = "androidx.lifecycle:lifecycle-process",
 androidx_splash = "androidx.core:core-splashscreen:1.0.1"
 androidx_media3_exoplayer = { module = "androidx.media3:media3-exoplayer", version.ref = "media3" }
 androidx_media3_ui = { module = "androidx.media3:media3-ui", version.ref = "media3" }
+androidx_media3_transformer = { module = "androidx.media3:media3-transformer", version.ref = "media3" }
+androidx_media3_effect = { module = "androidx.media3:media3-effect", version.ref = "media3" }
+androidx_media3_common = { module = "androidx.media3:media3-common", version.ref = "media3" }
 androidx_biometric = "androidx.biometric:biometric-ktx:1.2.0-alpha05"
 
 androidx_activity_activity = { module = "androidx.activity:activity", version.ref = "activity" }
@@ -182,7 +185,6 @@ sqldelight-coroutines = { module = "app.cash.sqldelight:coroutines-extensions",
 sqlcipher = "net.zetetic:sqlcipher-android:4.9.0"
 sqlite = "androidx.sqlite:sqlite-ktx:2.5.2"
 unifiedpush = "org.unifiedpush.android:connector:3.0.10"
-otaliastudios_transcoder = "com.otaliastudios:transcoder:0.11.2"
 vanniktech_blurhash = "com.vanniktech:blurhash:0.3.0"
 telephoto_zoomableimage = { module = "me.saket.telephoto:zoomable-image-coil", version.ref = "telephoto" }
 telephoto_flick = { module = "me.saket.telephoto:flick-android", version.ref = "telephoto" }

libraries/mediaupload/impl/build.gradle.kts

@@ -33,8 +33,10 @@ dependencies {
     implementation(projects.services.toolbox.api)
     implementation(libs.inject)
     implementation(libs.androidx.exifinterface)
+    implementation(libs.androidx.media3.transformer)
+    implementation(libs.androidx.media3.effect)
+    implementation(libs.androidx.media3.common)
     implementation(libs.coroutines.core)
-    implementation(libs.otaliastudios.transcoder)
     implementation(libs.vanniktech.blurhash)
 
     testImplementation(libs.test.junit)

libraries/mediaupload/impl/src/main/kotlin/io/element/android/libraries/mediaupload/impl/AndroidMediaPreProcessor.kt

@@ -192,12 +192,19 @@ class AndroidMediaPreProcessor @Inject constructor(
         val resultFile = runCatchingExceptions {
             videoCompressor.compress(uri, shouldBeCompressed)
                 .onEach {
-                    // TODO handle progress
+                    if (it is VideoTranscodingEvent.Progress) {
+                        Timber.d("Video compression progress: ${it.value}%")
+                    } else if (it is VideoTranscodingEvent.Completed) {
+                        Timber.d("Video compression completed: ${it.file.path}")
+                    }
                 }
                 .filterIsInstance<VideoTranscodingEvent.Completed>()
                 .first()
                 .file
         }
+            .onFailure {
+                Timber.e(it, "Failed to compress video: $uri")
+            }
             .getOrNull()
 
         if (resultFile != null) {
@@ -283,10 +290,17 @@ class AndroidMediaPreProcessor @Inject constructor(
     private fun extractVideoMetadata(file: File, mimeType: String?, thumbnailResult: ThumbnailResult?): VideoInfo =
         MediaMetadataRetriever().runAndRelease {
             setDataSource(context, Uri.fromFile(file))
+
+            val rotation = extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION)?.toInt() ?: 0
+            val rawWidth = extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH)?.toLong() ?: 0L
+            val rawHeight = extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT)?.toLong() ?: 0L
+
+            val (width, height) = if (rotation == 90 || rotation == 270) rawHeight to rawWidth else rawWidth to rawHeight
+
             VideoInfo(
                 duration = extractDuration(),
-                width = extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH)?.toLong() ?: 0L,
-                height = extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT)?.toLong() ?: 0L,
+                width = width,
+                height = height,
                 mimetype = mimeType,
                 size = file.length(),
                 thumbnailInfo = thumbnailResult?.info,

libraries/mediaupload/impl/src/main/kotlin/io/element/android/libraries/mediaupload/impl/VideoCompressor.kt

@@ -8,76 +8,146 @@
 package io.element.android.libraries.mediaupload.impl
 
 import android.content.Context
+import android.media.MediaCodecInfo
 import android.media.MediaMetadataRetriever
 import android.net.Uri
-import android.webkit.MimeTypeMap
-import com.otaliastudios.transcoder.Transcoder
-import com.otaliastudios.transcoder.TranscoderListener
-import com.otaliastudios.transcoder.internal.media.MediaFormatConstants
-import com.otaliastudios.transcoder.resize.AtMostResizer
-import com.otaliastudios.transcoder.strategy.DefaultVideoStrategy
-import com.otaliastudios.transcoder.strategy.PassThroughTrackStrategy
-import com.otaliastudios.transcoder.strategy.TrackStrategy
-import com.otaliastudios.transcoder.validator.WriteAlwaysValidator
+import androidx.annotation.OptIn
+import androidx.media3.common.MediaItem
+import androidx.media3.common.MimeTypes
+import androidx.media3.common.util.Size
+import androidx.media3.common.util.UnstableApi
+import androidx.media3.effect.Presentation
+import androidx.media3.transformer.Composition
+import androidx.media3.transformer.DefaultEncoderFactory
+import androidx.media3.transformer.EditedMediaItem
+import androidx.media3.transformer.Effects
+import androidx.media3.transformer.ExportException
+import androidx.media3.transformer.ExportResult
+import androidx.media3.transformer.ProgressHolder
+import androidx.media3.transformer.TransformationRequest
+import androidx.media3.transformer.Transformer
+import androidx.media3.transformer.VideoEncoderSettings
 import io.element.android.libraries.androidutils.file.createTmpFile
-import io.element.android.libraries.androidutils.file.getMimeType
 import io.element.android.libraries.androidutils.file.safeDelete
 import io.element.android.libraries.core.extensions.runCatchingExceptions
 import io.element.android.libraries.di.ApplicationContext
+import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.channels.awaitClose
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.callbackFlow
+import kotlinx.coroutines.isActive
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.withContext
 import timber.log.Timber
 import java.io.File
 import javax.inject.Inject
 
-private const val MP4_EXTENSION = "mp4"
-
 class VideoCompressor @Inject constructor(
     @ApplicationContext private val context: Context,
 ) {
-    fun compress(uri: Uri, shouldBeCompressed: Boolean) = callbackFlow {
+    @OptIn(UnstableApi::class)
+    fun compress(uri: Uri, shouldBeCompressed: Boolean): Flow<VideoTranscodingEvent> = callbackFlow {
         val metadata = getVideoMetadata(uri)
 
-        val expectedExtension = MimeTypeMap.getSingleton().getExtensionFromMimeType(context.getMimeType(uri))
-
-        val videoStrategy = VideoStrategyFactory.create(
-            expectedExtension = expectedExtension,
+        val videoCompressorConfig = VideoCompressorConfigFactory.create(
             metadata = metadata,
             shouldBeCompressed = shouldBeCompressed
         )
 
-        val tmpFile = context.createTmpFile(extension = MP4_EXTENSION)
-        val future = Transcoder.into(tmpFile.path)
-            .setVideoTrackStrategy(videoStrategy)
-            .addDataSource(context, uri)
-            // Force the output to be written, even if no transcoding was actually needed
-            .setValidator(WriteAlwaysValidator())
-            .setListener(object : TranscoderListener {
-                override fun onTranscodeProgress(progress: Double) {
-                    trySend(VideoTranscodingEvent.Progress(progress.toFloat()))
-                }
+        val tmpFile = context.createTmpFile(extension = "mp4")
+
+        val width = metadata?.width ?: Int.MAX_VALUE
+        val height = metadata?.height ?: Int.MAX_VALUE
 
-                override fun onTranscodeCompleted(successCode: Int) {
+        val videoResizeEffect = videoCompressorConfig.resizer?.let {
+            val outputSize = it.getOutputSize(Size(width, height))
+            if (metadata?.rotation == 90 || metadata?.rotation == 270) {
+                // If the video is rotated, we need to swap width and height
+                Presentation.createForWidthAndHeight(
+                    outputSize.height,
+                    outputSize.width,
+                    Presentation.LAYOUT_SCALE_TO_FIT,
+                )
+            } else {
+                // Otherwise, we can use the original width and height
+                Presentation.createForWidthAndHeight(
+                    outputSize.width,
+                    outputSize.height,
+                    Presentation.LAYOUT_SCALE_TO_FIT,
+                )
+            }
+        }
+
+        // If we are resizing, we also want to reduce set frame rate to the default value (30fps)
+        val newFrameRate = videoCompressorConfig.newFrameRate
+
+        // If we need to resize the video, we also want to recalculate the bitrate
+        val newBitrate = videoCompressorConfig.newBitRate
+
+        val inputMediaItem = MediaItem.fromUri(uri)
+        val outputMediaItem = EditedMediaItem.Builder(inputMediaItem)
+            .setFrameRate(newFrameRate)
+            .run {
+                if (videoResizeEffect != null) {
+                    setEffects(Effects(emptyList(), listOf(videoResizeEffect)))
+                } else {
+                    this
+                }
+            }
+            .build()
+
+        val encoderFactory = DefaultEncoderFactory.Builder(context)
+            .setRequestedVideoEncoderSettings(
+                VideoEncoderSettings.Builder()
+                    .setBitrateMode(MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR)
+                    .setBitrate(newBitrate)
+                    .build()
+            )
+            .build()
+
+        val videoTransformer = Transformer.Builder(context)
+            .setVideoMimeType(MimeTypes.VIDEO_H264)
+            .setAudioMimeType(MimeTypes.AUDIO_AAC)
+            .setPortraitEncodingEnabled(false)
+            .setEncoderFactory(encoderFactory)
+            .addListener(object : Transformer.Listener {
+                override fun onCompleted(composition: Composition, exportResult: ExportResult) {
                     trySend(VideoTranscodingEvent.Completed(tmpFile))
                     close()
                 }
 
-                override fun onTranscodeCanceled() {
+                override fun onError(composition: Composition, exportResult: ExportResult, exportException: ExportException) {
+                    Timber.e(exportException, "Video transcoding failed")
                     tmpFile.safeDelete()
-                    close()
+                    close(exportException)
                 }
 
-                override fun onTranscodeFailed(exception: Throwable) {
-                    tmpFile.safeDelete()
-                    close(exception)
-                }
+                override fun onFallbackApplied(
+                    composition: Composition,
+                    originalTransformationRequest: TransformationRequest,
+                    fallbackTransformationRequest: TransformationRequest
+                ) = Unit
             })
-            .transcode()
+            .build()
+
+        val progressJob = launch(Dispatchers.Main) {
+            val progressHolder = ProgressHolder()
+            while (isActive) {
+                val state = videoTransformer.getProgress(progressHolder)
+                if (state != Transformer.PROGRESS_STATE_NOT_STARTED) {
+                    channel.send(VideoTranscodingEvent.Progress(progressHolder.progress.toFloat()))
+                }
+                delay(500)
+            }
+        }
+
+        withContext(Dispatchers.Main) {
+            videoTransformer.start(outputMediaItem, tmpFile.path)
+        }
 
         awaitClose {
-            if (!future.isDone) {
-                future.cancel(true)
-            }
+            progressJob.cancel()
         }
     }
 
@@ -89,7 +159,7 @@ class VideoCompressor @Inject constructor(
                 val width = it.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH)?.toIntOrNull() ?: -1
                 val height = it.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT)?.toIntOrNull() ?: -1
                 val bitrate = it.extractMetadata(MediaMetadataRetriever.METADATA_KEY_BITRATE)?.toLongOrNull() ?: -1
-                val framerate = it.extractMetadata(MediaMetadataRetriever.METADATA_KEY_CAPTURE_FRAMERATE)?.toIntOrNull() ?: -1
+                val frameRate = it.extractMetadata(MediaMetadataRetriever.METADATA_KEY_CAPTURE_FRAMERATE)?.toIntOrNull() ?: -1
                 val rotation = it.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION)?.toIntOrNull() ?: 0
 
                 val (actualWidth, actualHeight) = if (width == -1 || height == -1) {
@@ -104,7 +174,7 @@ class VideoCompressor @Inject constructor(
                     width = actualWidth,
                     height = actualHeight,
                     bitrate = bitrate,
-                    frameRate = framerate,
+                    frameRate = frameRate,
                     rotation = rotation,
                 )
             }
@@ -126,45 +196,3 @@ sealed interface VideoTranscodingEvent {
     data class Progress(val value: Float) : VideoTranscodingEvent
     data class Completed(val file: File) : VideoTranscodingEvent
 }
-
-internal object VideoStrategyFactory {
-    // 720p
-    private const val MAX_COMPRESSED_PIXEL_SIZE = 1280
-
-    // 1080p
-    private const val MAX_PIXEL_SIZE = 1920
-
-    fun create(
-        expectedExtension: String?,
-        metadata: VideoFileMetadata?,
-        shouldBeCompressed: Boolean,
-    ): TrackStrategy {
-        val width = metadata?.width?.takeIf { it >= 0 } ?: Int.MAX_VALUE
-        val height = metadata?.height?.takeIf { it >= 0 } ?: Int.MAX_VALUE
-        val bitrate = metadata?.bitrate?.takeIf { it >= 0 }
-        val frameRate = metadata?.frameRate?.takeIf { it >= 0 }
-        val rotation = metadata?.rotation?.takeIf { it >= 0 }
-
-        // We only create a resizer if needed
-        val resizer = when {
-            shouldBeCompressed && (width > MAX_COMPRESSED_PIXEL_SIZE || height > MAX_COMPRESSED_PIXEL_SIZE) -> AtMostResizer(MAX_COMPRESSED_PIXEL_SIZE)
-            width > MAX_PIXEL_SIZE || height > MAX_PIXEL_SIZE -> AtMostResizer(MAX_PIXEL_SIZE)
-            else -> null
-        }
-
-        return if (resizer == null && rotation == 0 && expectedExtension == MP4_EXTENSION) {
-            // If there's no transcoding or resizing needed for the video file, just create a new file with the same contents but no metadata
-            // Rotation is not kept by the PassThroughTrackStrategy, so we need to ensure the video is not rotated
-            PassThroughTrackStrategy()
-        } else {
-            DefaultVideoStrategy.Builder()
-                .apply {
-                    resizer?.let { addResizer(it) }
-                    bitrate?.let { bitRate(it) }
-                    frameRate?.let { frameRate(it) }
-                }
-                .mimeType(MediaFormatConstants.MIMETYPE_VIDEO_AVC)
-                .build()
-        }
-    }
-}