refactor: build waveform after fetching audio asset and update waveform colors [WPB-22118] (#4474)

Co-authored-by: Mohamad Jaara <[email protected]>

Author: saleniuk

app/build.gradle.kts

@@ -270,7 +270,6 @@ dependencies {
     implementation(libs.aboutLibraries.compose.core)
     implementation(libs.aboutLibraries.compose.m3)
     implementation(libs.compose.qr.code)
-    implementation(libs.audio.amplituda)
     implementation(libs.enterprise.feedback)
 
     // screenshot testing

app/src/main/kotlin/com/wire/android/media/audiomessage/AudioMessageViewModel.kt

@@ -28,9 +28,14 @@ import com.wire.android.di.ScopedArgs
 import com.wire.android.di.ViewModelScopedPreview
 import com.wire.android.di.scopedArgs
 import com.wire.kalium.logic.data.id.ConversationId
+import com.wire.kalium.logic.data.message.AssetContent.AssetMetadata
+import com.wire.kalium.logic.data.message.MessageContent
+import com.wire.kalium.logic.feature.message.ObserveMessageByIdUseCase
 import dagger.hilt.android.lifecycle.HiltViewModel
 import kotlinx.coroutines.flow.collectLatest
 import kotlinx.coroutines.flow.distinctUntilChanged
+import kotlinx.coroutines.flow.firstOrNull
+import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.flow.mapNotNull
 import kotlinx.coroutines.launch
 import kotlinx.serialization.Serializable
@@ -47,6 +52,7 @@ interface AudioMessageViewModel {
 @HiltViewModel
 class AudioMessageViewModelImpl @Inject constructor(
     private val audioMessagePlayer: ConversationAudioMessagePlayer,
+    private val observeMessageById: ObserveMessageByIdUseCase,
     savedStateHandle: SavedStateHandle,
 ) : ViewModel(), AudioMessageViewModel {
 
@@ -58,7 +64,8 @@ class AudioMessageViewModelImpl @Inject constructor(
     init {
         observeAudioState()
         observeAudioSpeed()
-        initWavesMask()
+        preloadAudioMessage()
+        fetchWavesMask()
     }
 
     private fun observeAudioState() {
@@ -84,9 +91,26 @@ class AudioMessageViewModelImpl @Inject constructor(
         }
     }
 
-    private fun initWavesMask() {
+    private fun preloadAudioMessage() {
         viewModelScope.launch {
-            audioMessagePlayer.getOrBuildWavesMask(args.conversationId, args.messageId)
+            // calls preload to initially fetch the audio asset data to be ready and schedule waves mask generation if needed
+            audioMessagePlayer.preloadAudioMessage(args.conversationId, args.messageId)
+        }
+    }
+
+    private fun fetchWavesMask() {
+        viewModelScope.launch {
+            observeMessageById(args.conversationId, args.messageId)
+                .map {
+                    (it as? ObserveMessageByIdUseCase.Result.Success)?.message?.content?.let {
+                        (it as? MessageContent.Asset)?.value?.metadata?.let {
+                            (it as? AssetMetadata.Audio)?.normalizedLoudness?.toWavesMask()
+                        }
+                    }
+                }
+                .distinctUntilChanged()
+                .firstOrNull { it != null } // wait for the first non-null value
+                .let { state = state.copy(wavesMask = it) }
         }
     }
 
@@ -125,4 +149,5 @@ data class AudioMessageArgs(
 data class AudioMessageState(
     val audioSpeed: AudioSpeed = AudioSpeed.NORMAL,
     val audioState: AudioState = AudioState.DEFAULT,
+    val wavesMask: List<Int>? = null
 )

app/src/main/kotlin/com/wire/android/media/audiomessage/AudioState.kt

@@ -19,23 +19,16 @@ package com.wire.android.media.audiomessage
 
 import androidx.annotation.StringRes
 import com.wire.android.R
-import com.wire.android.media.audiomessage.ConversationAudioMessagePlayer.MessageIdWrapper
 import com.wire.android.util.ui.UIText
 import com.wire.kalium.logic.data.id.ConversationId
 
-data class AudioMessagesData(
-    val statesHistory: Map<MessageIdWrapper, AudioState>,
-    val playingMessage: PlayingAudioMessage
-)
-
 data class AudioState(
     val audioMediaPlayingState: AudioMediaPlayingState,
     val currentPositionInMs: Int,
     val totalTimeInMs: TotalTimeInMs,
-    val wavesMask: List<Int>?
 ) {
     companion object {
-        val DEFAULT = AudioState(AudioMediaPlayingState.Stopped, 0, TotalTimeInMs.NotKnown, null)
+        val DEFAULT = AudioState(AudioMediaPlayingState.Stopped, 0, TotalTimeInMs.NotKnown)
     }
 
     // if the back-end returned the total time, we use that, in case it didn't we use what we get from
@@ -142,12 +135,6 @@ sealed class AudioMediaPlayerStateUpdate(
         override val messageId: String,
         val totalTimeInMs: Int
     ) : AudioMediaPlayerStateUpdate(conversationId, messageId)
-
-    data class WaveMaskUpdate(
-        override val conversationId: ConversationId,
-        override val messageId: String,
-        val waveMask: List<Int>?
-    ) : AudioMediaPlayerStateUpdate(conversationId, messageId)
 }
 
 sealed class RecordAudioMediaPlayerStateUpdate {
@@ -162,8 +149,4 @@ sealed class RecordAudioMediaPlayerStateUpdate {
     data class TotalTimeUpdate(
         val totalTimeInMs: Int
     ) : RecordAudioMediaPlayerStateUpdate()
-
-    data class WaveMaskUpdate(
-        val waveMask: List<Int>
-    ) : RecordAudioMediaPlayerStateUpdate()
 }

app/src/main/kotlin/com/wire/android/media/audiomessage/ConversationAudioMessagePlayer.kt

@@ -30,12 +30,8 @@ import com.wire.android.util.extension.intervalFlow
 import com.wire.android.util.ui.UIText
 import com.wire.kalium.logic.CoreLogic
 import com.wire.kalium.logic.data.id.ConversationId
-import com.wire.kalium.logic.data.message.AssetContent.AssetMetadata
-import com.wire.kalium.logic.data.message.MessageContent
 import com.wire.kalium.logic.data.user.UserId
-import com.wire.kalium.logic.feature.asset.AudioNormalizedLoudnessBuilder
 import com.wire.kalium.logic.feature.asset.MessageAssetResult
-import com.wire.kalium.logic.feature.message.GetMessageByIdUseCase
 import com.wire.kalium.logic.feature.message.GetNextAudioMessageInConversationUseCase
 import com.wire.kalium.logic.feature.message.GetSenderNameByMessageIdUseCase
 import com.wire.kalium.logic.feature.session.CurrentSessionResult
@@ -62,7 +58,6 @@ import kotlinx.coroutines.launch
 import kotlinx.coroutines.sync.Mutex
 import kotlinx.coroutines.sync.withLock
 import kotlinx.coroutines.withContext
-import okio.Path
 import javax.inject.Inject
 import javax.inject.Singleton
 
@@ -72,7 +67,6 @@ class ConversationAudioMessagePlayer
 @Inject constructor(
     @ApplicationContext private val context: Context,
     private val audioMediaPlayer: MediaPlayer,
-    private val audioNormalizedLoudnessBuilder: AudioNormalizedLoudnessBuilder,
     private val servicesManager: Lazy<ServicesManager>,
     private val audioFocusHelper: AudioFocusHelper,
     @KaliumCoreLogic private val coreLogic: CoreLogic,
@@ -194,15 +188,6 @@ class ConversationAudioMessagePlayer
                         )
                     }
                 }
-
-                is AudioMediaPlayerStateUpdate.WaveMaskUpdate -> {
-                    audioMessageStateHistory = audioMessageStateHistory.toMutableMap().apply {
-                        put(
-                            messageIdKey,
-                            currentState.copy(wavesMask = audioMessageStateUpdate.waveMask)
-                        )
-                    }
-                }
             }
 
             audioMessageStateHistory
@@ -311,12 +296,6 @@ class ConversationAudioMessagePlayer
                     audioMediaPlayer.setDataSource(context, Uri.parse(result.decodedAssetPath.toString()))
                     audioMediaPlayer.prepare()
 
-                    getOrBuildWavesMask(
-                        conversationId = conversationId,
-                        messageId = messageId,
-                        assetPath = result.decodedAssetPath
-                    )
-
                     if (position != null) audioMediaPlayer.seekTo(position)
 
                     audioFocusHelper.request()
@@ -366,49 +345,13 @@ class ConversationAudioMessagePlayer
         seekToAudioPosition.emit(MessageIdWrapper(conversationId, messageId) to position)
     }
 
-    /**
-     * Gets waves mask from local database for the audio message or builds a new one if it does not exist and emits the update event.
-     * @param conversationId The ID of the conversation.
-     * @param messageId The ID of the message.
-     * @param assetPath Optional path to the decoded asset. If not provided, it will be fetched.
-     */
-    @Suppress("ReturnCount")
-    suspend fun getOrBuildWavesMask(conversationId: ConversationId, messageId: String, assetPath: Path? = null) {
+    suspend fun preloadAudioMessage(conversationId: ConversationId, messageId: String) {
         val currentAccountResult = coreLogic.getGlobalScope().session.currentSession()
         if (currentAccountResult is CurrentSessionResult.Failure) return
         val userId = (currentAccountResult as CurrentSessionResult.Success).accountInfo.userId
-        val messageResult = coreLogic.getSessionScope(userId).messages.getMessageById(conversationId, messageId)
-        if (messageResult is GetMessageByIdUseCase.Result.Failure) return
-        val messageContent = (messageResult as GetMessageByIdUseCase.Result.Success).message.content
-        if (messageContent !is MessageContent.Asset) return
-        if (messageContent.value.metadata !is AssetMetadata.Audio) return
-        val audioMetadata = messageContent.value.metadata as AssetMetadata.Audio
-
-        val currentNormalizedLoudness: ByteArray? = audioMetadata.normalizedLoudness ?: run {
-            (assetPath ?: getAssetPath(userId, conversationId, messageId))?.let { assetPath ->
-                audioNormalizedLoudnessBuilder(assetPath.toString())?.also {
-                    coreLogic.getSessionScope(userId).messages.updateAudioMessageNormalizedLoudnessUseCase(
-                        conversationId = conversationId,
-                        messageId = messageId,
-                        normalizedLoudness = it
-                    )
-                }
-            }
-        }
-        currentNormalizedLoudness?.let {
-            audioMessageStateUpdate.emit(
-                AudioMediaPlayerStateUpdate.WaveMaskUpdate(
-                    conversationId = conversationId,
-                    messageId = messageId,
-                    waveMask = currentNormalizedLoudness.toWavesMask()
-                )
-            )
-        }
+        getAssetMessage(userId, conversationId, messageId)
     }
 
-    private suspend fun getAssetPath(userId: UserId, conversationId: ConversationId, messageId: String): Path? =
-        (getAssetMessage(userId, conversationId, messageId) as? MessageAssetResult.Success)?.decodedAssetPath
-
     private suspend fun resumeOrPause(conversationId: ConversationId, messageId: String) {
         if (audioMediaPlayer.isPlaying) {
             pause(conversationId, messageId)

app/src/main/kotlin/com/wire/android/media/audiomessage/RecordAudioMessagePlayer.kt

@@ -21,7 +21,6 @@ import android.content.Context
 import android.media.MediaPlayer
 import androidx.core.net.toUri
 import com.wire.android.di.ApplicationScope
-import com.wire.kalium.logic.feature.asset.AudioNormalizedLoudnessBuilder
 import dagger.hilt.android.scopes.ViewModelScoped
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.channels.BufferOverflow
@@ -41,7 +40,6 @@ import javax.inject.Inject
 class RecordAudioMessagePlayer @Inject constructor(
     private val context: Context,
     private val audioMediaPlayer: MediaPlayer,
-    private val audioNormalizedLoudnessBuilder: AudioNormalizedLoudnessBuilder,
     private val audioFocusHelper: AudioFocusHelper,
     @ApplicationScope private val scope: CoroutineScope
 ) {
@@ -122,12 +120,6 @@ class RecordAudioMessagePlayer @Inject constructor(
                         )
                     )
                 }
-
-                is RecordAudioMediaPlayerStateUpdate.WaveMaskUpdate -> {
-                    audioState = audioState.copy(
-                        wavesMask = audioStateUpdate.waveMask
-                    )
-                }
             }
 
             audioState
@@ -185,14 +177,6 @@ class RecordAudioMessagePlayer @Inject constructor(
         audioMediaPlayer.seekTo(position)
         audioMediaPlayer.start()
 
-        audioNormalizedLoudnessBuilder(audioFile.path)?.let {
-            audioMessageStateUpdate.emit(
-                RecordAudioMediaPlayerStateUpdate.WaveMaskUpdate(
-                    waveMask = it.toWavesMask()
-                )
-            )
-        }
-
         audioMessageStateUpdate.emit(
             RecordAudioMediaPlayerStateUpdate.RecordAudioMediaPlayingStateUpdate(
                 audioMediaPlayingState = AudioMediaPlayingState.Playing

app/src/main/kotlin/com/wire/android/ui/home/conversations/messages/item/MessageStyle.kt

@@ -112,4 +112,18 @@ fun MessageStyle.textAlign(): TextAlign {
     }
 }
 
-private const val BUBBLE_OPACITY = 0.5F
+@Composable
+fun MessageStyle.playedColor() = when (this) {
+    MessageStyle.BUBBLE_SELF -> colorsScheme().selfBubble.onPrimary
+    MessageStyle.BUBBLE_OTHER -> colorsScheme().otherBubble.primaryOnSecondary
+    MessageStyle.NORMAL -> colorsScheme().primary
+}
+
+@Composable
+fun MessageStyle.remainingColor() = when (this) {
+    MessageStyle.BUBBLE_SELF -> colorsScheme().selfBubble.onPrimary.copy(alpha = 0.5F)
+    MessageStyle.BUBBLE_OTHER -> colorsScheme().secondaryText
+    MessageStyle.NORMAL -> colorsScheme().onTertiaryButtonDisabled
+}
+
+private const val BUBBLE_OPACITY = 0.7F

app/src/main/kotlin/com/wire/android/ui/home/conversations/mock/Mock.kt

@@ -350,8 +350,12 @@ fun mockAssetMessage(assetId: String = "asset1", messageId: String = "msg1") = U
     source = MessageSource.Self
 )
 
-fun mockAssetAudioMessage(assetId: String = "asset1", messageId: String = "msg1") = UIMessage.Regular(
-    conversationId = ConversationId("value", "domain"),
+fun mockAssetAudioMessage(
+    assetId: String = "asset1",
+    messageId: String = "msg1",
+    conversationId: ConversationId = ConversationId("value", "domain"),
+) = UIMessage.Regular(
+    conversationId = conversationId,
     userAvatarData = UserAvatarData(
         UserAvatarAsset(UserAssetId("a", "domain")),
         UserAvailabilityStatus.AVAILABLE