fix(liveness): Multiple Liveness Bug Fixes (#124)

Author: tylerjroach

liveness/api/liveness.api

@@ -2,6 +2,7 @@ public final class com/amplifyframework/ui/liveness/BuildConfig {
 	public static final field BUILD_TYPE Ljava/lang/String;
 	public static final field DEBUG Z
 	public static final field LIBRARY_PACKAGE_NAME Ljava/lang/String;
+	public static final field LIVENESS_VERSION_NAME Ljava/lang/String;
 	public static final field VERSION_NAME Ljava/lang/String;
 	public fun <init> ()V
 }

liveness/build.gradle.kts

@@ -12,6 +12,11 @@ android {
                 arguments += "-DCMAKE_VERBOSE_MAKEFILE=ON"
             }
         }
+        buildConfigField(
+            "String",
+            "LIVENESS_VERSION_NAME",
+            "\"${project.properties["VERSION_NAME"]}\""
+        )
         testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
     }
 

liveness/src/main/java/com/amplifyframework/ui/liveness/camera/FrameAnalyzer.kt

@@ -20,6 +20,7 @@ import android.graphics.Bitmap
 import android.util.Size
 import androidx.camera.core.ImageAnalysis
 import androidx.camera.core.ImageProxy
+import com.amplifyframework.core.Amplify
 import com.amplifyframework.ui.liveness.ml.FaceDetector
 import com.amplifyframework.ui.liveness.ml.FaceOval
 import com.amplifyframework.ui.liveness.state.LivenessState
@@ -42,7 +43,20 @@ internal class FrameAnalyzer(
     private var cachedBitmap: Bitmap? = null
     private var faceDetector = FaceDetector(livenessState)
 
+    private val logger = Amplify.Logging.forNamespace("Liveness")
+
     override fun analyze(image: ImageProxy) {
+        try {
+            attemptAnalyze(image)
+        } catch (e: Exception) {
+            // We've seen a few instances of exceptions thrown by copyPixelsFromBuffer.
+            // This indicates the image received may have been in an unexpected format.
+            // We discard this frame, in hopes that the next frame is readable.
+            logger.error("Failed to analyze frame", e)
+        }
+    }
+
+    private fun attemptAnalyze(image: ImageProxy) {
         if (cachedBitmap == null) {
             cachedBitmap = Bitmap.createBitmap(
                 image.width,

liveness/src/main/java/com/amplifyframework/ui/liveness/camera/LivenessCoordinator.kt

@@ -180,7 +180,7 @@ internal class LivenessCoordinator(
             sessionId,
             faceLivenessSessionInformation,
             faceLivenessSessionOptions,
-            BuildConfig.VERSION_NAME,
+            BuildConfig.LIVENESS_VERSION_NAME,
             { livenessState.onLivenessSessionReady(it) },
             {
                 disconnectEventReceived = true

liveness/src/main/java/com/amplifyframework/ui/liveness/camera/LivenessMuxer.kt

@@ -19,6 +19,7 @@ import android.media.MediaCodec
 import android.media.MediaFormat
 import android.media.MediaMuxer
 import androidx.annotation.WorkerThread
+import com.amplifyframework.core.Amplify
 import java.io.File
 import java.io.RandomAccessFile
 import java.nio.ByteBuffer
@@ -48,6 +49,8 @@ internal class LivenessMuxer(
         currentVideoStartTime = System.currentTimeMillis()
     }
 
+    private val logger = Amplify.Logging.forNamespace("Liveness")
+
     /*
     Attempt to notify listener that chunked data is available if minimum chunk interval has exceeded
     Write new frame to muxer
@@ -61,7 +64,13 @@ internal class LivenessMuxer(
             }
         }
 
-        muxer.writeSampleData(videoTrack, byteBuf, bufferInfo)
+        try {
+            muxer.writeSampleData(videoTrack, byteBuf, bufferInfo)
+        } catch (e: Exception) {
+            // writeSampleData can throw for various reasons, such as an empty byte buffer.
+            // If this happens, we discard the frame, in hopes that future frames are valid.
+            logger.error("Failed to write encoded chunk to muxer", e)
+        }
     }
 
     @WorkerThread

liveness/src/main/java/com/amplifyframework/ui/liveness/camera/LivenessVideoEncoder.kt

@@ -24,6 +24,7 @@ import android.os.Handler
 import android.os.HandlerThread
 import android.util.Log
 import androidx.annotation.WorkerThread
+import com.amplifyframework.core.Amplify
 import com.amplifyframework.ui.liveness.util.isKeyFrame
 import java.io.File
 
@@ -124,6 +125,7 @@ internal class LivenessVideoEncoder private constructor(
 
     private var encoding = false
     private var livenessMuxer: LivenessMuxer? = null
+    private val logger = Amplify.Logging.forNamespace("Liveness")
 
     init {
         encoder.start()
@@ -152,11 +154,18 @@ internal class LivenessVideoEncoder private constructor(
 
                     if (info.isKeyFrame()) {
                         if (livenessMuxer == null) {
-                            livenessMuxer = LivenessMuxer(
-                                outputFile,
-                                encoder.outputFormat,
-                                onMuxedSegment
-                            )
+                            try {
+                                val muxer = LivenessMuxer(
+                                    outputFile,
+                                    encoder.outputFormat,
+                                    onMuxedSegment
+                                )
+                                livenessMuxer = muxer
+                            } catch (e: Exception) {
+                                // This is likely an unrecoverable error, such as file creation failing.
+                                // However, if it fails, we will allow another attempt at the next keyframe.
+                                logger.error("Failed to create liveness muxer", e)
+                            }
                         }
                         framesSinceSyncRequest = 0 // reset keyframe request on keyframe receipt
                     } else {

liveness/src/main/java/com/amplifyframework/ui/liveness/ui/FaceLivenessDetector.kt

@@ -134,9 +134,12 @@ fun FaceLivenessDetector(
                 },
                 onChallengeFailed = {
                     scope.launch {
-                        isFinished = true
-                        resetOrientation()
-                        currentOnError.accept(it)
+                        // if we are already finished, we already provided a result in complete or failed
+                        if (!isFinished) {
+                            isFinished = true
+                            resetOrientation()
+                            currentOnError.accept(it)
+                        }
                     }
                 }
             )