feat(liveness): Update Liveness UI (#100)

Co-authored-by: Thomas Leing <[email protected]>

Author: tylerjroach

authenticator-screenshots/build.gradle.kts

@@ -21,10 +21,16 @@ plugins {
 
 android {
     namespace = "com.amplifyframework.ui.authenticator.screenshots"
+
+    compileOptions {
+        isCoreLibraryDesugaringEnabled = true
+    }
 }
 
 dependencies {
     implementation(libs.bundles.compose)
     implementation(libs.test.mockk)
     implementation(projects.authenticator)
+
+    coreLibraryDesugaring(libs.android.desugar)
 }

build-logic/plugins/src/main/kotlin/AndroidLibraryConventionPlugin.kt

@@ -101,8 +101,8 @@ class AndroidLibraryConventionPlugin : Plugin<Project> {
             lint {
                 warningsAsErrors = true
                 abortOnError = true
-                enable += listOf("UnusedResources", "NewerVersionAvailable")
-                disable += listOf("GradleDependency")
+                enable += listOf("UnusedResources")
+                disable += listOf("GradleDependency", "NewerVersionAvailable", "AndroidGradlePluginVersion")
             }
 
             compileOptions {

liveness/src/main/java/com/amplifyframework/ui/liveness/camera/LivenessCoordinator.kt

@@ -69,15 +69,17 @@ internal class LivenessCoordinator(
     private val sessionId: String,
     private val region: String,
     private val credentialsProvider: AWSCredentialsProvider<AWSCredentials>?,
+    disableStartView: Boolean,
     private val onChallengeComplete: OnChallengeComplete,
-    val onChallengeFailed: Consumer<FaceLivenessDetectionException>,
+    val onChallengeFailed: Consumer<FaceLivenessDetectionException>
 ) {
 
     private val analysisExecutor = Executors.newSingleThreadExecutor()
 
     val livenessState = LivenessState(
         sessionId,
         context,
+        disableStartView,
         this::processCaptureReady,
         this::startLivenessSession,
         this::processSessionError,
@@ -270,10 +272,10 @@ internal class LivenessCoordinator(
 
     fun destroy(context: Context) {
         // Destroy all resources so a new coordinator can safely be created
-        // livenessWebSocket.destroy()
         encoder.stop {
             encoder.destroy()
         }
+        livenessState.onDestroy(true)
         unbindCamera(context)
         analysisExecutor.shutdown()
     }

liveness/src/main/java/com/amplifyframework/ui/liveness/model/LivenessCheckState.kt

@@ -19,8 +19,11 @@ import android.graphics.RectF
 import com.amplifyframework.ui.liveness.R
 import com.amplifyframework.ui.liveness.ml.FaceDetector
 
-internal sealed class LivenessCheckState(val instructionId: Int? = null) {
-    class Initial(instructionId: Int? = null) : LivenessCheckState(instructionId) {
+internal sealed class LivenessCheckState(val instructionId: Int? = null, val isActionable: Boolean = true) {
+    class Initial(
+        instructionId: Int? = null,
+        isActionable: Boolean = true
+    ) : LivenessCheckState(instructionId, isActionable) {
         companion object {
             fun withMoveFaceMessage() =
                 Initial(R.string.amplify_ui_liveness_challenge_instruction_move_face)
@@ -29,10 +32,12 @@ internal sealed class LivenessCheckState(val instructionId: Int? = null) {
             fun withMoveFaceFurtherAwayMessage() =
                 Initial(R.string.amplify_ui_liveness_challenge_instruction_move_face_further)
             fun withConnectingMessage() =
-                Initial(R.string.amplify_ui_liveness_challenge_connecting)
+                Initial(R.string.amplify_ui_liveness_challenge_connecting, false)
+            fun withStartViewMessage() =
+                Initial(R.string.amplify_ui_liveness_get_ready_center_face_label)
         }
     }
-    class Running(instructionId: Int? = null) : LivenessCheckState(instructionId) {
+    class Running(instructionId: Int? = null) : LivenessCheckState(instructionId, true) {
         companion object {
             fun withMoveFaceMessage() = Running(
                 R.string.amplify_ui_liveness_challenge_instruction_move_face_closer
@@ -44,7 +49,7 @@ internal sealed class LivenessCheckState(val instructionId: Int? = null) {
                 Running(faceOvalPosition.instructionStringRes)
         }
     }
-    object Error : LivenessCheckState()
+    object Error : LivenessCheckState(isActionable = false)
     class Success(val faceGuideRect: RectF) :
-        LivenessCheckState(R.string.amplify_ui_liveness_challenge_verifying)
+        LivenessCheckState(R.string.amplify_ui_liveness_challenge_verifying, false)
 }

liveness/src/main/java/com/amplifyframework/ui/liveness/state/LivenessState.kt

@@ -36,17 +36,19 @@ import com.amplifyframework.ui.liveness.model.LivenessCheckState
 import com.amplifyframework.ui.liveness.ui.helper.VideoViewportSize
 import java.util.Date
 import java.util.Timer
+import java.util.TimerTask
 import kotlin.concurrent.schedule
 
 internal data class InitialStreamFace(val faceRect: RectF, val timestamp: Long)
 
 internal data class LivenessState(
     val sessionId: String,
     val context: Context,
+    val disableStartView: Boolean,
     val onCaptureReady: () -> Unit,
     val onFaceDistanceCheckPassed: () -> Unit,
     val onSessionError: (FaceLivenessDetectionException, Boolean) -> Unit,
-    val onFinalEventsSent: () -> Unit
+    val onFinalEventsSent: () -> Unit,
 ) {
     var videoViewportSize: VideoViewportSize? by mutableStateOf(null)
     var livenessCheckState = mutableStateOf<LivenessCheckState>(
@@ -58,13 +60,15 @@ internal data class LivenessState(
     var initialFaceDistanceCheckPassed by mutableStateOf(false)
     var initialLocalFaceFound by mutableStateOf(false)
 
+    var showingStartView by mutableStateOf(!disableStartView)
+
     private var initialStreamFace: InitialStreamFace? = null
     @VisibleForTesting
     var faceMatchOvalStart: Long? = null
     @VisibleForTesting
     var faceMatchOvalEnd: Long? = null
     private var initialFaceOvalIou = -1f
-    private var faceOvalMatchTimerStarted = false
+    private var faceOvalMatchTimer: TimerTask? = null
     private var detectedFaceMatchedOval = false
 
     @VisibleForTesting
@@ -85,6 +89,12 @@ internal data class LivenessState(
 
     fun onError(stopLivenessSession: Boolean) {
         livenessCheckState.value = LivenessCheckState.Error
+        onDestroy(stopLivenessSession)
+    }
+
+    // Cleans up state when challenge is completed or cancelled
+    fun onDestroy(stopLivenessSession: Boolean) {
+        faceOvalMatchTimer?.cancel()
         readyForOval = false
         faceGuideRect = null
         runningFreshness = false
@@ -127,28 +137,44 @@ internal data class LivenessState(
      * @return true if FrameAnalyzer should continue processing the frame
      */
     fun onFrameAvailable(): Boolean {
-        val livenessCheckState = livenessCheckState.value
-        if (livenessCheckState == LivenessCheckState.Error) return false
-        if (livenessCheckState !is LivenessCheckState.Success) return true
-
-        if (readyToSendFinalEvents) {
-            readyToSendFinalEvents = false
-
-            livenessSessionInfo!!.sendChallengeResponseEvent(
-                FaceTargetChallengeResponse(
-                    colorChallenge!!.challengeId,
-                    livenessCheckState.faceGuideRect,
-                    Date(faceMatchOvalStart!!),
-                    Date(faceMatchOvalEnd!!)
-                )
-            )
+        if (showingStartView) return false
+
+        return when (val livenessCheckState = livenessCheckState.value) {
+            is LivenessCheckState.Error -> false
+            is LivenessCheckState.Initial, is LivenessCheckState.Running -> {
+                /**
+                 * Start freshness check if the face has matched oval (we know this if faceMatchOvalStart is not null)
+                 * We trigger this in onFrameAvailable instead of onFrameFaceUpdate in the event the user moved the face
+                 * away from the camera. We want to run this check on every frame if the challenge is in process.
+                 */
+                if (!runningFreshness && colorChallenge?.challengeType ==
+                    ColorChallengeType.SEQUENTIAL &&
+                    faceMatchOvalStart?.let { (Date().time - it) > 1000 } == true
+                ) {
+                    runningFreshness = true
+                }
+                true
+            }
+            is LivenessCheckState.Success -> {
+                if (readyToSendFinalEvents) {
+                    readyToSendFinalEvents = false
+
+                    livenessSessionInfo!!.sendChallengeResponseEvent(
+                        FaceTargetChallengeResponse(
+                            colorChallenge!!.challengeId,
+                            livenessCheckState.faceGuideRect,
+                            Date(faceMatchOvalStart!!),
+                            Date(faceMatchOvalEnd!!)
+                        )
+                    )
 
-            // Send empty video event to signal we're done sending video
-            livenessSessionInfo!!.sendVideoEvent(VideoEvent(ByteArray(0), Date()))
-            onFinalEventsSent()
+                    // Send empty video event to signal we're done sending video
+                    livenessSessionInfo!!.sendVideoEvent(VideoEvent(ByteArray(0), Date()))
+                    onFinalEventsSent()
+                }
+                false
+            }
         }
-
-        return false
     }
 
     fun onFrameFaceCountUpdate(faceCount: Int) {
@@ -178,12 +204,19 @@ internal data class LivenessState(
         }
     }
 
+    /**
+     * returns true if face update inspect, false if thrown away
+     */
     fun onFrameFaceUpdate(
         faceRect: RectF,
         leftEye: FaceDetector.Landmark,
         rightEye: FaceDetector.Landmark,
         mouth: FaceDetector.Landmark
-    ) {
+    ): Boolean {
+        if (showingStartView) {
+            return false
+        }
+
         if (!initialFaceDistanceCheckPassed) {
             val faceDistance = FaceDetector.calculateFaceDistance(
                 leftEye, rightEye, mouth,
@@ -261,29 +294,25 @@ internal data class LivenessState(
 
             // Start timer and then timeout if the detected face doesn't match
             // the oval after a period of time
-            if (!detectedFaceMatchedOval && !faceOvalMatchTimerStarted) {
-                faceOvalMatchTimerStarted = true
-                Timer().schedule(faceTargetChallenge!!.faceTargetMatching.ovalFitTimeout.toLong()) {
-                    if (!detectedFaceMatchedOval && faceGuideRect != null) {
-                        readyForOval = false
-                        val timeoutError =
-                            FaceLivenessDetectionException(
-                                "Face did not match oval within time limit."
-                            )
-                        onSessionError(timeoutError, true)
+            if (!detectedFaceMatchedOval && faceOvalMatchTimer == null) {
+                faceOvalMatchTimer =
+                    Timer().schedule(faceTargetChallenge!!.faceTargetMatching.ovalFitTimeout.toLong()) {
+                        if (!detectedFaceMatchedOval && faceGuideRect != null) {
+                            readyForOval = false
+                            val timeoutError =
+                                FaceLivenessDetectionException(
+                                    "Face did not match oval within time limit."
+                                )
+                            onSessionError(timeoutError, true)
+                        }
+                        cancel()
                     }
-                    faceOvalMatchTimerStarted = false
-                    cancel()
-                }
-            }
-
-            // Start freshness check if it's not already started and face is in oval
-            if (!runningFreshness && colorChallenge?.challengeType ==
-                ColorChallengeType.SEQUENTIAL &&
-                faceOvalPosition == FaceDetector.FaceOvalPosition.MATCHED
-            ) {
-                runningFreshness = true
             }
         }
+        return true
+    }
+
+    fun onStartViewComplete() {
+        showingStartView = false
     }
 }

liveness/src/main/java/com/amplifyframework/ui/liveness/ui/FaceGuide.kt

@@ -40,7 +40,8 @@ import com.amplifyframework.ui.liveness.ui.helper.VideoViewportSize
 internal fun FaceGuide(
     modifier: Modifier,
     faceGuideRect: RectF?,
-    videoViewportSize: VideoViewportSize
+    videoViewportSize: VideoViewportSize,
+    backgroundColor: Color = Color.White
 ) {
 
     val scaledBoundingRect = faceGuideRect?.let {
@@ -50,7 +51,7 @@ internal fun FaceGuide(
     Canvas(modifier.graphicsLayer(alpha = 0.99f)) {
 
         drawRect(
-            color = Color.White,
+            color = backgroundColor,
             size = size
         )