feat(liveness): Add support for a no light liveness challenge

gradle/libs.versions.toml

@@ -1,7 +1,7 @@
 [versions]
 accompanist = "0.28.0"
 agp = "8.7.2"
-amplify = "2.27.2"
+amplify = "2.29.0"
 appcompat = "1.6.1"
 androidx-core = "1.9.0"
 androidx-junit = "1.1.4"

liveness/api/liveness.api

@@ -44,13 +44,72 @@ public final class com/amplifyframework/ui/liveness/model/FaceLivenessDetectionE
 	public synthetic fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/lang/Throwable;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
 }
 
+public final class com/amplifyframework/ui/liveness/model/FaceLivenessDetectionException$UnsupportedChallengeTypeException : com/amplifyframework/ui/liveness/model/FaceLivenessDetectionException {
+	public static final field $stable I
+	public fun <init> ()V
+	public fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/lang/Throwable;)V
+	public synthetic fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/lang/Throwable;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
+}
+
 public final class com/amplifyframework/ui/liveness/model/FaceLivenessDetectionException$UserCancelledException : com/amplifyframework/ui/liveness/model/FaceLivenessDetectionException {
 	public static final field $stable I
 	public fun <init> ()V
 	public fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/lang/Throwable;)V
 	public synthetic fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/lang/Throwable;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
 }
 
+public final class com/amplifyframework/ui/liveness/state/AttemptCounter {
+	public static final field $stable I
+	public static final field ATTEMPT_COUNT_RESET_INTERVAL_MS J
+	public static final field Companion Lcom/amplifyframework/ui/liveness/state/AttemptCounter$Companion;
+	public fun <init> ()V
+	public final fun countAttempt ()V
+	public final fun getCount ()I
+}
+
+public final class com/amplifyframework/ui/liveness/state/AttemptCounter$Companion {
+	public final fun getAttemptCount ()I
+	public final fun getLatestAttemptTimeStamp ()J
+	public final fun setAttemptCount (I)V
+	public final fun setLatestAttemptTimeStamp (J)V
+}
+
+public abstract class com/amplifyframework/ui/liveness/ui/Camera {
+	public static final field $stable I
+}
+
+public final class com/amplifyframework/ui/liveness/ui/Camera$Back : com/amplifyframework/ui/liveness/ui/Camera {
+	public static final field $stable I
+	public static final field INSTANCE Lcom/amplifyframework/ui/liveness/ui/Camera$Back;
+	public fun equals (Ljava/lang/Object;)Z
+	public fun hashCode ()I
+	public fun toString ()Ljava/lang/String;
+}
+
+public final class com/amplifyframework/ui/liveness/ui/Camera$Front : com/amplifyframework/ui/liveness/ui/Camera {
+	public static final field $stable I
+	public static final field INSTANCE Lcom/amplifyframework/ui/liveness/ui/Camera$Front;
+	public fun equals (Ljava/lang/Object;)Z
+	public fun hashCode ()I
+	public fun toString ()Ljava/lang/String;
+}
+
+public final class com/amplifyframework/ui/liveness/ui/ChallengeOptions {
+	public static final field $stable I
+	public fun <init> ()V
+	public fun <init> (Lcom/amplifyframework/ui/liveness/ui/LivenessChallenge$FaceMovementAndLight;Lcom/amplifyframework/ui/liveness/ui/LivenessChallenge$FaceMovement;)V
+	public synthetic fun <init> (Lcom/amplifyframework/ui/liveness/ui/LivenessChallenge$FaceMovementAndLight;Lcom/amplifyframework/ui/liveness/ui/LivenessChallenge$FaceMovement;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
+	public final fun component1 ()Lcom/amplifyframework/ui/liveness/ui/LivenessChallenge$FaceMovementAndLight;
+	public final fun component2 ()Lcom/amplifyframework/ui/liveness/ui/LivenessChallenge$FaceMovement;
+	public final fun copy (Lcom/amplifyframework/ui/liveness/ui/LivenessChallenge$FaceMovementAndLight;Lcom/amplifyframework/ui/liveness/ui/LivenessChallenge$FaceMovement;)Lcom/amplifyframework/ui/liveness/ui/ChallengeOptions;
+	public static synthetic fun copy$default (Lcom/amplifyframework/ui/liveness/ui/ChallengeOptions;Lcom/amplifyframework/ui/liveness/ui/LivenessChallenge$FaceMovementAndLight;Lcom/amplifyframework/ui/liveness/ui/LivenessChallenge$FaceMovement;ILjava/lang/Object;)Lcom/amplifyframework/ui/liveness/ui/ChallengeOptions;
+	public fun equals (Ljava/lang/Object;)Z
+	public final fun getFaceMovement ()Lcom/amplifyframework/ui/liveness/ui/LivenessChallenge$FaceMovement;
+	public final fun getFaceMovementAndLight ()Lcom/amplifyframework/ui/liveness/ui/LivenessChallenge$FaceMovementAndLight;
+	public fun hashCode ()I
+	public fun toString ()Ljava/lang/String;
+}
+
 public final class com/amplifyframework/ui/liveness/ui/ComposableSingletons$CancelChallengeButtonKt {
 	public static final field INSTANCE Lcom/amplifyframework/ui/liveness/ui/ComposableSingletons$CancelChallengeButtonKt;
 	public static field lambda-1 Lkotlin/jvm/functions/Function2;
@@ -110,6 +169,36 @@ public final class com/amplifyframework/ui/liveness/ui/ComposableSingletons$Reco
 
 public final class com/amplifyframework/ui/liveness/ui/FaceLivenessDetectorKt {
 	public static final fun FaceLivenessDetector (Ljava/lang/String;Ljava/lang/String;Lcom/amplifyframework/auth/AWSCredentialsProvider;ZLcom/amplifyframework/core/Action;Lcom/amplifyframework/core/Consumer;Landroidx/compose/runtime/Composer;II)V
+	public static final fun FaceLivenessDetector (Ljava/lang/String;Ljava/lang/String;Lcom/amplifyframework/auth/AWSCredentialsProvider;ZLcom/amplifyframework/core/Action;Lcom/amplifyframework/core/Consumer;Lcom/amplifyframework/ui/liveness/ui/ChallengeOptions;Landroidx/compose/runtime/Composer;II)V
+}
+
+public abstract class com/amplifyframework/ui/liveness/ui/LivenessChallenge {
+	public static final field $stable I
+	public synthetic fun <init> (Lcom/amplifyframework/ui/liveness/ui/Camera;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
+	public synthetic fun <init> (Lcom/amplifyframework/ui/liveness/ui/Camera;Lkotlin/jvm/internal/DefaultConstructorMarker;)V
+	public fun getCamera ()Lcom/amplifyframework/ui/liveness/ui/Camera;
+}
+
+public final class com/amplifyframework/ui/liveness/ui/LivenessChallenge$FaceMovement : com/amplifyframework/ui/liveness/ui/LivenessChallenge {
+	public static final field $stable I
+	public fun <init> ()V
+	public fun <init> (Lcom/amplifyframework/ui/liveness/ui/Camera;)V
+	public synthetic fun <init> (Lcom/amplifyframework/ui/liveness/ui/Camera;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
+	public final fun component1 ()Lcom/amplifyframework/ui/liveness/ui/Camera;
+	public final fun copy (Lcom/amplifyframework/ui/liveness/ui/Camera;)Lcom/amplifyframework/ui/liveness/ui/LivenessChallenge$FaceMovement;
+	public static synthetic fun copy$default (Lcom/amplifyframework/ui/liveness/ui/LivenessChallenge$FaceMovement;Lcom/amplifyframework/ui/liveness/ui/Camera;ILjava/lang/Object;)Lcom/amplifyframework/ui/liveness/ui/LivenessChallenge$FaceMovement;
+	public fun equals (Ljava/lang/Object;)Z
+	public fun getCamera ()Lcom/amplifyframework/ui/liveness/ui/Camera;
+	public fun hashCode ()I
+	public fun toString ()Ljava/lang/String;
+}
+
+public final class com/amplifyframework/ui/liveness/ui/LivenessChallenge$FaceMovementAndLight : com/amplifyframework/ui/liveness/ui/LivenessChallenge {
+	public static final field $stable I
+	public static final field INSTANCE Lcom/amplifyframework/ui/liveness/ui/LivenessChallenge$FaceMovementAndLight;
+	public fun equals (Ljava/lang/Object;)Z
+	public fun hashCode ()I
+	public fun toString ()Ljava/lang/String;
 }
 
 public final class com/amplifyframework/ui/liveness/ui/LivenessColorScheme {

liveness/src/main/java/com/amplifyframework/ui/liveness/camera/LivenessCoordinator.kt

@@ -35,15 +35,20 @@
 import com.amplifyframework.predictions.aws.exceptions.AccessDeniedException
 import com.amplifyframework.predictions.aws.exceptions.FaceLivenessSessionNotFoundException
 import com.amplifyframework.predictions.aws.exceptions.FaceLivenessSessionTimeoutException
+import com.amplifyframework.predictions.aws.exceptions.FaceLivenessUnsupportedChallengeTypeException
 import com.amplifyframework.predictions.aws.models.ColorChallengeResponse
 import com.amplifyframework.predictions.aws.models.RgbColor
 import com.amplifyframework.predictions.aws.options.AWSFaceLivenessSessionOptions
+import com.amplifyframework.predictions.models.Challenge
 import com.amplifyframework.predictions.models.FaceLivenessSessionInformation
 import com.amplifyframework.predictions.models.VideoEvent
 import com.amplifyframework.ui.liveness.BuildConfig
 import com.amplifyframework.ui.liveness.model.FaceLivenessDetectionException
 import com.amplifyframework.ui.liveness.model.LivenessCheckState
+import com.amplifyframework.ui.liveness.state.AttemptCounter
 import com.amplifyframework.ui.liveness.state.LivenessState
+import com.amplifyframework.ui.liveness.ui.Camera
+import com.amplifyframework.ui.liveness.ui.ChallengeOptions
 import com.amplifyframework.ui.liveness.util.WebSocketCloseCode
 import java.util.Date
 import java.util.concurrent.Executors
@@ -65,25 +70,26 @@
 @SuppressLint("UnsafeOptInUsageError")
 internal class LivenessCoordinator(
     val context: Context,
-    lifecycleOwner: LifecycleOwner,
+    private val lifecycleOwner: LifecycleOwner,
     private val sessionId: String,
     private val region: String,
     private val credentialsProvider: AWSCredentialsProvider<AWSCredentials>?,
-    disableStartView: Boolean,
+    private val disableStartView: Boolean,
+    private val challengeOptions: ChallengeOptions,
     private val onChallengeComplete: OnChallengeComplete,
     val onChallengeFailed: Consumer<FaceLivenessDetectionException>
 ) {
 
+    private val attemptCounter = AttemptCounter()
     private val analysisExecutor = Executors.newSingleThreadExecutor()
 
     val livenessState = LivenessState(
-        sessionId,
-        context,
-        disableStartView,
-        this::processCaptureReady,
-        this::startLivenessSession,
-        this::processSessionError,
-        this::processFinalEventsSent
+        sessionId = sessionId,
+        context = context,
+        disableStartView = disableStartView,
+        onCaptureReady = this::processCaptureReady,
+        onSessionError = this::processSessionError,
+        onFinalEventsSent = this::processFinalEventsSent
     )
 
     private val preview = Preview.Builder().apply {
@@ -138,6 +144,15 @@
     private var disconnectEventReceived = false
 
     init {
+        startLivenessSession()
+        if (challengeOptions.hasOneCameraConfigured()) {
+            launchCamera(challengeOptions.faceMovementAndLight.camera)
+        } else {
+            livenessState.loadingCameraPreview = true
+        }
+    }
+
+    private fun launchCamera(camera: Camera) {
         MainScope().launch {
             delay(5_000)
             if (!previewTextureView.hasReceivedUpdate) {
@@ -152,17 +167,24 @@
             getCameraProvider(context).apply {
                 if (lifecycleOwner.lifecycle.currentState != Lifecycle.State.DESTROYED) {
                     unbindAll()
-                    if (this.hasCamera(CameraSelector.DEFAULT_FRONT_CAMERA)) {
+
+                    val (chosenCamera, orientation) = when (camera) {
+                        Camera.Front -> Pair(CameraSelector.DEFAULT_FRONT_CAMERA, "front")
+                        Camera.Back -> Pair(CameraSelector.DEFAULT_BACK_CAMERA, "back")
+                    }
+
+                    if (this.hasCamera(chosenCamera)) {
                         bindToLifecycle(
                             lifecycleOwner,
-                            CameraSelector.DEFAULT_FRONT_CAMERA,
+                            chosenCamera,
                             preview,
                             analysis
                         )
                     } else {
+                        livenessState.loadingCameraPreview = false
                         val faceLivenessException = FaceLivenessDetectionException(
-                            "A front facing camera is required but no front facing camera detected.",
-                            "Enable a front facing camera."
+                            "A $orientation facing camera is required but no $orientation facing camera detected.",
+                            "Enable a $orientation facing camera."
                         )
                         processSessionError(faceLivenessException, true)
                     }
@@ -172,13 +194,19 @@
     }
 
     private fun startLivenessSession() {
-        livenessState.livenessCheckState.value = LivenessCheckState.Initial.withConnectingMessage()
+        livenessState.livenessCheckState = LivenessCheckState.Initial.withConnectingMessage()
+        attemptCounter.countAttempt()
 
         val faceLivenessSessionInformation = FaceLivenessSessionInformation(
-            TARGET_WIDTH.toFloat(),
-            TARGET_HEIGHT.toFloat(),
-            "FaceMovementAndLightChallenge_1.0.0",
-            region
+            videoWidth = TARGET_WIDTH.toFloat(),
+            videoHeight = TARGET_HEIGHT.toFloat(),
+            challengeVersions = listOf(
+                Challenge.FaceMovementAndLightChallenge("2.0.0"),
+                Challenge.FaceMovementChallenge("1.0.0")
+            ),
+            region = region,
+            preCheckViewEnabled = !disableStartView,
+            attemptCount = attemptCounter.getCount()
         )
 
         val faceLivenessSessionOptions = AWSFaceLivenessSessionOptions.builder().apply {
@@ -190,25 +218,33 @@
             faceLivenessSessionInformation,
             faceLivenessSessionOptions,
             BuildConfig.LIVENESS_VERSION_NAME,
-            { livenessState.onLivenessSessionReady(it) },
+            {
+                livenessState.onLivenessSessionReady(it)
+                if (!challengeOptions.hasOneCameraConfigured()) {
+                    val foundChallenge = challengeOptions.getLivenessChallenge(it.challengeType)
+                    launchCamera(foundChallenge.camera)
+                }
+            },
             {
                 disconnectEventReceived = true
                 onChallengeComplete()
             },
             { error ->
-                val faceLivenessException = when (error) {
+                val (faceLivenessException, shouldStopLivenessSession) = when (error) {
                     is AccessDeniedException ->
-                        FaceLivenessDetectionException.AccessDeniedException(throwable = error)
+                        FaceLivenessDetectionException.AccessDeniedException(throwable = error) to false
                     is FaceLivenessSessionNotFoundException ->
-                        FaceLivenessDetectionException.SessionNotFoundException(throwable = error)
+                        FaceLivenessDetectionException.SessionNotFoundException(throwable = error) to false
                     is FaceLivenessSessionTimeoutException ->
-                        FaceLivenessDetectionException.SessionTimedOutException(throwable = error)
+                        FaceLivenessDetectionException.SessionTimedOutException(throwable = error) to false
+                    is FaceLivenessUnsupportedChallengeTypeException ->
+                        FaceLivenessDetectionException.UnsupportedChallengeTypeException(throwable = error) to true
                     else -> FaceLivenessDetectionException(
                         error.message ?: "Unknown error.",
                         error.recoverySuggestion, error
-                    )
+                    ) to false
                 }
-                processSessionError(faceLivenessException, false)
+                processSessionError(faceLivenessException, shouldStopLivenessSession)
             }
         )
     }
@@ -256,8 +292,8 @@
         )
     }
 
-    fun processFreshnessChallengeComplete() {
-        livenessState.onFreshnessComplete()
+    fun processLivenessCheckComplete() {
+        livenessState.onLivenessChallengeComplete()
         stopEncoder { livenessState.onFullChallengeComplete() }
     }
 

liveness/src/main/java/com/amplifyframework/ui/liveness/ml/FaceDetector.kt

@@ -42,10 +42,12 @@
         outputScores: Array<Array<FloatArray>>
     ): List<Detection> {
         val detections = mutableListOf<Detection>()
+        val faceTargetChallenge = livenessState.faceTargetChallenge ?: return emptyList()
         for (i in 0 until NUM_BOXES) {
             var score = outputScores[0][i][0]
             score = computeSigmoid(score)
-            if (score < MIN_SCORE_THRESHOLD) {
+
+            if (score < faceTargetChallenge.faceTargetMatching.faceDetectionThreshold) {
                 continue
             }
 
@@ -159,6 +161,7 @@
                     scaledMouth,
                     scaledLeftEar,
                     scaledRightEar,
+                    faceTargetChallenge.faceTargetMatching.targetHeightWidthRatio
                 )
             renormalizedDetections.add(
                 Detection(
@@ -183,13 +186,14 @@
         nose: Landmark,
         mouth: Landmark,
         leftEar: Landmark,
-        rightEar: Landmark
+        rightEar: Landmark,
+        heightWidthRatio: Float
     ): RectF {
         val pupilDistance = calculatePupilDistance(leftEye, rightEye)
         val faceHeight = calculateFaceHeight(leftEye, rightEye, mouth)
 
         val ow = (ALPHA * pupilDistance + GAMMA * faceHeight) / 2
-        val oh = GOLDEN_RATIO * ow
+        val oh = heightWidthRatio * ow
 
         val eyeCenterX = (leftEye.x + rightEye.x) / 2
         val eyeCenterY = (leftEye.y + rightEye.y) / 2
@@ -450,7 +454,6 @@
 
     companion object {
         private const val MIN_SUPPRESSION_THRESHOLD = 0.3f
-        private const val MIN_SCORE_THRESHOLD = 0.7f
         private val strides = listOf(8, 16, 16, 16)
         private const val ASPECT_RATIOS_SIZE = 1
         private const val MIN_SCALE = 0.1484375f
@@ -459,7 +462,6 @@
         private const val ANCHOR_OFFSET_Y = 0.5f
         private const val INPUT_SIZE_HEIGHT = 128
         private const val INPUT_SIZE_WIDTH = 128
-        private const val GOLDEN_RATIO = 1.618f
         private const val ALPHA = 2.0f
         private const val GAMMA = 1.8f
         const val X_SCALE = 128f
@@ -479,7 +481,6 @@
          * 14, 15 - right eye tragion
          */
         const val NUM_COORDS = 16
-        const val INITIAL_FACE_DISTANCE_THRESHOLD = 0.32f
 
         fun loadModel(context: Context): Interpreter {
             val modelFileDescriptor =

liveness/src/main/java/com/amplifyframework/ui/liveness/model/FaceLivenessDetectionException.kt

@@ -45,6 +45,13 @@
         throwable: Throwable? = null
     ) : FaceLivenessDetectionException(message, recoverySuggestion, throwable)
 
+    class UnsupportedChallengeTypeException(
+        message: String = "Received an unsupported ChallengeType from the backend.",
+        recoverySuggestion: String = "Verify that the Challenges configured in your backend are supported by " +
+            "this library.",
+        throwable: Throwable? = null
+    ) : FaceLivenessDetectionException(message, recoverySuggestion, throwable)
+
     class UserCancelledException(
         message: String = "User cancelled the face liveness check.",
         recoverySuggestion: String = "Retry the face liveness check.",