fix(liveness): use updated Liveness oval algorithm (#95)

Co-authored-by: Thomas Leing <[email protected]>
Co-authored-by: tjroach <[email protected]>

Author: 3 people

liveness/src/main/java/com/amplifyframework/ui/liveness/ml/FaceDetector.kt

@@ -17,6 +17,7 @@ package com.amplifyframework.ui.liveness.ml
 
 import android.content.Context
 import android.graphics.RectF
+import androidx.annotation.VisibleForTesting
 import com.amplifyframework.predictions.aws.models.FaceTargetMatchingParameters
 import com.amplifyframework.ui.liveness.R
 import com.amplifyframework.ui.liveness.camera.LivenessCoordinator.Companion.TARGET_HEIGHT
@@ -64,6 +65,14 @@ internal class FaceDetector(private val livenessState: LivenessState) {
             var mouthX = outputBoxes[0][i][10]
             var mouthY = outputBoxes[0][i][11]
 
+            // the face's right ear is actually the one on the left on screen, and vice versa.
+            // this is the same for the eyes, but we need the ears to be correct with respect to the
+            // bounding box for the algorithm to work.
+            var rightEarX = outputBoxes[0][i][12]
+            var rightEarY = outputBoxes[0][i][13]
+            var leftEarX = outputBoxes[0][i][14]
+            var leftEarY = outputBoxes[0][i][15]
+
             xCenter = xCenter / X_SCALE * anchors[i].w + anchors[i].xCenter
             yCenter = yCenter / Y_SCALE * anchors[i].h + anchors[i].yCenter
             h = h / H_SCALE * anchors[i].h
@@ -85,13 +94,20 @@ internal class FaceDetector(private val livenessState: LivenessState) {
             mouthX = mouthX / X_SCALE * anchors[i].w + anchors[i].xCenter
             mouthY = mouthY / Y_SCALE * anchors[i].h + anchors[i].yCenter
 
+            leftEarX = leftEarX / X_SCALE * anchors[i].w + anchors[i].xCenter
+            leftEarY = leftEarY / Y_SCALE * anchors[i].h + anchors[i].yCenter
+            rightEarX = rightEarX / X_SCALE * anchors[i].w + anchors[i].xCenter
+            rightEarY = rightEarY / Y_SCALE * anchors[i].h + anchors[i].yCenter
+
             detections.add(
                 Detection(
                     RectF(xMin, yMin, xMax, yMax),
                     Landmark(leftEyeX, leftEyeY),
                     Landmark(rightEyeX, rightEyeY),
                     Landmark(noseX, noseY),
                     Landmark(mouthX, mouthY),
+                    Landmark(leftEarX, leftEarY),
+                    Landmark(rightEarX, rightEarY),
                     score
                 )
             )
@@ -111,6 +127,8 @@ internal class FaceDetector(private val livenessState: LivenessState) {
         val renormalizedDetections = mutableListOf<Detection>()
         weightedDetections.forEach { detection ->
             // Change landmark coordinates to be for actual image size instead of model input size
+            val scaledBottom = (detection.location.bottom / Y_SCALE) * TARGET_HEIGHT
+
             val scaledLeftEyeX = (detection.leftEye.x / X_SCALE) * TARGET_WIDTH
             val scaledLeftEyeY = (detection.leftEye.y / Y_SCALE) * TARGET_HEIGHT
             val scaledRightEyeX = (detection.rightEye.x / X_SCALE) * TARGET_WIDTH
@@ -119,19 +137,28 @@ internal class FaceDetector(private val livenessState: LivenessState) {
             val scaledNoseY = (detection.nose.y / Y_SCALE) * TARGET_HEIGHT
             val scaledMouthX = (detection.mouth.x / X_SCALE) * TARGET_WIDTH
             val scaledMouthY = (detection.mouth.y / Y_SCALE) * TARGET_HEIGHT
+            val scaledLeftEarX = (detection.leftEar.x / X_SCALE) * TARGET_WIDTH
+            val scaledLeftEarY = (detection.leftEar.y / Y_SCALE) * TARGET_HEIGHT
+            val scaledRightEarX = (detection.rightEar.x / X_SCALE) * TARGET_WIDTH
+            val scaledRightEarY = (detection.rightEar.y / Y_SCALE) * TARGET_HEIGHT
 
             val scaledLeftEye = Landmark(scaledLeftEyeX, scaledLeftEyeY)
             val scaledRightEye = Landmark(scaledRightEyeX, scaledRightEyeY)
             val scaledNose = Landmark(scaledNoseX, scaledNoseY)
             val scaledMouth = Landmark(scaledMouthX, scaledMouthY)
+            val scaledLeftEar = Landmark(scaledLeftEarX, scaledLeftEarY)
+            val scaledRightEar = Landmark(scaledRightEarX, scaledRightEarY)
 
             // Generate the face bounding box from the landmarks
             val renormalizedBoundingBox =
                 generateBoundingBoxFromLandmarks(
+                    scaledBottom,
                     scaledLeftEye,
                     scaledRightEye,
                     scaledNose,
-                    scaledMouth
+                    scaledMouth,
+                    scaledLeftEar,
+                    scaledRightEar,
                 )
             renormalizedDetections.add(
                 Detection(
@@ -140,18 +167,23 @@ internal class FaceDetector(private val livenessState: LivenessState) {
                     scaledRightEye,
                     scaledNose,
                     scaledMouth,
+                    scaledLeftEar,
+                    scaledRightEar,
                     detection.score
                 )
             )
         }
         return renormalizedDetections
     }
 
-    private fun generateBoundingBoxFromLandmarks(
+    fun generateBoundingBoxFromLandmarks(
+        faceBottom: Float,
         leftEye: Landmark,
         rightEye: Landmark,
         nose: Landmark,
-        mouth: Landmark
+        mouth: Landmark,
+        leftEar: Landmark,
+        rightEar: Landmark
     ): RectF {
         val pupilDistance = calculatePupilDistance(leftEye, rightEye)
         val faceHeight = calculateFaceHeight(leftEye, rightEye, mouth)
@@ -163,18 +195,16 @@ internal class FaceDetector(private val livenessState: LivenessState) {
         val eyeCenterY = (leftEye.y + rightEye.y) / 2
 
         var cx = eyeCenterX
-        var cy = eyeCenterY
         val ovalInfo = livenessState.faceTargetChallenge
         if (ovalInfo != null && eyeCenterY <= ovalInfo.targetCenterY / 2) {
             cx = (eyeCenterX + nose.x) / 2
-            cy = (eyeCenterY + nose.y) / 2
         }
 
-        val left = cx - ow / 2
-        val top = cy - oh / 2
-        val right = left + ow
-        val bottom = top + oh
-        return RectF(left, top, right, bottom)
+        val top = faceBottom - oh
+        val left = min(cx - ow / 2, rightEar.x)
+        val right = max(cx + ow / 2, leftEar.x)
+
+        return RectF(left, top, right, faceBottom)
     }
 
     private fun generateAnchors(): List<Anchor> {
@@ -284,6 +314,8 @@ internal class FaceDetector(private val livenessState: LivenessState) {
             var weightedRightEye = detection.rightEye
             var weightedNose = detection.nose
             var weightedMouth = detection.mouth
+            var weightedLeftEar = detection.leftEar
+            var weightedRightEar = detection.rightEar
             if (candidates.isNotEmpty()) {
                 var wXMin = 0.0f
                 var wYMin = 0.0f
@@ -297,6 +329,10 @@ internal class FaceDetector(private val livenessState: LivenessState) {
                 var wNoseY = 0.0f
                 var wMouthX = 0.0f
                 var wMouthY = 0.0f
+                var wLeftEarX = 0.0f
+                var wLeftEarY = 0.0f
+                var wRightEarX = 0.0f
+                var wRightEarY = 0.0f
                 var totalScore = 0.0f
                 candidates.forEach { candidate ->
                     totalScore += candidate.score
@@ -305,6 +341,8 @@ internal class FaceDetector(private val livenessState: LivenessState) {
                     val rightEye = detections[candidate.index].rightEye
                     val nose = detections[candidate.index].nose
                     val mouth = detections[candidate.index].mouth
+                    val leftEar = detections[candidate.index].leftEar
+                    val rightEar = detections[candidate.index].rightEar
 
                     wXMin += bbox.left * candidate.score
                     wYMin += bbox.top * candidate.score
@@ -321,6 +359,11 @@ internal class FaceDetector(private val livenessState: LivenessState) {
 
                     wMouthX += mouth.x * candidate.score
                     wMouthY += mouth.y * candidate.score
+
+                    wLeftEarX += leftEar.x * candidate.score
+                    wLeftEarY += leftEar.y * candidate.score
+                    wRightEarX += rightEar.x * candidate.score
+                    wRightEarY += rightEar.y * candidate.score
                 }
                 weightedLocation.left = wXMin / totalScore * INPUT_SIZE_WIDTH
                 weightedLocation.top = wYMin / totalScore * INPUT_SIZE_HEIGHT
@@ -342,6 +385,13 @@ internal class FaceDetector(private val livenessState: LivenessState) {
                 val weightedMouthX = wMouthX / totalScore * INPUT_SIZE_WIDTH
                 val weightedMouthY = wMouthY / totalScore * INPUT_SIZE_HEIGHT
                 weightedMouth = Landmark(weightedMouthX, weightedMouthY)
+
+                val weightedLeftEarX = wLeftEarX / totalScore * INPUT_SIZE_WIDTH
+                val weightedLeftEarY = wLeftEarY / totalScore * INPUT_SIZE_HEIGHT
+                weightedLeftEar = Landmark(weightedLeftEarX, weightedLeftEarY)
+                val weightedRightEarX = wRightEarX / totalScore * INPUT_SIZE_WIDTH
+                val weightedRightEarY = wRightEarY / totalScore * INPUT_SIZE_HEIGHT
+                weightedRightEar = Landmark(weightedRightEarX, weightedRightEarY)
             }
             remainedIndexedScores.clear()
             remainedIndexedScores.addAll(remained)
@@ -351,6 +401,8 @@ internal class FaceDetector(private val livenessState: LivenessState) {
                 weightedRightEye,
                 weightedNose,
                 weightedMouth,
+                weightedLeftEar,
+                weightedRightEar,
                 detection.score
             )
             outputLocations.add(weightedDetection)
@@ -382,6 +434,8 @@ internal class FaceDetector(private val livenessState: LivenessState) {
         val rightEye: Landmark,
         val nose: Landmark,
         val mouth: Landmark,
+        val leftEar: Landmark,
+        val rightEar: Landmark,
         val score: Float
     )
     private class IndexedScore(val index: Int, val score: Float)
@@ -508,11 +562,13 @@ internal class FaceDetector(private val livenessState: LivenessState) {
             return calibratedPupilDistance / ovalWidth
         }
 
-        private fun calculatePupilDistance(leftEye: Landmark, rightEye: Landmark): Float {
+        @VisibleForTesting(VisibleForTesting.PRIVATE)
+        internal fun calculatePupilDistance(leftEye: Landmark, rightEye: Landmark): Float {
             return sqrt((leftEye.x - rightEye.x).pow(2) + (leftEye.y - rightEye.y).pow(2))
         }
 
-        private fun calculateFaceHeight(leftEye: Landmark, rightEye: Landmark, mouth: Landmark):
+        @VisibleForTesting(VisibleForTesting.PRIVATE)
+        internal fun calculateFaceHeight(leftEye: Landmark, rightEye: Landmark, mouth: Landmark):
             Float {
             val eyeCenterX = (leftEye.x + rightEye.x) / 2
             val eyeCenterY = (leftEye.y + rightEye.y) / 2

liveness/src/test/java/com/amplifyframework/ui/liveness/ml/FaceDetectorTest.kt

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package com.amplifyframework.ui.liveness.ml
+
+import com.amplifyframework.ui.liveness.ml.FaceDetector.Landmark
+import com.amplifyframework.ui.liveness.state.LivenessState
+import io.mockk.every
+import io.mockk.mockk
+import org.junit.Assert.assertEquals
+import org.junit.Before
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.robolectric.RobolectricTestRunner
+
+@RunWith(RobolectricTestRunner::class)
+internal class FaceDetectorTest {
+    private lateinit var detector: FaceDetector
+
+    // given
+    private val faceBottom = 0.88746625f
+    private val leftEye = Landmark(0.668633f, 0.48738188f)
+    private val rightEye = Landmark(0.35714725f, 0.46644497f)
+    private val nose = Landmark(0.52836484f, 0.53194016f)
+    private val mouth = Landmark(0.5062596f, 0.68926525f)
+    private val leftEar = Landmark(0.78989476f, 0.5973732f)
+    private val rightEar = Landmark(0.16585289f, 0.5668279f)
+    private val width = 0.65490234f
+    private val height = 0.49117205f
+
+    @Before
+    fun setup() {
+        val state = mockk<LivenessState> {
+            every { faceTargetChallenge } returns null
+        }
+        detector = FaceDetector(state)
+    }
+
+    @Test
+    fun `test detected face`() {
+        val faceDistance = FaceDetector.calculateFaceDistance(leftEye, rightEye, mouth, 1, 1)
+        assertApprox(0.31462398f, faceDistance)
+        val pupilDistance = FaceDetector.calculatePupilDistance(leftEye, rightEye)
+        assertApprox(0.3121886f, pupilDistance)
+        val faceHeight = FaceDetector.calculateFaceHeight(leftEye, rightEye, mouth)
+        assertApprox(0.21245532f, faceHeight)
+    }
+
+    @Test
+    fun `test detected bounding box`() {
+        // when
+        val boundingBox = detector.generateBoundingBoxFromLandmarks(
+            faceBottom,
+            leftEye,
+            rightEye,
+            nose,
+            mouth,
+            leftEar,
+            rightEar
+        )
+
+        // then
+        assertApprox(0.16585289f, boundingBox.left)
+        assertApprox(0.07296771f, boundingBox.top)
+        assertApprox(0.16585289f + 0.62404186f, boundingBox.right)
+        assertApprox(0.07296771f + 0.8144985f, boundingBox.bottom)
+    }
+
+    private fun assertApprox(expected: Float, actual: Float, delta: Float = 0.000001f) {
+        assertEquals(expected, actual, delta)
+    }
+}