Skip to content

Commit 4fa97f7

Browse files
jvh-awsjvh-aws
authored
fix(liveness): Improve liveness encoder and muxer creation error handling to propagate errors up early (#289)
1 parent 8b31810 commit 4fa97f7

File tree

6 files changed

+300
-44
lines changed

6 files changed

+300
-44
lines changed

‎liveness/api/liveness.api‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,20 @@ public final class com/amplifyframework/ui/liveness/model/FaceLivenessDetectionE
5858
public synthetic fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/lang/Throwable;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
5959
}
6060

61+
public final class com/amplifyframework/ui/liveness/model/FaceLivenessDetectionException$VideoEncodingException : com/amplifyframework/ui/liveness/model/FaceLivenessDetectionException {
62+
public static final field $stable I
63+
public fun <init> ()V
64+
public fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/lang/Throwable;)V
65+
public synthetic fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/lang/Throwable;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
66+
}
67+
68+
public final class com/amplifyframework/ui/liveness/model/FaceLivenessDetectionException$VideoMuxingException : com/amplifyframework/ui/liveness/model/FaceLivenessDetectionException {
69+
public static final field $stable I
70+
public fun <init> ()V
71+
public fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/lang/Throwable;)V
72+
public synthetic fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/lang/Throwable;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
73+
}
74+
6175
public final class com/amplifyframework/ui/liveness/state/AttemptCounter {
6276
public static final field $stable I
6377
public static final field ATTEMPT_COUNT_RESET_INTERVAL_MS J

‎liveness/src/main/java/com/amplifyframework/ui/liveness/camera/LivenessCoordinator.kt‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,14 +123,30 @@ internal class LivenessCoordinator(
123123
}
124124

125125
private val encoder = LivenessVideoEncoder.create(
126-
context = context,
126+
cacheDir = context.cacheDir,
127127
width = TARGET_WIDTH,
128128
height = TARGET_HEIGHT,
129129
bitrate = TARGET_ENCODE_BITRATE,
130130
framerate = TARGET_FPS_MAX,
131131
keyframeInterval = TARGET_ENCODE_KEYFRAME_INTERVAL,
132132
onMuxedSegment = { bytes, time ->
133133
livenessState.livenessSessionInfo?.sendVideoEvent(VideoEvent(bytes, Date(time)))
134+
},
135+
onEncoderError = { error ->
136+
processSessionError(
137+
FaceLivenessDetectionException.VideoEncodingException(
138+
throwable = error
139+
),
140+
true
141+
)
142+
},
143+
onMuxerError = { error ->
144+
processSessionError(
145+
FaceLivenessDetectionException.VideoMuxingException(
146+
throwable = error
147+
),
148+
true
149+
)
134150
}
135151
) ?: throw IllegalStateException("Failed to start the encoder.")
136152

‎liveness/src/main/java/com/amplifyframework/ui/liveness/camera/LivenessVideoEncoder.kt‎

Lines changed: 74 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,17 @@
1515

1616
package com.amplifyframework.ui.liveness.camera
1717

18-
import android.content.Context
1918
import android.media.MediaCodec
2019
import android.media.MediaCodecInfo
2120
import android.media.MediaFormat
2221
import android.os.Bundle
2322
import android.os.Handler
2423
import android.os.HandlerThread
2524
import android.util.Log
25+
import androidx.annotation.VisibleForTesting
2626
import androidx.annotation.WorkerThread
2727
import com.amplifyframework.core.Amplify
28+
import com.amplifyframework.logging.Logger
2829
import com.amplifyframework.ui.liveness.util.isKeyFrame
2930
import java.io.File
3031
import kotlin.coroutines.resume
@@ -37,23 +38,34 @@ internal class LivenessVideoEncoder private constructor(
3738
private val frameRate: Int,
3839
private val keyframeInterval: Int,
3940
private val outputFile: File,
40-
private val onMuxedSegment: OnMuxedSegment
41+
private val onMuxedSegment: OnMuxedSegment,
42+
private val onEncoderError: (MediaCodec.CodecException) -> Unit,
43+
private val onMuxerError: (Exception) -> Unit,
44+
private val muxerFactory: (File, MediaFormat, OnMuxedSegment) -> LivenessMuxer = { file, format, callback ->
45+
LivenessMuxer(file, format, callback)
46+
}
4147
) {
4248

4349
companion object {
4450

4551
const val TAG = "LivenessVideoEncoder"
4652
const val LOGGING_ENABLED = false
4753
const val MIME_TYPE = "video/x-vnd.on2.vp8"
54+
const val MAX_MUXER_CREATION_ATTEMPTS = 3
4855

4956
fun create(
50-
context: Context,
57+
cacheDir: File,
5158
width: Int,
5259
height: Int,
5360
bitrate: Int,
5461
framerate: Int,
5562
keyframeInterval: Int,
56-
onMuxedSegment: OnMuxedSegment
63+
onMuxedSegment: OnMuxedSegment,
64+
onEncoderError: (MediaCodec.CodecException) -> Unit,
65+
onMuxerError: (Exception) -> Unit,
66+
muxerFactory: (File, MediaFormat, OnMuxedSegment) -> LivenessMuxer = { file, format, callback ->
67+
LivenessMuxer(file, format, callback)
68+
}
5769
): LivenessVideoEncoder? {
5870
return try {
5971
LivenessVideoEncoder(
@@ -62,17 +74,20 @@ internal class LivenessVideoEncoder private constructor(
6274
bitrate,
6375
framerate,
6476
keyframeInterval,
65-
createTempOutputFile(context),
66-
onMuxedSegment
77+
createTempOutputFile(cacheDir),
78+
onMuxedSegment,
79+
onEncoderError,
80+
onMuxerError,
81+
muxerFactory
6782
)
6883
} catch (e: Exception) {
6984
null
7085
}
7186
}
7287

73-
private fun createTempOutputFile(context: Context) = File(
88+
private fun createTempOutputFile(cacheDir: File) = File(
7489
File(
75-
context.cacheDir,
90+
cacheDir,
7691
"amplify_liveness_temp"
7792
).apply {
7893
if (exists()) {
@@ -98,36 +113,17 @@ internal class LivenessVideoEncoder private constructor(
98113
}
99114

100115
private val encoderHandler = Handler(HandlerThread(TAG).apply { start() }.looper)
116+
private val logger = Amplify.Logging.forNamespace("Liveness")
101117

102118
private val encoder = MediaCodec.createEncoderByType(MIME_TYPE).apply {
103119
configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
104-
setCallback(
105-
object : MediaCodec.Callback() {
106-
override fun onInputBufferAvailable(codec: MediaCodec, index: Int) {
107-
}
108-
109-
override fun onOutputBufferAvailable(
110-
codec: MediaCodec,
111-
index: Int,
112-
info: MediaCodec.BufferInfo
113-
) {
114-
handleFrame(index, info)
115-
}
116-
117-
override fun onOutputFormatChanged(codec: MediaCodec, format: MediaFormat) {
118-
}
119-
120-
override fun onError(codec: MediaCodec, e: MediaCodec.CodecException) {
121-
}
122-
},
123-
encoderHandler
124-
)
120+
setCallback(EncoderCallback(::handleFrame, onEncoderError, logger), encoderHandler)
125121
}
126122
val inputSurface = encoder.createInputSurface()
127123

128124
private var encoding = false
129125
private var livenessMuxer: LivenessMuxer? = null
130-
private val logger = Amplify.Logging.forNamespace("Liveness")
126+
var muxerCreationAttempts = 0
131127

132128
init {
133129
encoder.start()
@@ -140,8 +136,8 @@ internal class LivenessVideoEncoder private constructor(
140136
var framesSinceSyncRequest = 0
141137

142138
@WorkerThread
143-
144139
fun handleFrame(outputBufferId: Int, info: MediaCodec.BufferInfo) {
140+
145141
try {
146142
encoder.getOutputBuffer(outputBufferId)?.let { byteBuffer ->
147143
if (encoding) {
@@ -156,18 +152,7 @@ internal class LivenessVideoEncoder private constructor(
156152

157153
if (info.isKeyFrame()) {
158154
if (livenessMuxer == null) {
159-
try {
160-
val muxer = LivenessMuxer(
161-
outputFile,
162-
encoder.outputFormat,
163-
onMuxedSegment
164-
)
165-
livenessMuxer = muxer
166-
} catch (e: Exception) {
167-
// This is likely an unrecoverable error, such as file creation failing.
168-
// However, if it fails, we will allow another attempt at the next keyframe.
169-
logger.error("Failed to create liveness muxer", e)
170-
}
155+
createMuxer()
171156
}
172157
framesSinceSyncRequest = 0 // reset keyframe request on keyframe receipt
173158
} else {
@@ -196,6 +181,28 @@ internal class LivenessVideoEncoder private constructor(
196181
}
197182
}
198183

184+
@VisibleForTesting()
185+
fun createMuxer() {
186+
muxerCreationAttempts++
187+
try {
188+
val muxer = muxerFactory(
189+
outputFile,
190+
encoder.outputFormat,
191+
onMuxedSegment
192+
)
193+
livenessMuxer = muxer
194+
} catch (e: Exception) {
195+
// This is likely an unrecoverable error, such as file creation failing.
196+
// However, if it fails, we will allow multiple attempt at the next keyframe.
197+
logger.error("Failed to create liveness muxer (attempt $muxerCreationAttempts)", e)
198+
if (muxerCreationAttempts >= MAX_MUXER_CREATION_ATTEMPTS) {
199+
// Propagate error up
200+
onMuxerError(e)
201+
return
202+
}
203+
}
204+
}
205+
199206
fun start() {
200207
encoderHandler.post {
201208
if (!encoding) {
@@ -248,3 +255,27 @@ internal class LivenessVideoEncoder private constructor(
248255
}
249256
}
250257
}
258+
259+
internal class EncoderCallback(
260+
private val handleFrame: (Int, MediaCodec.BufferInfo) -> Unit,
261+
private val onEncoderError: (MediaCodec.CodecException) -> Unit,
262+
private val logger: Logger
263+
) : MediaCodec.Callback() {
264+
265+
override fun onInputBufferAvailable(codec: MediaCodec, index: Int) {}
266+
267+
override fun onOutputBufferAvailable(codec: MediaCodec, index: Int, info: MediaCodec.BufferInfo) {
268+
handleFrame(index, info)
269+
}
270+
271+
override fun onOutputFormatChanged(codec: MediaCodec, format: MediaFormat) {}
272+
273+
override fun onError(codec: MediaCodec, e: MediaCodec.CodecException) {
274+
if (!e.isTransient) {
275+
logger.error("MediaCodec encoder error", e)
276+
onEncoderError(e)
277+
} else {
278+
logger.warn("Transient MediaCodec encoder error", e)
279+
}
280+
}
281+
}

‎liveness/src/main/java/com/amplifyframework/ui/liveness/model/FaceLivenessDetectionException.kt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,19 @@ open class FaceLivenessDetectionException(
5858
throwable: Throwable? = null
5959
) : FaceLivenessDetectionException(message, recoverySuggestion, throwable)
6060

61+
class VideoEncodingException(
62+
message: String = "Video encoding failed.",
63+
recoverySuggestion: String = "The device may not support video encoding. " +
64+
"Please try again or use a different device.",
65+
throwable: Throwable? = null
66+
) : FaceLivenessDetectionException(message, recoverySuggestion, throwable)
67+
68+
class VideoMuxingException(
69+
message: String = "Video muxer creation failed.",
70+
recoverySuggestion: String = "Retry the face liveness check.",
71+
throwable: Throwable? = null
72+
) : FaceLivenessDetectionException(message, recoverySuggestion, throwable)
73+
6174
/**
6275
* This is not an error we have determined to publicly expose.
6376
* The error will come to the customer in onError, but only instance checked as FaceLivenessDetectionException.

0 commit comments

Comments
 (0)