feat: Android double masking with frame drop

e2e/android/app/src/main/java/com/example/androidobservability/masking/ComposeUserForm.kt

@@ -89,8 +89,7 @@ fun UserInfoForm(modifier: Modifier = Modifier) {
 	) {
 		Text(
 			text = "User Information Form",
-			style = MaterialTheme.typography.headlineMedium,
-			modifier = Modifier.ldMask()
+			style = MaterialTheme.typography.headlineMedium
 		)
 
 		// Password Section
@@ -112,7 +111,8 @@ fun UserInfoForm(modifier: Modifier = Modifier) {
 					label = { Text("Password") },
 					visualTransformation = PasswordVisualTransformation(),
 					keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Password),
-					modifier = Modifier.fillMaxWidth()
+					modifier = Modifier
+                        .fillMaxWidth()
 				)
 			}
 		}
@@ -160,7 +160,9 @@ fun UserInfoForm(modifier: Modifier = Modifier) {
 					onValueChange = { zipCode = it },
 					label = { Text("ZIP Code") },
 					keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Number),
-					modifier = Modifier.fillMaxWidth()
+					modifier = Modifier
+                        .fillMaxWidth()
+                        .ldMask()
 				)
 			}
 		}
@@ -208,8 +210,10 @@ fun UserInfoForm(modifier: Modifier = Modifier) {
 						label = { Text("CVV") },
 						keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Number),
 						visualTransformation = PasswordVisualTransformation(),
-						modifier = Modifier.weight(1f)
-					)
+						modifier = Modifier
+                            .weight(1f)
+                            .ldMask()
+                    )
 				}
 			}
 		}

e2e/android/app/src/main/java/com/example/androidobservability/masking/XMLMaskingActivity.kt

@@ -11,7 +11,6 @@ import android.widget.Toast
 import android.graphics.Color
 import android.graphics.PixelFormat
 import android.view.Gravity
-import android.view.ViewGroup
 import android.view.WindowManager
 import android.widget.FrameLayout
 import android.widget.TextView

sdk/@launchdarkly/observability-android/lib/src/main/kotlin/com/launchdarkly/observability/replay/capture/CaptureSource.kt

@@ -2,8 +2,6 @@ package com.launchdarkly.observability.replay.capture
 
 import android.graphics.Bitmap
 import android.graphics.Canvas
-import android.graphics.Color
-import android.graphics.Paint
 import android.graphics.Rect
 import android.os.Build
 import android.os.Handler
@@ -16,7 +14,6 @@ import android.view.Window
 import android.view.WindowManager.LayoutParams.TYPE_APPLICATION
 import android.view.WindowManager.LayoutParams.TYPE_BASE_APPLICATION
 import androidx.annotation.RequiresApi
-import android.graphics.Path
 import com.launchdarkly.logging.LDLogger
 import com.launchdarkly.observability.coroutines.DispatcherProviderHolder
 import com.launchdarkly.observability.replay.masking.MaskMatcher
@@ -33,6 +30,8 @@ import kotlin.coroutines.resume
 import androidx.core.graphics.withTranslation
 import com.launchdarkly.observability.replay.masking.Mask
 import androidx.core.graphics.createBitmap
+import com.launchdarkly.observability.replay.masking.MaskApplier
+import kotlin.collections.mutableMapOf
 
 /**
  * A source of [CaptureEvent]s taken from the lowest visible window. Captures
@@ -49,21 +48,19 @@ class CaptureSource(
     data class CaptureResult(
         val windowEntry: WindowEntry,
         val bitmap: Bitmap,
-        val masks: List<Mask>
     )
 
     private val _captureEventFlow = MutableSharedFlow<CaptureEvent>()
     val captureFlow: SharedFlow<CaptureEvent> = _captureEventFlow.asSharedFlow()
     private val windowInspector = WindowInspector(logger)
     private val maskCollector = MaskCollector(logger)
-    private val maskPaint = Paint().apply {
-        color = Color.GRAY
-        style = Paint.Style.FILL
-    }
+    private val maskApplier = MaskApplier()
 
     private val tiledSignatureManager = TiledSignatureManager()
+
     @Volatile
     private var tiledSignature: TiledSignature? = null
+
     /**
      * Requests a [CaptureEvent] be taken now.
      */
@@ -96,7 +93,8 @@ class CaptureSource(
                 return@withContext null
             }
 
-            val baseWindowEntry = pickBaseWindow(windowsEntries) ?: return@withContext null
+            val baseIndex = pickBaseWindow(windowsEntries) ?: return@withContext null
+            val baseWindowEntry = windowsEntries[baseIndex]
             val rect = baseWindowEntry.rect()
 
             // protect against race condition where decor view has no size
@@ -107,36 +105,60 @@ class CaptureSource(
             // TODO: O11Y-625 - optimize memory allocations
             // TODO: O11Y-625 - see if holding bitmap is more efficient than base64 encoding immediately after compression
             // TODO: O11Y-628 - use captureQuality option for scaling and adjust this bitmap accordingly, may need to investigate power of 2 rounding for performance
-            // Create a bitmap with the window dimensions
-            val baseResult = captureViewResult(baseWindowEntry) ?: return@withContext null
-
-            // capture rest of views on top of base
-            val pairs = mutableListOf<CaptureResult>()
-            var afterBase = false
-            for (windowEntry in windowsEntries) {
-                if (afterBase) {
-                    captureViewResult(windowEntry)?.let { result ->
-                        pairs.add(result)
+
+            val capturingWindowEntries = windowsEntries.subList(baseIndex, windowsEntries.size)
+
+            var beforeMasks = collectMasks(capturingWindowEntries)
+
+            val captureResults: MutableList<CaptureResult?> = MutableList(capturingWindowEntries.size) { null }
+            var captured = 0
+            for (i in capturingWindowEntries.indices) {
+                val windowEntry = capturingWindowEntries[i]
+                val captureResult = captureViewResult(windowEntry)
+                if (captureResult == null) {
+                    if (i == 0) {
+                        return@withContext null
+                    }
+                    beforeMasks[i] = null
+                }
+
+                captured++
+                captureResults[i] = captureResult
+            }
+            if (captured == 0) {
+                return@withContext null
+            }
+
+            // Synchronize with UI rendering frame
+            suspendCancellableCoroutine { continuation ->
+                Choreographer.getInstance().postFrameCallback {
+                    if (continuation.isActive) {
+                        continuation.resume(Unit)
                     }
-                } else if (windowEntry === baseWindowEntry) {
-                    afterBase = true
                 }
             }
 
+            val afterMasks = collectMasksFromResults(captureResults)
+
+            // off the main thread to avoid blocking the UI thread
             return@withContext withContext(DispatcherProviderHolder.current.default) {
-                // off the main thread to avoid blocking the UI thread
-                if (pairs.isNotEmpty() || baseResult.masks.isNotEmpty()) {
+                val baseResult = captureResults[0] ?: return@withContext null
+                beforeMasks = maskApplier.filteredBeforeMasksMap(beforeMasks, afterMasks) ?: return@withContext null
+
+                // if need to draw something on base bitmap additionally
+                if (captureResults.size > 1 || afterMasks.isNotEmpty()) {
                     val canvas = Canvas(baseResult.bitmap)
-                    drawMasks(canvas, baseResult.masks)
+                    maskApplier.drawMasks(canvas, beforeMasks[0], afterMasks[0])
 
-                    for (res in pairs) {
+                    for (i in 1 until  captureResults.size) {
+                        val res = captureResults[i] ?: continue
                         val entry = res.windowEntry
                         val dx = (entry.screenLeft - baseWindowEntry.screenLeft).toFloat()
                         val dy = (entry.screenTop - baseWindowEntry.screenTop).toFloat()
 
                         canvas.withTranslation(dx, dy) {
                             drawBitmap(res.bitmap, 0f, 0f, null)
-                            drawMasks(canvas, res.masks)
+                            maskApplier.drawMasks(canvas, beforeMasks[i], afterMasks[i])
                         }
                         res.bitmap.recycle()
                     }
@@ -154,25 +176,38 @@ class CaptureSource(
             }
         }
 
-    private fun pickBaseWindow(windowsEntries: List<WindowEntry>): WindowEntry? {
-        windowsEntries.firstOrNull {
+    private fun collectMasks(capturingWindowEntries: List<WindowEntry>): MutableList<List<Mask>?> {
+        return capturingWindowEntries.map {
+            maskCollector.collectMasks( it.rootView, maskMatchers)
+        }.toMutableList()
+    }
+
+    private fun collectMasksFromResults(captureResults: List<CaptureResult?>): MutableList<List<Mask>?> {
+        return captureResults.map { result ->
+            result?.windowEntry?.rootView?.let { rv -> maskCollector.collectMasks(rv, maskMatchers) }
+        }.toMutableList()
+    }
+
+    private fun pickBaseWindow(windowsEntries: List<WindowEntry>): Int? {
+        val appIdx = windowsEntries.indexOfFirst {
             val wmType = it.layoutParams?.type ?: 0
-            (wmType == TYPE_APPLICATION || wmType == TYPE_BASE_APPLICATION)
-        }?.let { return it }
+            wmType == TYPE_APPLICATION || wmType == TYPE_BASE_APPLICATION
+        }
+        if (appIdx >= 0) return appIdx
 
-        windowsEntries.firstOrNull { it.type == WindowType.ACTIVITY }?.let { return it }
+        val activityIdx = windowsEntries.indexOfFirst { it.type == WindowType.ACTIVITY }
+        if (activityIdx >= 0) return activityIdx
 
-        windowsEntries.firstOrNull { it.type == WindowType.DIALOG }?.let { return it }
+        val dialogIdx = windowsEntries.indexOfFirst { it.type == WindowType.DIALOG }
+        if (dialogIdx >= 0) return dialogIdx
 
         // Fallback to the first available
-        return windowsEntries.firstOrNull()
+        return if (windowsEntries.isNotEmpty()) 0 else null
     }
 
     private suspend fun captureViewResult(windowEntry: WindowEntry): CaptureResult? {
         val bitmap = captureViewBitmap(windowEntry) ?: return null
-        val masks = maskCollector.collectMasks(windowEntry.rootView, maskMatchers)
-
-        return CaptureResult(windowEntry, bitmap, masks)
+        return CaptureResult(windowEntry, bitmap)
     }
 
     private suspend fun captureViewBitmap(windowEntry: WindowEntry): Bitmap? {
@@ -194,6 +229,7 @@ class CaptureSource(
             return@withContext canvasDraw(view)
         }
     }
+
     @RequiresApi(Build.VERSION_CODES.O)
     private suspend fun pixelCopy(
         window: Window,
@@ -292,39 +328,4 @@ class CaptureSource(
             }
         }
     }
-
-    /**
-     * Applies masking rectangles to the provided [canvas] using the provided [masks].
-     *
-     * @param canvas The canvas to mask
-     * @param masks areas that will be masked
-     */
-    private fun drawMasks(canvas: Canvas, masks: List<Mask>) {
-        val path = Path()
-        masks.forEach { mask ->
-            drawMask(mask, path, canvas, maskPaint)
-        }
-    }
-
-    private val maskIntRect = Rect()
-    private fun drawMask(mask: Mask, path: Path, canvas: Canvas, paint: Paint) {
-        if (mask.points != null) {
-            val pts = mask.points
-
-            path.reset()
-            path.moveTo(pts[0], pts[1])
-            path.lineTo(pts[2], pts[3])
-            path.lineTo(pts[4], pts[5])
-            path.lineTo(pts[6], pts[7])
-            path.close()
-
-            canvas.drawPath(path, paint)
-        } else {
-            maskIntRect.left = mask.rect.left.toInt()
-            maskIntRect.top = mask.rect.top.toInt()
-            maskIntRect.right = mask.rect.right.toInt()
-            maskIntRect.bottom =  mask.rect.bottom.toInt()
-            canvas.drawRect(maskIntRect, paint)
-        }
-    }
 }

sdk/@launchdarkly/observability-android/lib/src/main/kotlin/com/launchdarkly/observability/replay/masking/MaskApplier.kt

@@ -0,0 +1,102 @@
+package com.launchdarkly.observability.replay.masking
+
+import android.graphics.Canvas
+import android.graphics.Color
+import android.graphics.Paint
+import android.graphics.Path
+import android.graphics.Rect
+import kotlin.math.abs
+
+class MaskApplier {
+    private val beforeMaskPaint = Paint().apply {
+        color = Color.DKGRAY
+        style = Paint.Style.FILL
+    }
+    private val afterMaskPaint = Paint().apply {
+        color = Color.GRAY
+        style = Paint.Style.FILL
+    }
+
+    private val maskIntRect = Rect()
+    private val path = Path()
+
+    fun drawMasks(canvas: Canvas, beforeMasks: List<Mask>?, afterMasks: List<Mask>?) {
+        if (afterMasks == null && beforeMasks == null) return
+
+        beforeMasks?.forEach { mask ->
+            drawMask(mask, path, canvas, beforeMaskPaint)
+        }
+        afterMasks?.forEach { mask ->
+            drawMask(mask, path, canvas, afterMaskPaint)
+        }
+    }
+
+    private fun drawMask(mask: Mask, path: Path, canvas: Canvas, paint: Paint) {
+        if (mask.points != null) {
+            val pts = mask.points
+
+            path.reset()
+            path.moveTo(pts[0], pts[1])
+            path.lineTo(pts[2], pts[3])
+            path.lineTo(pts[4], pts[5])
+            path.lineTo(pts[6], pts[7])
+            path.close()
+
+            canvas.drawPath(path, paint)
+        } else {
+            maskIntRect.left = mask.rect.left.toInt()
+            maskIntRect.top = mask.rect.top.toInt()
+            maskIntRect.right = mask.rect.right.toInt()
+            maskIntRect.bottom =  mask.rect.bottom.toInt()
+            canvas.drawRect(maskIntRect, paint)
+        }
+    }
+
+    fun filteredBeforeMasksMap(
+        beforeMasksMap: List<List<Mask>?>,
+        afterMasksMap: List<List<Mask>?>
+    ): MutableList<List<Mask>?>? {
+        if (afterMasksMap.count() == 0) {
+            return null
+        }
+
+        val result: MutableList<List<Mask>?> = MutableList(beforeMasksMap.size) { null }
+        for (i in beforeMasksMap.indices) {
+            val before = beforeMasksMap[i] ?: continue
+            val after = afterMasksMap[i] ?: return null
+            val merged = filteredBeforeMasks(before, after) ?: return null
+            result[i] = merged
+        }
+
+        return result
+    }
+
+    // Check if masks are stable and returns null if not
+    private fun filteredBeforeMasks(
+        beforeMasks: List<Mask>,
+        afterMasks: List<Mask>
+    ): List<Mask>? {
+        if (afterMasks.count() != beforeMasks.count()) {
+            return null
+        }
+
+        if (afterMasks.count() == 0) {
+            return listOf()
+        }
+
+        val stabilityTolerance = 40f
+        val resultMasks = mutableListOf<Mask>()
+        for ((before, after) in beforeMasks.zip(afterMasks)) {
+            if (before.viewId != after.viewId) {
+                return null
+            }
+            val diff = abs(after.rect.top - before.rect.top)
+            if (diff > stabilityTolerance) {
+                return null
+            }
+            resultMasks += before
+        }
+
+        return resultMasks
+    }
+}