diff --git a/e2e/android/app/src/main/java/com/example/androidobservability/masking/ComposeUserForm.kt b/e2e/android/app/src/main/java/com/example/androidobservability/masking/ComposeUserForm.kt index 2200444ec..26f1174f8 100644 --- a/e2e/android/app/src/main/java/com/example/androidobservability/masking/ComposeUserForm.kt +++ b/e2e/android/app/src/main/java/com/example/androidobservability/masking/ComposeUserForm.kt @@ -89,8 +89,7 @@ fun UserInfoForm(modifier: Modifier = Modifier) { ) { Text( text = "User Information Form", - style = MaterialTheme.typography.headlineMedium, - modifier = Modifier.ldMask() + style = MaterialTheme.typography.headlineMedium ) // Password Section @@ -112,7 +111,8 @@ fun UserInfoForm(modifier: Modifier = Modifier) { label = { Text("Password") }, visualTransformation = PasswordVisualTransformation(), keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Password), - modifier = Modifier.fillMaxWidth() + modifier = Modifier + .fillMaxWidth() ) } } @@ -160,7 +160,9 @@ fun UserInfoForm(modifier: Modifier = Modifier) { onValueChange = { zipCode = it }, label = { Text("ZIP Code") }, keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Number), - modifier = Modifier.fillMaxWidth() + modifier = Modifier + .fillMaxWidth() + .ldMask() ) } } @@ -208,8 +210,10 @@ fun UserInfoForm(modifier: Modifier = Modifier) { label = { Text("CVV") }, keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Number), visualTransformation = PasswordVisualTransformation(), - modifier = Modifier.weight(1f) - ) + modifier = Modifier + .weight(1f) + .ldMask() + ) } } } diff --git a/e2e/android/app/src/main/java/com/example/androidobservability/masking/XMLMaskingActivity.kt b/e2e/android/app/src/main/java/com/example/androidobservability/masking/XMLMaskingActivity.kt index 3bca916c1..83da7ef5a 100644 --- a/e2e/android/app/src/main/java/com/example/androidobservability/masking/XMLMaskingActivity.kt +++ b/e2e/android/app/src/main/java/com/example/androidobservability/masking/XMLMaskingActivity.kt @@ -11,7 +11,6 @@ import android.widget.Toast import android.graphics.Color import android.graphics.PixelFormat import android.view.Gravity -import android.view.ViewGroup import android.view.WindowManager import android.widget.FrameLayout import android.widget.TextView diff --git a/sdk/@launchdarkly/observability-android/lib/src/main/kotlin/com/launchdarkly/observability/replay/capture/CaptureSource.kt b/sdk/@launchdarkly/observability-android/lib/src/main/kotlin/com/launchdarkly/observability/replay/capture/CaptureSource.kt index dedeb10b0..92b9eee48 100644 --- a/sdk/@launchdarkly/observability-android/lib/src/main/kotlin/com/launchdarkly/observability/replay/capture/CaptureSource.kt +++ b/sdk/@launchdarkly/observability-android/lib/src/main/kotlin/com/launchdarkly/observability/replay/capture/CaptureSource.kt @@ -2,8 +2,6 @@ package com.launchdarkly.observability.replay.capture import android.graphics.Bitmap import android.graphics.Canvas -import android.graphics.Color -import android.graphics.Paint import android.graphics.Rect import android.os.Build import android.os.Handler @@ -16,7 +14,6 @@ import android.view.Window import android.view.WindowManager.LayoutParams.TYPE_APPLICATION import android.view.WindowManager.LayoutParams.TYPE_BASE_APPLICATION import androidx.annotation.RequiresApi -import android.graphics.Path import com.launchdarkly.logging.LDLogger import com.launchdarkly.observability.coroutines.DispatcherProviderHolder import com.launchdarkly.observability.replay.masking.MaskMatcher @@ -33,6 +30,8 @@ import kotlin.coroutines.resume import androidx.core.graphics.withTranslation import com.launchdarkly.observability.replay.masking.Mask import androidx.core.graphics.createBitmap +import com.launchdarkly.observability.replay.masking.MaskApplier +import kotlin.collections.mutableMapOf /** * A source of [CaptureEvent]s taken from the lowest visible window. Captures @@ -49,21 +48,19 @@ class CaptureSource( data class CaptureResult( val windowEntry: WindowEntry, val bitmap: Bitmap, - val masks: List ) private val _captureEventFlow = MutableSharedFlow() val captureFlow: SharedFlow = _captureEventFlow.asSharedFlow() private val windowInspector = WindowInspector(logger) private val maskCollector = MaskCollector(logger) - private val maskPaint = Paint().apply { - color = Color.GRAY - style = Paint.Style.FILL - } + private val maskApplier = MaskApplier() private val tiledSignatureManager = TiledSignatureManager() + @Volatile private var tiledSignature: TiledSignature? = null + /** * Requests a [CaptureEvent] be taken now. */ @@ -96,7 +93,8 @@ class CaptureSource( return@withContext null } - val baseWindowEntry = pickBaseWindow(windowsEntries) ?: return@withContext null + val baseIndex = pickBaseWindow(windowsEntries) ?: return@withContext null + val baseWindowEntry = windowsEntries[baseIndex] val rect = baseWindowEntry.rect() // protect against race condition where decor view has no size @@ -108,35 +106,60 @@ class CaptureSource( // TODO: O11Y-625 - see if holding bitmap is more efficient than base64 encoding immediately after compression // TODO: O11Y-628 - use captureQuality option for scaling and adjust this bitmap accordingly, may need to investigate power of 2 rounding for performance // Create a bitmap with the window dimensions - val baseResult = captureViewResult(baseWindowEntry) ?: return@withContext null - - // capture rest of views on top of base - val pairs = mutableListOf() - var afterBase = false - for (windowEntry in windowsEntries) { - if (afterBase) { - captureViewResult(windowEntry)?.let { result -> - pairs.add(result) + + val capturingWindowEntries = windowsEntries.subList(baseIndex, windowsEntries.size) + + var beforeMasks = collectMasks(capturingWindowEntries) + + val captureResults: MutableList = MutableList(capturingWindowEntries.size) { null } + var captured = 0 + for (i in capturingWindowEntries.indices) { + val windowEntry = capturingWindowEntries[i] + val captureResult = captureViewResult(windowEntry) + if (captureResult == null) { + if (i == 0) { + return@withContext null + } + beforeMasks[i] = null + } + + captured++ + captureResults[i] = captureResult + } + if (captured == 0) { + return@withContext null + } + + // Synchronize with UI rendering frame + suspendCancellableCoroutine { continuation -> + Choreographer.getInstance().postFrameCallback { + if (continuation.isActive) { + continuation.resume(Unit) } - } else if (windowEntry === baseWindowEntry) { - afterBase = true } } + val afterMasks = collectMasksFromResults(captureResults) + + // off the main thread to avoid blocking the UI thread return@withContext withContext(DispatcherProviderHolder.current.default) { - // off the main thread to avoid blocking the UI thread - if (pairs.isNotEmpty() || baseResult.masks.isNotEmpty()) { + val baseResult = captureResults[0] ?: return@withContext null + beforeMasks = maskApplier.filteredBeforeMasksMap(beforeMasks, afterMasks) ?: return@withContext null + + // if need to draw something on base bitmap additionally + if (captureResults.size > 1 || afterMasks.isNotEmpty()) { val canvas = Canvas(baseResult.bitmap) - drawMasks(canvas, baseResult.masks) + maskApplier.drawMasks(canvas, beforeMasks[0], afterMasks[0]) - for (res in pairs) { + for (i in 1 until captureResults.size) { + val res = captureResults[i] ?: continue val entry = res.windowEntry val dx = (entry.screenLeft - baseWindowEntry.screenLeft).toFloat() val dy = (entry.screenTop - baseWindowEntry.screenTop).toFloat() canvas.withTranslation(dx, dy) { drawBitmap(res.bitmap, 0f, 0f, null) - drawMasks(canvas, res.masks) + maskApplier.drawMasks(canvas, beforeMasks[i], afterMasks[i]) } res.bitmap.recycle() } @@ -154,25 +177,38 @@ class CaptureSource( } } - private fun pickBaseWindow(windowsEntries: List): WindowEntry? { - windowsEntries.firstOrNull { + private fun collectMasks(capturingWindowEntries: List): MutableList?> { + return capturingWindowEntries.map { + maskCollector.collectMasks( it.rootView, maskMatchers) + }.toMutableList() + } + + private fun collectMasksFromResults(captureResults: List): MutableList?> { + return captureResults.map { result -> + result?.windowEntry?.rootView?.let { rv -> maskCollector.collectMasks(rv, maskMatchers) } + }.toMutableList() + } + + private fun pickBaseWindow(windowsEntries: List): Int? { + val appIdx = windowsEntries.indexOfFirst { val wmType = it.layoutParams?.type ?: 0 - (wmType == TYPE_APPLICATION || wmType == TYPE_BASE_APPLICATION) - }?.let { return it } + wmType == TYPE_APPLICATION || wmType == TYPE_BASE_APPLICATION + } + if (appIdx >= 0) return appIdx - windowsEntries.firstOrNull { it.type == WindowType.ACTIVITY }?.let { return it } + val activityIdx = windowsEntries.indexOfFirst { it.type == WindowType.ACTIVITY } + if (activityIdx >= 0) return activityIdx - windowsEntries.firstOrNull { it.type == WindowType.DIALOG }?.let { return it } + val dialogIdx = windowsEntries.indexOfFirst { it.type == WindowType.DIALOG } + if (dialogIdx >= 0) return dialogIdx // Fallback to the first available - return windowsEntries.firstOrNull() + return if (windowsEntries.isNotEmpty()) 0 else null } private suspend fun captureViewResult(windowEntry: WindowEntry): CaptureResult? { val bitmap = captureViewBitmap(windowEntry) ?: return null - val masks = maskCollector.collectMasks(windowEntry.rootView, maskMatchers) - - return CaptureResult(windowEntry, bitmap, masks) + return CaptureResult(windowEntry, bitmap) } private suspend fun captureViewBitmap(windowEntry: WindowEntry): Bitmap? { @@ -194,6 +230,7 @@ class CaptureSource( return@withContext canvasDraw(view) } } + @RequiresApi(Build.VERSION_CODES.O) private suspend fun pixelCopy( window: Window, @@ -276,39 +313,4 @@ class CaptureSource( } } } - - /** - * Applies masking rectangles to the provided [canvas] using the provided [masks]. - * - * @param canvas The canvas to mask - * @param masks areas that will be masked - */ - private fun drawMasks(canvas: Canvas, masks: List) { - val path = Path() - masks.forEach { mask -> - drawMask(mask, path, canvas, maskPaint) - } - } - - private val maskIntRect = Rect() - private fun drawMask(mask: Mask, path: Path, canvas: Canvas, paint: Paint) { - if (mask.points != null) { - val pts = mask.points - - path.reset() - path.moveTo(pts[0], pts[1]) - path.lineTo(pts[2], pts[3]) - path.lineTo(pts[4], pts[5]) - path.lineTo(pts[6], pts[7]) - path.close() - - canvas.drawPath(path, paint) - } else { - maskIntRect.left = mask.rect.left.toInt() - maskIntRect.top = mask.rect.top.toInt() - maskIntRect.right = mask.rect.right.toInt() - maskIntRect.bottom = mask.rect.bottom.toInt() - canvas.drawRect(maskIntRect, paint) - } - } } diff --git a/sdk/@launchdarkly/observability-android/lib/src/main/kotlin/com/launchdarkly/observability/replay/masking/ComposeMaskTarget.kt b/sdk/@launchdarkly/observability-android/lib/src/main/kotlin/com/launchdarkly/observability/replay/masking/ComposeMaskTarget.kt index 9d1c8cdc3..2c29738ef 100644 --- a/sdk/@launchdarkly/observability-android/lib/src/main/kotlin/com/launchdarkly/observability/replay/masking/ComposeMaskTarget.kt +++ b/sdk/@launchdarkly/observability-android/lib/src/main/kotlin/com/launchdarkly/observability/replay/masking/ComposeMaskTarget.kt @@ -135,11 +135,6 @@ data class ComposeMaskTarget( t4.x, t4.y ) - for (i in pts.indices step 2) { - pts[i] -= context.rootX - pts[i + 1] -= context.rootY - } - return pts } } diff --git a/sdk/@launchdarkly/observability-android/lib/src/main/kotlin/com/launchdarkly/observability/replay/masking/MaskApplier.kt b/sdk/@launchdarkly/observability-android/lib/src/main/kotlin/com/launchdarkly/observability/replay/masking/MaskApplier.kt new file mode 100644 index 000000000..38a5fa7d6 --- /dev/null +++ b/sdk/@launchdarkly/observability-android/lib/src/main/kotlin/com/launchdarkly/observability/replay/masking/MaskApplier.kt @@ -0,0 +1,102 @@ +package com.launchdarkly.observability.replay.masking + +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Paint +import android.graphics.Path +import android.graphics.Rect +import kotlin.math.abs + +class MaskApplier { + private val beforeMaskPaint = Paint().apply { + color = Color.DKGRAY + style = Paint.Style.FILL + } + private val afterMaskPaint = Paint().apply { + color = Color.GRAY + style = Paint.Style.FILL + } + + private val maskIntRect = Rect() + private val path = Path() + + fun drawMasks(canvas: Canvas, beforeMasks: List?, afterMasks: List?) { + if (afterMasks == null && beforeMasks == null) return + + beforeMasks?.forEach { mask -> + drawMask(mask, path, canvas, beforeMaskPaint) + } + afterMasks?.forEach { mask -> + drawMask(mask, path, canvas, afterMaskPaint) + } + } + + private fun drawMask(mask: Mask, path: Path, canvas: Canvas, paint: Paint) { + if (mask.points != null) { + val pts = mask.points + + path.reset() + path.moveTo(pts[0], pts[1]) + path.lineTo(pts[2], pts[3]) + path.lineTo(pts[4], pts[5]) + path.lineTo(pts[6], pts[7]) + path.close() + + canvas.drawPath(path, paint) + } else { + maskIntRect.left = mask.rect.left.toInt() + maskIntRect.top = mask.rect.top.toInt() + maskIntRect.right = mask.rect.right.toInt() + maskIntRect.bottom = mask.rect.bottom.toInt() + canvas.drawRect(maskIntRect, paint) + } + } + + fun filteredBeforeMasksMap( + beforeMasksMap: List?>, + afterMasksMap: List?> + ): MutableList?>? { + if (afterMasksMap.count() == 0) { + return null + } + + val result: MutableList?> = MutableList(beforeMasksMap.size) { null } + for (i in beforeMasksMap.indices) { + val before = beforeMasksMap[i] ?: continue + val after = afterMasksMap[i] ?: return null + val merged = filteredBeforeMasks(before, after) ?: return null + result[i] = merged + } + + return result + } + + // Check if masks are stable and returns null if not + private fun filteredBeforeMasks( + beforeMasks: List, + afterMasks: List + ): List? { + if (afterMasks.count() != beforeMasks.count()) { + return null + } + + if (afterMasks.count() == 0) { + return listOf() + } + + val stabilityTolerance = 40f + val resultMasks = mutableListOf() + for ((before, after) in beforeMasks.zip(afterMasks)) { + if (before.viewId != after.viewId) { + return null + } + val diff = abs(after.rect.top - before.rect.top) + if (diff > stabilityTolerance) { + return null + } + resultMasks += before + } + + return resultMasks + } +} \ No newline at end of file diff --git a/sdk/@launchdarkly/observability-android/lib/src/main/kotlin/com/launchdarkly/observability/replay/masking/NativeMaskTarget.kt b/sdk/@launchdarkly/observability-android/lib/src/main/kotlin/com/launchdarkly/observability/replay/masking/NativeMaskTarget.kt index 16521b70a..ed0a81464 100644 --- a/sdk/@launchdarkly/observability-android/lib/src/main/kotlin/com/launchdarkly/observability/replay/masking/NativeMaskTarget.kt +++ b/sdk/@launchdarkly/observability-android/lib/src/main/kotlin/com/launchdarkly/observability/replay/masking/NativeMaskTarget.kt @@ -79,7 +79,7 @@ data class NativeMaskTarget( if (width <= 0 || height <= 0) { return null } val matrix = context.matrix - matrix.reset() + matrix.reset() // reusing matrix for performance view.transformMatrixToGlobal(matrix) val pts = floatArrayOf(