Skip to content

Commit 5ff93f6

Browse files
abelonogov-ldabelonogov-ld
authored
feat: Android Incremental Image Diff compression (#390)
## Summary Incremental Image Diff compression, layers = 15, backtracking = true <!-- CURSOR_SUMMARY --> --- > [!NOTE] > **High Risk** > Reworks the session replay capture/export pipeline and RRWeb event generation to send incremental tile diffs with backtracking, which can impact replay correctness, performance, and memory usage. Several trace-related E2E tests are now ignored, reducing coverage while these behavioral changes land. > > **Overview** > Adds **incremental image-diff compression** to Android Session Replay by replacing single full-screen `CaptureEvent` exports with `ExportFrame` batches of *add/remove image tiles* plus keyframe tracking/backtracking. > > Refactors capture into `ImageCaptureService` (raw bitmap capture + masking) and a new `CaptureManager`/`ExportDiffManager`/`TileDiffManager` pipeline that computes per-frame signatures, crops diffs, and manages keyframe layers. Updates export to a new `RRWebEventGenerator` that emits RRWeb DOM mutation events for tile adds/removes (and drops frames that don’t match the known keyframe), adjusts queue cost accounting, and adds extensive unit tests for diff/backtracking behavior; some trace-disabling E2E tests are marked `@Ignore`. > > <sup>Written by [Cursor Bugbot](https://cursor.com/dashboard?tab=bugbot) for commit 5a1d363. This will update automatically on new commits. Configure [here](https://cursor.com/dashboard?tab=bugbot).</sup> <!-- /CURSOR_SUMMARY -->
1 parent e187cf3 commit 5ff93f6

22 files changed

+2134
-543
lines changed

‎sdk/@launchdarkly/observability-android/lib/src/main/kotlin/com/launchdarkly/observability/replay/RRWebTypes.kt‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
package com.launchdarkly.observability.replay
2+
3+
enum class RRWebEventType(val code: Int) {
4+
DOM_CONTENT_LOADED(0),
5+
LOAD(1),
6+
FULL_SNAPSHOT(2),
7+
INCREMENTAL_SNAPSHOT(3),
8+
META(4),
9+
CUSTOM(5),
10+
PLUGIN(6),
11+
}
12+
13+
enum class RRWebNodeType(val code: Int) {
14+
DOCUMENT(0),
15+
DOCUMENT_TYPE(1),
16+
ELEMENT(2),
17+
TEXT(3),
18+
CDATA(4),
19+
COMMENT(5),
20+
}
21+
22+
enum class RRWebIncrementalSource(val code: Int) {
23+
MUTATION(0),
24+
MOUSE_MOVE(1),
25+
MOUSE_INTERACTION(2),
26+
SCROLL(3),
27+
VIEWPORT_RESIZE(4),
28+
INPUT(5),
29+
TOUCH_MOVE(6),
30+
MEDIA_INTERACTION(7),
31+
STYLE_SHEET_RULE(8),
32+
CANVAS_MUTATION(9),
33+
FONT(10),
34+
LOG(11),
35+
DRAG(12),
36+
STYLE_DECLARATION(13),
37+
SELECTION(14),
38+
ADOPTED_STYLE_SHEET(15),
39+
CUSTOM_ELEMENT(16),
40+
}
41+
42+
enum class RRWebMouseInteraction(val code: Int) {
43+
MOUSE_UP(0),
44+
MOUSE_DOWN(1),
45+
CLICK(2),
46+
CONTEXT_MENU(3),
47+
DOUBLE_CLICK(4),
48+
FOCUS(5),
49+
BLUR(6),
50+
TOUCH_START(7),
51+
TOUCH_MOVE_DEPARTED(8),
52+
TOUCH_END(9),
53+
TOUCH_CANCEL(10),
54+
}
55+
56+
enum class RRWebCustomDataTag(val wireValue: String) {
57+
CLICK("Click"),
58+
FOCUS("Focus"),
59+
VIEWPORT("Viewport"),
60+
RELOAD("Reload"),
61+
IDENTIFY("Identify"),
62+
}

‎sdk/@launchdarkly/observability-android/lib/src/main/kotlin/com/launchdarkly/observability/replay/ReplayInstrumentation.kt‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import com.launchdarkly.logging.LDLogger
88
import com.launchdarkly.observability.client.ObservabilityContext
99
import com.launchdarkly.observability.coroutines.DispatcherProviderHolder
1010
import com.launchdarkly.observability.interfaces.LDExtendedInstrumentation
11-
import com.launchdarkly.observability.replay.capture.CaptureSource
11+
import com.launchdarkly.observability.replay.capture.CaptureManager
1212
import com.launchdarkly.observability.replay.exporter.IdentifyItemPayload
1313
import com.launchdarkly.observability.replay.exporter.ImageItemPayload
1414
import com.launchdarkly.observability.replay.exporter.InteractionItemPayload
@@ -70,7 +70,7 @@ class ReplayInstrumentation(
7070
private val logger: LDLogger = observabilityContext.logger
7171
private val eventQueue = EventQueue()
7272
private val batchWorker = BatchWorker(eventQueue, logger)
73-
private var captureSource: CaptureSource? = null
73+
private var captureManager: CaptureManager? = null
7474
private var interactionSource: InteractionSource? = null
7575
private val instrumentationScope = CoroutineScope(DispatcherProviderHolder.current.default + SupervisorJob())
7676
private var captureJob: Job? = null
@@ -90,7 +90,7 @@ class ReplayInstrumentation(
9090
if (isInstalled) return
9191

9292
sessionManager = ctx.sessionManager
93-
captureSource = CaptureSource(
93+
captureManager = CaptureManager(
9494
sessionManager = ctx.sessionManager,
9595
options = options,
9696
logger = observabilityContext.logger
@@ -126,7 +126,7 @@ class ReplayInstrumentation(
126126
private fun startCollectors() {
127127
// Images collector
128128
instrumentationScope.launch {
129-
captureSource?.captureFlow?.collect { capture ->
129+
captureManager?.captureFlow?.collect { capture ->
130130
if (!isEnabled.value) return@collect
131131
eventQueue.send(ImageItemPayload(capture))
132132
}
@@ -162,7 +162,7 @@ class ReplayInstrumentation(
162162
logger.debug("Session replay capture running")
163163
while (isActive) {
164164
try {
165-
captureSource?.captureNow()
165+
captureManager?.captureNow()
166166
} catch (e: CancellationException) {
167167
throw e
168168
} catch (e: OutOfMemoryError) {

‎sdk/@launchdarkly/observability-android/lib/src/main/kotlin/com/launchdarkly/observability/replay/ReplayOptions.kt‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,23 @@ package com.launchdarkly.observability.replay
88
* @property capturePeriodMillis period between captures
99
* @property scale optional replay scale override. When null, no additional scaling is applied. Usually from 1-4. 1 = 160DPI
1010
* @property enabled controls whether session replay starts capturing immediately on initialization
11+
* @property compression compression strategy for frame export
1112
*/
1213
data class ReplayOptions(
1314
val enabled: Boolean = true,
1415
val debug: Boolean = false,
1516
val privacyProfile: PrivacyProfile = PrivacyProfile(),
1617
val capturePeriodMillis: Long = 1000, // defaults to ever 1 second
1718
/** Optional replay scale. Null disables scaling override. */
18-
val scale: Float? = 1.0f
19+
val scale: Float? = 1.0f,
20+
val compression: CompressionMethod = CompressionMethod.OverlayTiles()
1921
// TODO O11Y-623 - Add storage options
20-
)
22+
) {
23+
sealed class CompressionMethod {
24+
data object ScreenImage : CompressionMethod()
25+
data class OverlayTiles(
26+
val layers: Int = 15,
27+
val backtracking: Boolean = true,
28+
) : CompressionMethod()
29+
}
30+
}

‎sdk/@launchdarkly/observability-android/lib/src/main/kotlin/com/launchdarkly/observability/replay/capture/CaptureEvent.kt‎

Lines changed: 0 additions & 19 deletions
This file was deleted.

‎sdk/@launchdarkly/observability-android/lib/src/main/kotlin/com/launchdarkly/observability/replay/capture/CaptureManager.kt‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package com.launchdarkly.observability.replay.capture
2+
3+
import com.launchdarkly.logging.LDLogger
4+
import com.launchdarkly.observability.replay.ReplayOptions
5+
import io.opentelemetry.android.session.SessionManager
6+
import kotlinx.coroutines.flow.MutableSharedFlow
7+
import kotlinx.coroutines.flow.SharedFlow
8+
import kotlinx.coroutines.flow.asSharedFlow
9+
10+
/**
11+
* A source of [ExportFrame]s taken from the lowest visible window. Captures
12+
* are emitted on the [captureFlow] property of this class.
13+
*
14+
* @param sessionManager Used to get current session for tagging [ExportFrame] with session id
15+
*/
16+
class CaptureManager(
17+
private val sessionManager: SessionManager,
18+
private val options: ReplayOptions,
19+
private val logger: LDLogger,
20+
// TODO: O11Y-628 - add captureQuality options
21+
) {
22+
private val _captureEventFlow = MutableSharedFlow<ExportFrame>()
23+
val captureFlow: SharedFlow<ExportFrame> = _captureEventFlow.asSharedFlow()
24+
private val imageCaptureService = ImageCaptureService(options, logger)
25+
private val exportDiffManager = ExportDiffManager(
26+
compression = options.compression,
27+
scale = options.scale ?: 1f,
28+
)
29+
30+
/**
31+
* Requests a [ExportFrame] be taken now.
32+
*/
33+
suspend fun captureNow() {
34+
val rawFrame = imageCaptureService.captureRawFrame() ?: return
35+
36+
val session = sessionManager.getSessionId()
37+
val exportFrame = exportDiffManager.createCaptureEvent(rawFrame, session) ?: return
38+
_captureEventFlow.emit(exportFrame)
39+
}
40+
}

‎sdk/@launchdarkly/observability-android/lib/src/main/kotlin/com/launchdarkly/observability/replay/capture/ExportDiffManager.kt‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
package com.launchdarkly.observability.replay.capture
2+
3+
import android.graphics.Bitmap
4+
import android.util.Base64
5+
import com.launchdarkly.observability.replay.ReplayOptions
6+
import java.io.ByteArrayOutputStream
7+
8+
class ExportDiffManager(
9+
private val compression: ReplayOptions.CompressionMethod,
10+
private val scale: Float = 1f,
11+
private val tileDiffManager: TileDiffManager = TileDiffManager(compression = compression, scale = scale),
12+
) {
13+
private val currentImages = mutableListOf<ExportFrame.RemoveImage>()
14+
private val currentImagesIndex = mutableMapOf<ImageSignature, Int>()
15+
private val lock = Any()
16+
private val format = ExportFrame.ExportFormat.Webp(quality = 30)
17+
private var keyFrameId = 0
18+
19+
fun createCaptureEvent(rawFrame: ImageCaptureService.RawFrame, session: String): ExportFrame? {
20+
synchronized(lock) {
21+
val tiledFrame = tileDiffManager.computeTiledFrame(rawFrame) ?: return null
22+
return createCaptureEventInternal(tiledFrame, session)
23+
}
24+
}
25+
26+
private fun createCaptureEventInternal(tiledFrame: TiledFrame, session: String): ExportFrame? {
27+
try {
28+
val adds = mutableListOf<ExportFrame.AddImage>()
29+
var removes = mutableListOf<ExportFrame.RemoveImage>()
30+
31+
if (tiledFrame.isKeyframe) {
32+
removes = currentImages.toMutableList()
33+
currentImages.clear()
34+
currentImagesIndex.clear()
35+
keyFrameId += 1
36+
}
37+
38+
val signature = tiledFrame.imageSignature
39+
val useBacktracking =
40+
compression is ReplayOptions.CompressionMethod.OverlayTiles &&
41+
compression.backtracking
42+
43+
if (signature != null && useBacktracking) {
44+
val lastKeyNodeIdx = currentImagesIndex[signature]
45+
if (lastKeyNodeIdx != null && lastKeyNodeIdx < currentImages.size) {
46+
removes = currentImages.subList(lastKeyNodeIdx + 1, currentImages.size).toMutableList()
47+
currentImages.subList(lastKeyNodeIdx + 1, currentImages.size).clear()
48+
49+
val filtered = currentImagesIndex.filterValues { value -> value <= lastKeyNodeIdx }
50+
currentImagesIndex.clear()
51+
currentImagesIndex.putAll(filtered)
52+
} else {
53+
for (tile in tiledFrame.tiles) {
54+
val addImage =
55+
tile.bitmap.asExportedImage(format = format, rect = tile.rect, imageSignature = signature)
56+
?: return null
57+
adds.add(addImage)
58+
currentImages.add(
59+
ExportFrame.RemoveImage(
60+
keyFrameId = keyFrameId,
61+
imageSignature = signature,
62+
)
63+
)
64+
}
65+
currentImagesIndex[signature] = currentImages.size - 1
66+
}
67+
} else {
68+
for (tile in tiledFrame.tiles) {
69+
val imageSignature = tiledFrame.imageSignature
70+
val addImage = tile.bitmap.asExportedImage(format = format, rect = tile.rect, imageSignature = imageSignature)
71+
?: return null
72+
adds.add(addImage)
73+
if (imageSignature != null) {
74+
currentImages.add(
75+
ExportFrame.RemoveImage(
76+
keyFrameId = keyFrameId,
77+
imageSignature = imageSignature,
78+
)
79+
)
80+
}
81+
}
82+
if (signature != null) {
83+
currentImagesIndex[signature] = currentImages.size - 1
84+
}
85+
}
86+
87+
if (adds.isEmpty() && removes.isEmpty()) {
88+
return null
89+
}
90+
91+
return ExportFrame(
92+
keyFrameId = keyFrameId,
93+
addImages = adds,
94+
removeImages = removes,
95+
originalSize = tiledFrame.originalSize,
96+
scale = tiledFrame.scale,
97+
format = format,
98+
timestamp = tiledFrame.timestamp,
99+
orientation = tiledFrame.orientation,
100+
isKeyframe = tiledFrame.isKeyframe,
101+
imageSignature = tiledFrame.imageSignature,
102+
session = session,
103+
)
104+
} finally {
105+
tiledFrame.recycleBitmaps()
106+
}
107+
}
108+
}
109+
110+
private fun Bitmap.asExportedImage(
111+
format: ExportFrame.ExportFormat,
112+
rect: IntRect,
113+
imageSignature: ImageSignature?,
114+
): ExportFrame.AddImage? {
115+
val outputStream = ByteArrayOutputStream()
116+
return try {
117+
val compressionOk = when (format) {
118+
ExportFrame.ExportFormat.Png -> compress(Bitmap.CompressFormat.PNG, 100, outputStream)
119+
is ExportFrame.ExportFormat.Jpeg -> compress(
120+
Bitmap.CompressFormat.JPEG,
121+
(format.quality * 100).toInt().coerceIn(0, 100),
122+
outputStream
123+
)
124+
is ExportFrame.ExportFormat.Webp -> compress(
125+
Bitmap.CompressFormat.WEBP,
126+
format.quality.coerceIn(0, 100),
127+
outputStream
128+
)
129+
}
130+
if (!compressionOk) {
131+
return null
132+
}
133+
val compressedImage = Base64.encodeToString(outputStream.toByteArray(), Base64.NO_WRAP)
134+
ExportFrame.AddImage(
135+
imageBase64 = compressedImage,
136+
rect = rect,
137+
imageSignature = imageSignature,
138+
)
139+
} finally {
140+
outputStream.close()
141+
}
142+
}

0 commit comments

Comments
 (0)