Skip to content

Commit 0af0984

Browse files
romtsnromtsn
authored
[SR] Capture gestures/motion events
2 parents d4ac484 + d93e609 commit 0af0984

20 files changed

+1515
-18
lines changed

‎sentry-android-replay/api/sentry-android-replay.api‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ public final class io/sentry/android/replay/ReplayCache : java/io/Closeable {
3737
public final fun rotate (J)V
3838
}
3939

40-
public final class io/sentry/android/replay/ReplayIntegration : android/content/ComponentCallbacks, io/sentry/Integration, io/sentry/ReplayController, io/sentry/android/replay/ScreenshotRecorderCallback, java/io/Closeable {
40+
public final class io/sentry/android/replay/ReplayIntegration : android/content/ComponentCallbacks, io/sentry/Integration, io/sentry/ReplayController, io/sentry/android/replay/ScreenshotRecorderCallback, io/sentry/android/replay/TouchRecorderCallback, java/io/Closeable {
4141
public fun <init> (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;)V
4242
public fun <init> (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;Lkotlin/jvm/functions/Function0;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function1;)V
4343
public synthetic fun <init> (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;Lkotlin/jvm/functions/Function0;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function1;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
@@ -49,6 +49,7 @@ public final class io/sentry/android/replay/ReplayIntegration : android/content/
4949
public fun onLowMemory ()V
5050
public fun onScreenshotRecorded (Landroid/graphics/Bitmap;)V
5151
public fun onScreenshotRecorded (Ljava/io/File;J)V
52+
public fun onTouchEvent (Landroid/view/MotionEvent;)V
5253
public fun pause ()V
5354
public fun register (Lio/sentry/IHub;Lio/sentry/SentryOptions;)V
5455
public fun resume ()V
@@ -89,6 +90,40 @@ public final class io/sentry/android/replay/ScreenshotRecorderConfig$Companion {
8990
public final fun from (Landroid/content/Context;Lio/sentry/SentryReplayOptions;)Lio/sentry/android/replay/ScreenshotRecorderConfig;
9091
}
9192

93+
public abstract interface class io/sentry/android/replay/TouchRecorderCallback {
94+
public abstract fun onTouchEvent (Landroid/view/MotionEvent;)V
95+
}
96+
97+
public class io/sentry/android/replay/util/FixedWindowCallback : android/view/Window$Callback {
98+
public final field delegate Landroid/view/Window$Callback;
99+
public fun <init> (Landroid/view/Window$Callback;)V
100+
public fun dispatchGenericMotionEvent (Landroid/view/MotionEvent;)Z
101+
public fun dispatchKeyEvent (Landroid/view/KeyEvent;)Z
102+
public fun dispatchKeyShortcutEvent (Landroid/view/KeyEvent;)Z
103+
public fun dispatchPopulateAccessibilityEvent (Landroid/view/accessibility/AccessibilityEvent;)Z
104+
public fun dispatchTouchEvent (Landroid/view/MotionEvent;)Z
105+
public fun dispatchTrackballEvent (Landroid/view/MotionEvent;)Z
106+
public fun onActionModeFinished (Landroid/view/ActionMode;)V
107+
public fun onActionModeStarted (Landroid/view/ActionMode;)V
108+
public fun onAttachedToWindow ()V
109+
public fun onContentChanged ()V
110+
public fun onCreatePanelMenu (ILandroid/view/Menu;)Z
111+
public fun onCreatePanelView (I)Landroid/view/View;
112+
public fun onDetachedFromWindow ()V
113+
public fun onMenuItemSelected (ILandroid/view/MenuItem;)Z
114+
public fun onMenuOpened (ILandroid/view/Menu;)Z
115+
public fun onPanelClosed (ILandroid/view/Menu;)V
116+
public fun onPointerCaptureChanged (Z)V
117+
public fun onPreparePanel (ILandroid/view/View;Landroid/view/Menu;)Z
118+
public fun onProvideKeyboardShortcuts (Ljava/util/List;Landroid/view/Menu;I)V
119+
public fun onSearchRequested ()Z
120+
public fun onSearchRequested (Landroid/view/SearchEvent;)Z
121+
public fun onWindowAttributesChanged (Landroid/view/WindowManager$LayoutParams;)V
122+
public fun onWindowFocusChanged (Z)V
123+
public fun onWindowStartingActionMode (Landroid/view/ActionMode$Callback;)Landroid/view/ActionMode;
124+
public fun onWindowStartingActionMode (Landroid/view/ActionMode$Callback;I)Landroid/view/ActionMode;
125+
}
126+
92127
public abstract interface class io/sentry/android/replay/video/SimpleFrameMuxer {
93128
public abstract fun getVideoTime ()J
94129
public abstract fun isStarted ()Z

‎sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import android.content.Context
55
import android.content.res.Configuration
66
import android.graphics.Bitmap
77
import android.os.Build
8+
import android.view.MotionEvent
89
import io.sentry.Hint
910
import io.sentry.IHub
1011
import io.sentry.Integration
@@ -32,7 +33,7 @@ public class ReplayIntegration(
3233
private val recorderProvider: (() -> Recorder)? = null,
3334
private val recorderConfigProvider: ((configChanged: Boolean) -> ScreenshotRecorderConfig)? = null,
3435
private val replayCacheProvider: ((replayId: SentryId) -> ReplayCache)? = null
35-
) : Integration, Closeable, ScreenshotRecorderCallback, ReplayController, ComponentCallbacks {
36+
) : Integration, Closeable, ScreenshotRecorderCallback, TouchRecorderCallback, ReplayController, ComponentCallbacks {
3637

3738
// needed for the Java's call site
3839
constructor(context: Context, dateProvider: ICurrentDateProvider) : this(
@@ -72,7 +73,7 @@ public class ReplayIntegration(
7273
}
7374

7475
this.hub = hub
75-
recorder = recorderProvider?.invoke() ?: WindowRecorder(options, this)
76+
recorder = recorderProvider?.invoke() ?: WindowRecorder(options, this, this)
7677
isEnabled.set(true)
7778

7879
try {
@@ -220,4 +221,8 @@ public class ReplayIntegration(
220221
}
221222

222223
override fun onLowMemory() = Unit
224+
225+
override fun onTouchEvent(event: MotionEvent) {
226+
captureStrategy?.onTouchEvent(event)
227+
}
223228
}

‎sentry-android-replay/src/main/java/io/sentry/android/replay/WindowRecorder.kt‎

Lines changed: 68 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,13 @@
11
package io.sentry.android.replay
22

33
import android.annotation.TargetApi
4+
import android.view.MotionEvent
45
import android.view.View
6+
import android.view.Window
7+
import io.sentry.SentryLevel.DEBUG
8+
import io.sentry.SentryLevel.ERROR
59
import io.sentry.SentryOptions
10+
import io.sentry.android.replay.util.FixedWindowCallback
611
import io.sentry.android.replay.util.gracefullyShutdown
712
import io.sentry.android.replay.util.scheduleAtFixedRateSafely
813
import java.lang.ref.WeakReference
@@ -16,7 +21,8 @@ import kotlin.LazyThreadSafetyMode.NONE
1621
@TargetApi(26)
1722
internal class WindowRecorder(
1823
private val options: SentryOptions,
19-
private val screenshotRecorderCallback: ScreenshotRecorderCallback? = null
24+
private val screenshotRecorderCallback: ScreenshotRecorderCallback? = null,
25+
private val touchRecorderCallback: TouchRecorderCallback? = null
2026
) : Recorder {
2127

2228
internal companion object {
@@ -39,7 +45,11 @@ internal class WindowRecorder(
3945
if (added) {
4046
rootViews.add(WeakReference(root))
4147
recorder?.bind(root)
48+
49+
root.startGestureTracking()
4250
} else {
51+
root.stopGestureTracking()
52+
4353
recorder?.unbind(root)
4454
rootViews.removeAll { it.get() == root }
4555

@@ -86,6 +96,60 @@ internal class WindowRecorder(
8696
isRecording.set(false)
8797
}
8898

99+
override fun close() {
100+
stop()
101+
capturer.gracefullyShutdown(options)
102+
}
103+
104+
private fun View.startGestureTracking() {
105+
val window = phoneWindow
106+
if (window == null) {
107+
options.logger.log(DEBUG, "Window is invalid, not tracking gestures")
108+
return
109+
}
110+
111+
if (touchRecorderCallback == null) {
112+
options.logger.log(DEBUG, "TouchRecorderCallback is null, not tracking gestures")
113+
return
114+
}
115+
116+
val delegate = window.callback
117+
window.callback = SentryReplayGestureRecorder(options, touchRecorderCallback, delegate)
118+
}
119+
120+
private fun View.stopGestureTracking() {
121+
val window = phoneWindow
122+
if (window == null) {
123+
options.logger.log(DEBUG, "Window was null in stopGestureTracking")
124+
return
125+
}
126+
127+
if (window.callback is SentryReplayGestureRecorder) {
128+
val delegate = (window.callback as SentryReplayGestureRecorder).delegate
129+
window.callback = delegate
130+
}
131+
}
132+
133+
private class SentryReplayGestureRecorder(
134+
private val options: SentryOptions,
135+
private val touchRecorderCallback: TouchRecorderCallback?,
136+
delegate: Window.Callback?
137+
) : FixedWindowCallback(delegate) {
138+
override fun dispatchTouchEvent(event: MotionEvent?): Boolean {
139+
if (event != null) {
140+
val copy: MotionEvent = MotionEvent.obtainNoHistory(event)
141+
try {
142+
touchRecorderCallback?.onTouchEvent(copy)
143+
} catch (e: Throwable) {
144+
options.logger.log(ERROR, "Error dispatching touch event", e)
145+
} finally {
146+
copy.recycle()
147+
}
148+
}
149+
return super.dispatchTouchEvent(event)
150+
}
151+
}
152+
89153
private class RecorderExecutorServiceThreadFactory : ThreadFactory {
90154
private var cnt = 0
91155
override fun newThread(r: Runnable): Thread {
@@ -94,9 +158,8 @@ internal class WindowRecorder(
94158
return ret
95159
}
96160
}
161+
}
97162

98-
override fun close() {
99-
stop()
100-
capturer.gracefullyShutdown(options)
101-
}
163+
public interface TouchRecorderCallback {
164+
fun onTouchEvent(event: MotionEvent)
102165
}

‎sentry-android-replay/src/main/java/io/sentry/android/replay/capture/BaseCaptureStrategy.kt‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package io.sentry.android.replay.capture
22

3+
import android.view.MotionEvent
34
import io.sentry.Breadcrumb
45
import io.sentry.DateUtils
56
import io.sentry.Hint
@@ -17,13 +18,19 @@ import io.sentry.android.replay.util.submitSafely
1718
import io.sentry.protocol.SentryId
1819
import io.sentry.rrweb.RRWebBreadcrumbEvent
1920
import io.sentry.rrweb.RRWebEvent
21+
import io.sentry.rrweb.RRWebIncrementalSnapshotEvent
22+
import io.sentry.rrweb.RRWebInteractionEvent
23+
import io.sentry.rrweb.RRWebInteractionEvent.InteractionType
24+
import io.sentry.rrweb.RRWebInteractionMoveEvent
25+
import io.sentry.rrweb.RRWebInteractionMoveEvent.Position
2026
import io.sentry.rrweb.RRWebMetaEvent
2127
import io.sentry.rrweb.RRWebSpanEvent
2228
import io.sentry.rrweb.RRWebVideoEvent
2329
import io.sentry.transport.ICurrentDateProvider
2430
import io.sentry.util.FileUtils
2531
import java.io.File
2632
import java.util.Date
33+
import java.util.LinkedList
2734
import java.util.concurrent.Executors
2835
import java.util.concurrent.ScheduledExecutorService
2936
import java.util.concurrent.ThreadFactory
@@ -51,6 +58,10 @@ internal abstract class BaseCaptureStrategy(
5158
"http.response_content_length",
5259
"http.request_content_length"
5360
)
61+
62+
// rrweb values
63+
private const val TOUCH_MOVE_DEBOUNCE_THRESHOLD = 50
64+
private const val CAPTURE_MOVE_EVENT_THRESHOLD = 500
5465
}
5566

5667
protected var cache: ReplayCache? = null
@@ -60,6 +71,12 @@ internal abstract class BaseCaptureStrategy(
6071
override val currentSegment = AtomicInteger(0)
6172
override val replayCacheDir: File? get() = cache?.replayCacheDir
6273

74+
protected val currentEvents = LinkedList<RRWebEvent>()
75+
private val currentEventsLock = Any()
76+
private val currentPositions = mutableListOf<Position>()
77+
private var touchMoveBaseline = 0L
78+
private var lastCapturedMoveEvent = 0L
79+
6380
protected val replayExecutor: ScheduledExecutorService by lazy {
6481
executor ?: Executors.newSingleThreadScheduledExecutor(ReplayExecutorServiceThreadFactory())
6582
}
@@ -249,6 +266,12 @@ internal abstract class BaseCaptureStrategy(
249266
}
250267
}
251268

269+
rotateCurrentEvents(endTimestamp.time) { event ->
270+
if (event.timestamp >= segmentTimestamp.time) {
271+
recordingPayload += event
272+
}
273+
}
274+
252275
val recording = ReplayRecording().apply {
253276
this.segmentId = segmentId
254277
payload = recordingPayload.sortedBy { it.timestamp }
@@ -265,10 +288,33 @@ internal abstract class BaseCaptureStrategy(
265288
this.recorderConfig = recorderConfig
266289
}
267290

291+
override fun onTouchEvent(event: MotionEvent) {
292+
val rrwebEvent = event.toRRWebIncrementalSnapshotEvent()
293+
if (rrwebEvent != null) {
294+
synchronized(currentEventsLock) {
295+
currentEvents += rrwebEvent
296+
}
297+
}
298+
}
299+
268300
override fun close() {
269301
replayExecutor.gracefullyShutdown(options)
270302
}
271303

304+
protected fun rotateCurrentEvents(
305+
until: Long,
306+
callback: ((RRWebEvent) -> Unit)? = null
307+
) {
308+
synchronized(currentEventsLock) {
309+
var event = currentEvents.peek()
310+
while (event != null && event.timestamp <= until) {
311+
callback?.invoke(event)
312+
currentEvents.remove()
313+
event = currentEvents.peek()
314+
}
315+
}
316+
}
317+
272318
private class ReplayExecutorServiceThreadFactory : ThreadFactory {
273319
private var cnt = 0
274320
override fun newThread(r: Runnable): Thread {
@@ -335,4 +381,63 @@ internal abstract class BaseCaptureStrategy(
335381
data = breadcrumbData
336382
}
337383
}
384+
385+
private fun MotionEvent.toRRWebIncrementalSnapshotEvent(): RRWebIncrementalSnapshotEvent? {
386+
val event = this
387+
return when (val action = event.actionMasked) {
388+
MotionEvent.ACTION_MOVE -> {
389+
// we only throttle move events as those can be overwhelming
390+
val now = dateProvider.currentTimeMillis
391+
if (lastCapturedMoveEvent != 0L && lastCapturedMoveEvent + TOUCH_MOVE_DEBOUNCE_THRESHOLD > now) {
392+
return null
393+
}
394+
lastCapturedMoveEvent = now
395+
396+
// idk why but rrweb does it like dis
397+
if (touchMoveBaseline == 0L) {
398+
touchMoveBaseline = now
399+
}
400+
401+
currentPositions += Position().apply {
402+
x = event.x * recorderConfig.scaleFactorX
403+
y = event.y * recorderConfig.scaleFactorY
404+
id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE
405+
timeOffset = now - touchMoveBaseline
406+
}
407+
408+
val totalOffset = now - touchMoveBaseline
409+
return if (totalOffset > CAPTURE_MOVE_EVENT_THRESHOLD) {
410+
RRWebInteractionMoveEvent().apply {
411+
timestamp = now
412+
positions = currentPositions.map { pos ->
413+
pos.timeOffset -= totalOffset
414+
pos
415+
}
416+
}.also {
417+
currentPositions.clear()
418+
touchMoveBaseline = 0L
419+
}
420+
} else {
421+
null
422+
}
423+
}
424+
425+
MotionEvent.ACTION_DOWN, MotionEvent.ACTION_UP, MotionEvent.ACTION_CANCEL -> {
426+
RRWebInteractionEvent().apply {
427+
timestamp = dateProvider.currentTimeMillis
428+
x = event.x * recorderConfig.scaleFactorX
429+
y = event.y * recorderConfig.scaleFactorY
430+
id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE
431+
interactionType = when (action) {
432+
MotionEvent.ACTION_UP -> InteractionType.TouchEnd
433+
MotionEvent.ACTION_DOWN -> InteractionType.TouchStart
434+
MotionEvent.ACTION_CANCEL -> InteractionType.TouchCancel
435+
else -> InteractionType.TouchMove_Departed // should not happen
436+
}
437+
}
438+
}
439+
440+
else -> null
441+
}
442+
}
338443
}

‎sentry-android-replay/src/main/java/io/sentry/android/replay/capture/BufferCaptureStrategy.kt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package io.sentry.android.replay.capture
22

3+
import android.view.MotionEvent
34
import io.sentry.DateUtils
45
import io.sentry.Hint
56
import io.sentry.IHub
@@ -171,4 +172,10 @@ internal class BufferCaptureStrategy(
171172
captureStrategy.start(segmentId = currentSegment.get(), replayId = currentReplayId.get(), cleanupOldReplays = false)
172173
return captureStrategy
173174
}
175+
176+
override fun onTouchEvent(event: MotionEvent) {
177+
super.onTouchEvent(event)
178+
val bufferLimit = dateProvider.currentTimeMillis - options.experimental.sessionReplay.errorReplayDuration
179+
rotateCurrentEvents(bufferLimit)
180+
}
174181
}

‎sentry-android-replay/src/main/java/io/sentry/android/replay/capture/CaptureStrategy.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package io.sentry.android.replay.capture
22

3+
import android.view.MotionEvent
34
import io.sentry.Hint
45
import io.sentry.android.replay.ReplayCache
56
import io.sentry.android.replay.ScreenshotRecorderConfig
@@ -27,6 +28,8 @@ internal interface CaptureStrategy {
2728

2829
fun onConfigurationChanged(recorderConfig: ScreenshotRecorderConfig)
2930

31+
fun onTouchEvent(event: MotionEvent)
32+
3033
fun convert(): CaptureStrategy
3134

3235
fun close()

0 commit comments

Comments
 (0)