Add sent_at to envelope header item (#2638)

Co-authored-by: Sentry Github Bot <[email protected]>
Co-authored-by: Roman Zavarnitsyn <[email protected]>

Author: 3 people

CHANGELOG.md

@@ -8,6 +8,7 @@
 - Make log4j2 integration compatible with log4j 3.0 ([#2634](https://github.com/getsentry/sentry-java/pull/2634))
     - Instead of relying on package scanning, we now use an annotation processor to generate `Log4j2Plugins.dat`
 - Create `User` and `Breadcrumb` from map ([#2614](https://github.com/getsentry/sentry-java/pull/2614))
+- Add `sent_at` to envelope header item ([#2638](https://github.com/getsentry/sentry-java/pull/2638))
 
 ### Fixes
 

sentry-apache-http-client-5/src/main/java/io/sentry/transport/apache/ApacheHttpClientTransport.java

@@ -2,8 +2,10 @@
 
 import static io.sentry.SentryLevel.*;
 
+import io.sentry.DateUtils;
 import io.sentry.Hint;
 import io.sentry.RequestDetails;
+import io.sentry.SentryDate;
 import io.sentry.SentryEnvelope;
 import io.sentry.SentryLevel;
 import io.sentry.SentryOptions;
@@ -76,6 +78,12 @@ public void send(final @NotNull SentryEnvelope envelope, final @NotNull Hint hin
             options.getClientReportRecorder().attachReportToEnvelope(filteredEnvelope);
 
         if (envelopeWithClientReport != null) {
+
+          @NotNull SentryDate now = options.getDateProvider().now();
+          envelopeWithClientReport
+              .getHeader()
+              .setSentAt(DateUtils.nanosToDate(now.nanoTimestamp()));
+
           currentlyRunning.increment();
 
           try (final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();

sentry-apache-http-client-5/src/test/kotlin/io/sentry/transport/apache/ApacheHttpClientTransportTest.kt

@@ -1,10 +1,12 @@
 package io.sentry.transport.apache
 
+import io.sentry.Hint
 import io.sentry.ILogger
 import io.sentry.RequestDetails
 import io.sentry.SentryEnvelope
 import io.sentry.SentryEvent
 import io.sentry.SentryLevel
+import io.sentry.SentryNanotimeDate
 import io.sentry.SentryOptions
 import io.sentry.SentryOptionsManipulator
 import io.sentry.clientreport.NoOpClientReportRecorder
@@ -23,6 +25,7 @@ import org.mockito.kotlin.spy
 import org.mockito.kotlin.times
 import org.mockito.kotlin.verify
 import org.mockito.kotlin.whenever
+import java.util.Date
 import java.util.concurrent.CompletableFuture
 import java.util.concurrent.Executors
 import kotlin.test.AfterTest
@@ -182,4 +185,30 @@ class ApacheHttpClientTransportTest {
         verify(fixture.logger).log(SentryLevel.WARNING, "Failed to flush all events within %s ms", 200L)
         verify(fixture.currentlyRunning, times(1)).decrement()
     }
+
+    @Test
+    fun `sets current date to sent_at in envelope header`() {
+        val now = Date(9001)
+        val sut = fixture.getSut()
+        fixture.options.dateProvider = mock()
+        whenever(fixture.options.dateProvider.now()).thenReturn(SentryNanotimeDate(now, 0))
+
+        val envelope = SentryEnvelope.from(fixture.options.serializer, SentryEvent(), null)
+        sut.send(envelope)
+
+        assertEquals(envelope.header.sentAt, now)
+    }
+
+    @Test
+    fun `sets current date to sent_at in envelope header when sent with hint`() {
+        val now = Date(9001)
+        val sut = fixture.getSut()
+        fixture.options.dateProvider = mock()
+        whenever(fixture.options.dateProvider.now()).thenReturn(SentryNanotimeDate(now, 0))
+
+        val envelope = SentryEnvelope.from(fixture.options.serializer, SentryEvent(), null)
+        sut.send(envelope, Hint())
+
+        assertEquals(envelope.header.sentAt, now)
+    }
 }

sentry/api/sentry.api

@@ -1360,9 +1360,11 @@ public final class io/sentry/SentryEnvelopeHeader : io/sentry/JsonSerializable,
 	public fun <init> (Lio/sentry/protocol/SentryId;Lio/sentry/protocol/SdkVersion;Lio/sentry/TraceContext;)V
 	public fun getEventId ()Lio/sentry/protocol/SentryId;
 	public fun getSdkVersion ()Lio/sentry/protocol/SdkVersion;
+	public fun getSentAt ()Ljava/util/Date;
 	public fun getTraceContext ()Lio/sentry/TraceContext;
 	public fun getUnknown ()Ljava/util/Map;
 	public fun serialize (Lio/sentry/JsonObjectWriter;Lio/sentry/ILogger;)V
+	public fun setSentAt (Ljava/util/Date;)V
 	public fun setUnknown (Ljava/util/Map;)V
 }
 
@@ -1375,6 +1377,7 @@ public final class io/sentry/SentryEnvelopeHeader$Deserializer : io/sentry/JsonD
 public final class io/sentry/SentryEnvelopeHeader$JsonKeys {
 	public static final field EVENT_ID Ljava/lang/String;
 	public static final field SDK Ljava/lang/String;
+	public static final field SENT_AT Ljava/lang/String;
 	public static final field TRACE Ljava/lang/String;
 	public fun <init> ()V
 }

sentry/src/main/java/io/sentry/SentryClient.java

@@ -303,6 +303,7 @@ private boolean shouldSendSessionUpdateForDroppedEvent(
     if (!envelopeItems.isEmpty()) {
       final SentryEnvelopeHeader envelopeHeader =
           new SentryEnvelopeHeader(sentryId, options.getSdkVersion(), traceContext);
+
       return new SentryEnvelope(envelopeHeader, envelopeItems);
     }
 

sentry/src/main/java/io/sentry/SentryEnvelopeHeader.java

@@ -4,6 +4,7 @@
 import io.sentry.protocol.SentryId;
 import io.sentry.vendor.gson.stream.JsonToken;
 import java.io.IOException;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
 import org.jetbrains.annotations.ApiStatus;
@@ -20,6 +21,8 @@ public final class SentryEnvelopeHeader implements JsonSerializable, JsonUnknown
 
   private final @Nullable TraceContext traceContext;
 
+  private @Nullable Date sentAt;
+
   private @Nullable Map<String, Object> unknown;
 
   public SentryEnvelopeHeader(
@@ -56,12 +59,33 @@ public SentryEnvelopeHeader() {
     return traceContext;
   }
 
+  /**
+   * Get the timestamp when the event was sent from the SDK as string in RFC 3339 format. Used for
+   * clock drift correction of the event timestamp. The time zone must be UTC.
+   */
+  public @Nullable Date getSentAt() {
+    return sentAt;
+  }
+
+  /**
+   * Set he timestamp when the event was sent from the SDK as string in RFC 3339 format. Used * for
+   * clock drift correction of the event timestamp. The time zone must be UTC.
+   *
+   * @param sentAt The timestamp should be generated as close as possible to the transmission of the
+   *     event, so that the delay between sending the envelope and receiving it on the server-side
+   *     is minimized.
+   */
+  public void setSentAt(@Nullable Date sentAt) {
+    this.sentAt = sentAt;
+  }
+
   // JsonSerializable
 
   public static final class JsonKeys {
     public static final String EVENT_ID = "event_id";
     public static final String SDK = "sdk";
     public static final String TRACE = "trace";
+    public static final String SENT_AT = "sent_at";
   }
 
   @Override
@@ -77,6 +101,9 @@ public void serialize(@NotNull JsonObjectWriter writer, @NotNull ILogger logger)
     if (traceContext != null) {
       writer.name(JsonKeys.TRACE).value(logger, traceContext);
     }
+    if (sentAt != null) {
+      writer.name(JsonKeys.SENT_AT).value(logger, DateUtils.getTimestamp(sentAt));
+    }
     if (unknown != null) {
       for (String key : unknown.keySet()) {
         Object value = unknown.get(key);
@@ -96,6 +123,7 @@ public static final class Deserializer implements JsonDeserializer<SentryEnvelop
       SentryId eventId = null;
       SdkVersion sdkVersion = null;
       TraceContext traceContext = null;
+      Date sentAt = null;
       Map<String, Object> unknown = null;
 
       while (reader.peek() == JsonToken.NAME) {
@@ -110,6 +138,9 @@ public static final class Deserializer implements JsonDeserializer<SentryEnvelop
           case JsonKeys.TRACE:
             traceContext = reader.nextOrNull(logger, new TraceContext.Deserializer());
             break;
+          case JsonKeys.SENT_AT:
+            sentAt = reader.nextDateOrNull(logger);
+            break;
           default:
             if (unknown == null) {
               unknown = new HashMap<>();
@@ -120,6 +151,7 @@ public static final class Deserializer implements JsonDeserializer<SentryEnvelop
       }
       SentryEnvelopeHeader sentryEnvelopeHeader =
           new SentryEnvelopeHeader(eventId, sdkVersion, traceContext);
+      sentryEnvelopeHeader.setSentAt(sentAt);
       sentryEnvelopeHeader.setUnknown(unknown);
       reader.endObject();
       return sentryEnvelopeHeader;

sentry/src/main/java/io/sentry/transport/AsyncHttpTransport.java

@@ -1,8 +1,10 @@
 package io.sentry.transport;
 
+import io.sentry.DateUtils;
 import io.sentry.Hint;
 import io.sentry.ILogger;
 import io.sentry.RequestDetails;
+import io.sentry.SentryDate;
 import io.sentry.SentryEnvelope;
 import io.sentry.SentryLevel;
 import io.sentry.SentryOptions;
@@ -219,6 +221,7 @@ public void run() {
     private @NotNull TransportResult flush() {
       TransportResult result = this.failedResult;
 
+      envelope.getHeader().setSentAt(null);
       envelopeCache.store(envelope, hint);
 
       HintUtils.runIfHasType(
@@ -233,6 +236,12 @@ public void run() {
         final SentryEnvelope envelopeWithClientReport =
             options.getClientReportRecorder().attachReportToEnvelope(envelope);
         try {
+
+          @NotNull SentryDate now = options.getDateProvider().now();
+          envelopeWithClientReport
+              .getHeader()
+              .setSentAt(DateUtils.nanosToDate(now.nanoTimestamp()));
+
           result = connection.send(envelopeWithClientReport);
           if (result.isSuccess()) {
             envelopeCache.discard(envelope);

sentry/src/test/java/io/sentry/protocol/SentryEnvelopeHeaderSerializationTest.kt

@@ -1,5 +1,6 @@
 package io.sentry.protocol
 
+import io.sentry.DateUtils
 import io.sentry.FileFromResources
 import io.sentry.ILogger
 import io.sentry.JsonObjectReader
@@ -22,7 +23,9 @@ class SentryEnvelopeHeaderSerializationTest {
             SentryIdSerializationTest.Fixture().getSut(),
             SdkVersionSerializationTest.Fixture().getSut(),
             TraceContextSerializationTest.Fixture().getSut()
-        )
+        ).apply {
+            sentAt = DateUtils.getDateTime("2020-02-07T14:16:00.000Z")
+        }
     }
     private val fixture = Fixture()
 

sentry/src/test/java/io/sentry/transport/AsyncHttpTransportTest.kt

@@ -1,10 +1,12 @@
 package io.sentry.transport
 
 import io.sentry.CachedEvent
+import io.sentry.Hint
 import io.sentry.SentryEnvelope
 import io.sentry.SentryEnvelopeHeader
 import io.sentry.SentryEnvelopeItem
 import io.sentry.SentryEvent
+import io.sentry.SentryNanotimeDate
 import io.sentry.SentryOptions
 import io.sentry.SentryOptionsManipulator
 import io.sentry.Session
@@ -22,6 +24,7 @@ import org.mockito.kotlin.never
 import org.mockito.kotlin.verify
 import org.mockito.kotlin.whenever
 import java.io.IOException
+import java.util.Date
 import kotlin.test.Test
 import kotlin.test.assertEquals
 
@@ -228,7 +231,7 @@ class AsyncHttpTransportTest {
         whenever(fixture.connection.send(any<SentryEnvelope>())).thenReturn(TransportResult.success())
         fixture.getSUT().send(envelope)
         verify(fixture.connection).send(
-            check<SentryEnvelope> {
+            check {
                 assertEquals(1, it.items.count())
             }
         )
@@ -270,6 +273,54 @@ class AsyncHttpTransportTest {
         verify(fixture.executor).waitTillIdle(500)
     }
 
+    @Test
+    fun `sets current date to sent_at in envelope header before send`() {
+        // given
+        val now = Date(9001)
+        fixture.sentryOptions.dateProvider = mock()
+        whenever(fixture.sentryOptions.dateProvider.now()).thenReturn(SentryNanotimeDate(now, 0))
+
+        val envelope = SentryEnvelope.from(fixture.sentryOptions.serializer, createSession(), null)
+        whenever(fixture.transportGate.isConnected).thenReturn(true)
+        whenever(fixture.rateLimiter.filter(any(), anyOrNull())).thenAnswer { it.arguments[0] }
+        whenever(fixture.connection.send(any())).thenReturn(TransportResult.success())
+
+        // when
+        val sut = fixture.getSUT()
+        sut.send(envelope)
+
+        // then
+        verify(fixture.connection).send(
+            check {
+                assertEquals(it.header.sentAt, now)
+            }
+        )
+    }
+
+    @Test
+    fun `sets current date to sent_at in envelope header before send when sent with hint`() {
+        // given
+        val now = Date(9001)
+        fixture.sentryOptions.dateProvider = mock()
+        whenever(fixture.sentryOptions.dateProvider.now()).thenReturn(SentryNanotimeDate(now, 0))
+
+        val envelope = SentryEnvelope.from(fixture.sentryOptions.serializer, createSession(), null)
+        whenever(fixture.transportGate.isConnected).thenReturn(true)
+        whenever(fixture.rateLimiter.filter(any(), anyOrNull())).thenAnswer { it.arguments[0] }
+        whenever(fixture.connection.send(any())).thenReturn(TransportResult.success())
+
+        // when
+        val sut = fixture.getSUT()
+        sut.send(envelope, Hint())
+
+        // then
+        verify(fixture.connection).send(
+            check {
+                assertEquals(it.header.sentAt, now)
+            }
+        )
+    }
+
     private fun createSession(): Session {
         return Session("123", User(), "env", "release")
     }

sentry/src/test/resources/json/sentry_envelope_header.json

@@ -27,5 +27,6 @@
         "user_segment": "f7d8662b-5551-4ef8-b6a8-090f0561a530",
         "transaction": "0252ec25-cd0a-4230-bd2f-936a4585637e",
         "sample_rate": "0.00000021"
-    }
+    },
+    "sent_at": "2020-02-07T14:16:00.000Z"
 }