feat(ci-visibility): gzip agentless request payload (#13301)

Author: gnufede

ddtrace/internal/ci_visibility/writer.py

@@ -160,6 +160,7 @@ def __init__(
             # self._report_metrics = report_metrics,
             reuse_connections=reuse_connections,
             headers=headers,
+            use_gzip=not use_evp,
         )
 
     def stop(self, timeout=None):

ddtrace/internal/writer/writer.py

@@ -1,6 +1,7 @@
 import abc
 import binascii
 from collections import defaultdict
+import gzip
 import logging
 import os
 import sys
@@ -152,13 +153,15 @@ def __init__(
         reuse_connections: Optional[bool] = None,
         headers: Optional[Dict[str, str]] = None,
         report_metrics: bool = True,
+        use_gzip: bool = False,
     ) -> None:
         if processing_interval is None:
             processing_interval = config._trace_writer_interval_seconds
         if timeout is None:
             timeout = agent_config.trace_agent_timeout_seconds
         super(HTTPWriter, self).__init__(interval=processing_interval)
         self.intake_url = intake_url
+        self._intake_accepts_gzip = use_gzip
         self._buffer_size = buffer_size
         self._max_payload_size = max_payload_size
         self._headers = headers or {}
@@ -369,8 +372,20 @@ def _flush_queue_with_client(self, client: WriterClientBase, raise_exc: bool = F
         n_traces = len(client.encoder)
         try:
             encoded, n_traces = client.encoder.encode()
+
             if encoded is None:
                 return
+
+            # Should gzip the payload if intake accepts it
+            if self._intake_accepts_gzip:
+                original_size = len(encoded)
+                # Replace the value to send with the gzipped the value
+                encoded = gzip.compress(encoded, compresslevel=6)
+                log.debug("Original size in bytes: %s, Compressed size: %s", original_size, len(encoded))
+
+                # And add the header
+                self._headers["Content-Encoding"] = "gzip"
+
         except Exception:
             # FIXME(munir): if client.encoder raises an Exception n_traces may not be accurate due to race conditions
             log.error("failed to encode trace with encoder %r", client.encoder, exc_info=True)

releasenotes/notes/ci-vis-gzip-payload-a93c4b5d4687f1c4.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    CI Visibility: This introduces the ability to gzip the payload when using the agentless setup, incurring in less network bandwith consumption.

tests/integration/test_integration_civisibility.py

@@ -73,6 +73,8 @@ def test_civisibility_intake_with_apikey():
 
 @pytest.mark.subprocess()
 def test_civisibility_intake_payloads():
+    import gzip
+
     import mock
 
     from ddtrace.internal.ci_visibility.constants import COVERAGE_TAG_NAME
@@ -101,10 +103,10 @@ def test_civisibility_intake_payloads():
             t.shutdown()
         assert 2 <= conn.request.call_count <= 3
         assert conn.request.call_args_list[0].args[1] == "api/v2/citestcycle"
-        assert (
-            b"svc-no-cov" in conn.request.call_args_list[0].args[2]
+        assert b"svc-no-cov" in gzip.decompress(
+            conn.request.call_args_list[0].args[2]
         ), "requests to the cycle endpoint should include non-coverage spans"
         assert conn.request.call_args_list[1].args[1] == "api/v2/citestcov"
-        assert (
-            b"svc-no-cov" not in conn.request.call_args_list[1].args[2]
+        assert b"svc-no-cov" not in gzip.decompress(
+            conn.request.call_args_list[1].args[2]
         ), "requests to the coverage endpoint should not include non-coverage spans"