Cleanup of zstd dependency and jps functionality (#9010)

* Make the zstd dependency more lightweight

* Simplify the JVMs pids collection

Author: jbachorik

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java

@@ -3,7 +3,6 @@
 import static datadog.trace.api.ConfigDefaults.DEFAULT_STARTUP_LOGS_ENABLED;
 import static datadog.trace.api.Platform.isJavaVersionAtLeast;
 import static datadog.trace.api.Platform.isOracleJDK8;
-import static datadog.trace.api.telemetry.LogCollector.SEND_TELEMETRY;
 import static datadog.trace.bootstrap.Library.WILDFLY;
 import static datadog.trace.bootstrap.Library.detectLibraries;
 import static datadog.trace.util.AgentThreadFactory.AgentThread.JMX_STARTUP;
@@ -45,7 +44,6 @@
 import datadog.trace.util.AgentTaskScheduler;
 import datadog.trace.util.AgentThreadFactory.AgentThread;
 import datadog.trace.util.throwable.FatalAgentMisconfigurationError;
-import de.thetaphi.forbiddenapis.SuppressForbidden;
 import java.lang.instrument.Instrumentation;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
@@ -291,8 +289,6 @@ public static void start(
     codeOriginEnabled = isFeatureEnabled(AgentFeature.CODE_ORIGIN);
     agentlessLogSubmissionEnabled = isFeatureEnabled(AgentFeature.AGENTLESS_LOG_SUBMISSION);
 
-    patchJPSAccess(inst);
-
     if (profilingEnabled) {
       if (!isOracleJDK8()) {
         // Profiling agent startup code is written in a way to allow `startProfilingAgent` be called
@@ -422,23 +418,6 @@ private static void injectAgentArgsConfig(String agentArgs) {
     }
   }
 
-  @SuppressForbidden
-  public static void patchJPSAccess(Instrumentation inst) {
-    if (Platform.isJavaVersionAtLeast(9)) {
-      // Unclear if supported for J9, may need to revisit
-      try {
-        Class.forName("datadog.trace.util.JPMSJPSAccess")
-            .getMethod("patchModuleAccess", Instrumentation.class)
-            .invoke(null, inst);
-      } catch (Exception e) {
-        log.debug(
-            SEND_TELEMETRY,
-            "Failed to patch module access for jvmstat and Java version "
-                + Platform.getRuntimeVersion());
-      }
-    }
-  }
-
   public static void shutdown(final boolean sync) {
     StaticEventLogger.end("Agent");
     StaticEventLogger.stop();

dd-java-agent/build.gradle

@@ -83,6 +83,16 @@ ext.generalShadowJarConfig = {
   exclude '**/com/kenai/jffi/Init.class'
   relocate('com.kenai.jffi.Init', 'com.kenai.jffi.PatchInit')
 
+  // Minimize and relocate the airlift compressor dependency for ZSTD
+  exclude '**/io/airlift/compress/bzip2/**'
+  exclude '**/io/airlift/compress/deflate/**'
+  exclude '**/io/airlift/compress/gzip/**'
+  exclude '**/io/airlift/compress/hadoop/**'
+  exclude '**/io/airlift/compress/lz4/**'
+  exclude '**/io/airlift/compress/lzo/**'
+  exclude '**/io/airlift/compress/snappy/**'
+  relocate 'io.airlift', 'datadog.io.airlift'
+
   final String projectName = "${project.name}"
 
   // Prevents conflict with other instances, but doesn't relocate instrumentation

dd-smoke-tests/profiling-integration-tests/build.gradle

@@ -20,14 +20,16 @@ dependencies {
   implementation project(':dd-trace-api')
   api project(':dd-trace-ot')
   implementation 'org.apache.commons:commons-math3:3.6.1'
-  implementation 'org.lz4:lz4-java:1.8.0'
+  implementation libs.lz4
+  implementation libs.aircompressor
   implementation 'org.xerial.snappy:snappy-java:1.1.8.4'
 
   testImplementation project(':dd-smoke-tests')
   testImplementation project(':dd-java-agent:agent-profiling:profiling-testing')
   testImplementation libs.bundles.junit5
   testImplementation libs.bundles.mockito
   testImplementation libs.bundles.jmc
+  testImplementation libs.aircompressor
   testImplementation(group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.9.10')
 }
 

dd-smoke-tests/profiling-integration-tests/src/test/java/datadog/smoketest/JFRBasedProfilingIntegrationTest.java

@@ -17,9 +17,11 @@
 import datadog.trace.api.Platform;
 import datadog.trace.api.config.ProfilingConfig;
 import delight.fileupload.FileUpload;
+import io.airlift.compress.zstd.ZstdInputStream;
 import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.InetAddress;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -150,7 +152,7 @@ void teardown() throws Exception {
   }
 
   @Test
-  @DisplayName("Test continuous recording - no jmx delay, no jmethodid cache")
+  @DisplayName("Test continuous recording - no jmx delay, default compression")
   public void testContinuousRecording_no_jmx_delay(final TestInfo testInfo) throws Exception {
     testWithRetry(
         () ->
@@ -161,7 +163,7 @@ public void testContinuousRecording_no_jmx_delay(final TestInfo testInfo) throws
   }
 
   @Test
-  @DisplayName("Test continuous recording - no jmx delay, jmethodid cache")
+  @DisplayName("Test continuous recording - no jmx delay, zstd compression")
   public void testContinuousRecording_no_jmx_delay_jmethodid_cache(final TestInfo testInfo)
       throws Exception {
     testWithRetry(
@@ -173,7 +175,7 @@ public void testContinuousRecording_no_jmx_delay_jmethodid_cache(final TestInfo
   }
 
   @Test
-  @DisplayName("Test continuous recording - 1 sec jmx delay, no jmethodid cache")
+  @DisplayName("Test continuous recording - 1 sec jmx delay, default compression")
   public void testContinuousRecording(final TestInfo testInfo) throws Exception {
     testWithRetry(
         () ->
@@ -184,7 +186,7 @@ public void testContinuousRecording(final TestInfo testInfo) throws Exception {
   }
 
   @Test
-  @DisplayName("Test continuous recording - 1 sec jmx delay, jmethodid cache")
+  @DisplayName("Test continuous recording - 1 sec jmx delay, zstd compression")
   public void testContinuousRecording_jmethodid_cache(final TestInfo testInfo) throws Exception {
     testWithRetry(
         () ->
@@ -198,7 +200,7 @@ private void testContinuousRecording(
       final int jmxFetchDelay,
       final boolean endpointCollectionEnabled,
       final boolean asyncProfilerEnabled,
-      final boolean jmethodIdCacheEnabled)
+      final boolean withZstd)
       throws Exception {
     final ObjectMapper mapper = new ObjectMapper();
     try {
@@ -207,7 +209,7 @@ private void testContinuousRecording(
                   jmxFetchDelay,
                   endpointCollectionEnabled,
                   asyncProfilerEnabled,
-                  jmethodIdCacheEnabled,
+                  withZstd,
                   logFilePath)
               .start();
 
@@ -261,7 +263,11 @@ private void testContinuousRecording(
       assertEquals(InetAddress.getLocalHost().getHostName(), requestTags.get("host"));
 
       assertFalse(logHasErrors(logFilePath));
-      IItemCollection events = JfrLoaderToolkit.loadEvents(new ByteArrayInputStream(rawJfr.get()));
+      InputStream eventStream = new ByteArrayInputStream(rawJfr.get());
+      if (withZstd) {
+        eventStream = new ZstdInputStream(eventStream);
+      }
+      IItemCollection events = JfrLoaderToolkit.loadEvents(eventStream);
       assertTrue(events.hasItems());
       Pair<Instant, Instant> rangeStartAndEnd = getRangeStartAndEnd(events);
       // This nano-second compensates for the added nano second in
@@ -308,7 +314,11 @@ private void testContinuousRecording(
           period > 0 && period <= upperLimit,
           () -> "Upload period = " + period + "ms, expected (0, " + upperLimit + "]ms");
 
-      events = JfrLoaderToolkit.loadEvents(new ByteArrayInputStream(rawJfr.get()));
+      eventStream = new ByteArrayInputStream(rawJfr.get());
+      if (withZstd) {
+        eventStream = new ZstdInputStream(eventStream);
+      }
+      events = JfrLoaderToolkit.loadEvents(eventStream);
       assertTrue(events.hasItems());
       verifyDatadogEventsNotCorrupt(events);
       rangeStartAndEnd = getRangeStartAndEnd(events);
@@ -689,7 +699,7 @@ private ProcessBuilder createDefaultProcessBuilder(
       final int jmxFetchDelay,
       final boolean endpointCollectionEnabled,
       final boolean asyncProfilerEnabled,
-      final boolean jmethodIdCacheEnabled,
+      final boolean withZstd,
       final Path logFilePath) {
     return createProcessBuilder(
         VALID_API_KEY,
@@ -698,7 +708,7 @@ private ProcessBuilder createDefaultProcessBuilder(
         PROFILING_UPLOAD_PERIOD_SECONDS,
         endpointCollectionEnabled,
         asyncProfilerEnabled,
-        jmethodIdCacheEnabled,
+        withZstd,
         0,
         logFilePath);
   }
@@ -710,7 +720,7 @@ private ProcessBuilder createProcessBuilder(
       final int profilingUploadPeriodSecs,
       final boolean endpointCollectionEnabled,
       final boolean asyncProfilerEnabled,
-      final boolean jmethodIdCacheEnabled,
+      final boolean withZstd,
       final int exitDelay,
       final Path logFilePath) {
     return createProcessBuilder(
@@ -722,7 +732,7 @@ private ProcessBuilder createProcessBuilder(
         profilingUploadPeriodSecs,
         endpointCollectionEnabled,
         asyncProfilerEnabled,
-        jmethodIdCacheEnabled,
+        withZstd,
         exitDelay,
         logFilePath);
   }
@@ -736,7 +746,7 @@ private static ProcessBuilder createProcessBuilder(
       final int profilingUploadPeriodSecs,
       final boolean endpointCollectionEnabled,
       final boolean asyncProfilerEnabled,
-      final boolean jmethodIdCacheEnabled,
+      final boolean withZstd,
       final int exitDelay,
       final Path logFilePath) {
     final String templateOverride =
@@ -770,7 +780,7 @@ private static ProcessBuilder createProcessBuilder(
             "-Ddd.profiling.debug.dump_path=/tmp/dd-profiler",
             "-Ddd.profiling.queueing.time.enabled=true",
             "-Ddd.profiling.queueing.time.threshold.millis=0",
-            "-Ddd.profiling.experimental.jmethodid_cache.enabled=" + jmethodIdCacheEnabled,
+            "-Ddd.profiling.debug.upload.compression=" + (withZstd ? "zstd" : "on"),
             "-Ddatadog.slf4j.simpleLogger.defaultLogLevel=debug",
             "-Ddd.profiling.context.attributes=foo,bar",
             "-Dorg.slf4j.simpleLogger.defaultLogLevel=debug",

gradle/dependencies.gradle

@@ -36,6 +36,7 @@ final class CachedData {
       exclude(dependency('com.datadoghq.okio:okio'))
       exclude(dependency('com.squareup.okio:okio'))
       exclude(dependency('org.lz4:lz4-java'))
+      exclude(dependency('io.airlift:aircompressor'))
 
       // dogstatsd and its transitives
       exclude(dependency('com.datadoghq:java-dogstatsd-client'))
@@ -67,7 +68,8 @@ CachedData.deps.shared = [
   libs.jnr.unixsocket,
   libs.moshi,
   libs.jctools,
-  libs.lz4
+  libs.lz4,
+  libs.aircompressor
 ]
 
 ext {