Merge branch 'master' into ygree/servicetalk-concurrent-instrumentation-0.42.56

Author: ygree

.circleci/config.continue.yml.j2

@@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
-default_system_tests_commit: &default_system_tests_commit 9049791de92dcb72b104e99a80f6f9914c1b96fb
+default_system_tests_commit: &default_system_tests_commit ab6cf66b9c744f4fc273907423e93598440cd211
 
 parameters:
   nightly:

components/yaml/build.gradle.kts

@@ -4,7 +4,6 @@ plugins {
 
 apply(from = "$rootDir/gradle/java.gradle")
 
-// https://repo1.maven.org/maven2/org/yaml/snakeyaml/2.4/snakeyaml-2.4.pom
 dependencies {
-  implementation("org.yaml", "snakeyaml", "2.4")
+  implementation("org.snakeyaml", "snakeyaml-engine", "2.9")
 }

components/yaml/src/main/java/datadog/yaml/YamlParser.java

@@ -1,15 +1,18 @@
 package datadog.yaml;
 
-import org.yaml.snakeyaml.Yaml;
+import org.snakeyaml.engine.v2.api.Load;
+import org.snakeyaml.engine.v2.api.LoadSettings;
 
 public class YamlParser {
-  // Supports clazz == null for default yaml parsing
-  public static <T> T parse(String content, Class<T> clazz) {
-    Yaml yaml = new Yaml();
-    if (clazz == null) {
-      return yaml.load(content);
-    } else {
-      return yaml.loadAs(content, clazz);
-    }
+  /**
+   * Parses YAML content. Duplicate keys are not allowed and will result in a runtime exception..
+   *
+   * @param content - text context to be parsed as YAML
+   * @return - a parsed representation as a composition of map and list objects.
+   */
+  public static Object parse(String content) {
+    LoadSettings settings = LoadSettings.builder().build();
+    Load yaml = new Load(settings);
+    return yaml.loadFromString(content);
   }
 }

dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/ProfilerSettingsSupport.java

@@ -123,10 +123,12 @@ protected ProfilerSettingsSupport(
         configProvider.getInteger(
             ProfilingConfig.PROFILING_UPLOAD_TIMEOUT,
             ProfilingConfig.PROFILING_UPLOAD_TIMEOUT_DEFAULT);
+    // First try the new debug upload compression property, and fall back to the deprecated one
     uploadCompression =
         configProvider.getString(
-            ProfilingConfig.PROFILING_UPLOAD_COMPRESSION,
-            ProfilingConfig.PROFILING_UPLOAD_COMPRESSION_DEFAULT);
+            ProfilingConfig.PROFILING_DEBUG_UPLOAD_COMPRESSION,
+            ProfilingConfig.PROFILING_DEBUG_UPLOAD_COMPRESSION_DEFAULT,
+            ProfilingConfig.PROFILING_UPLOAD_COMPRESSION);
     allocationProfilingEnabled =
         configProvider.getBoolean(
             ProfilingConfig.PROFILING_ALLOCATION_ENABLED,

dd-java-agent/agent-profiling/profiling-uploader/build.gradle

@@ -31,6 +31,7 @@ dependencies {
 
   implementation libs.okhttp
   implementation libs.lz4
+  implementation group: 'io.airlift', name: 'aircompressor', version: '2.0.2'
 
   testImplementation libs.bundles.junit5
   testImplementation project(':dd-java-agent:agent-profiling:profiling-testing')

dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/CompressingRequestBody.java

@@ -2,6 +2,7 @@
 
 import datadog.trace.api.Platform;
 import datadog.trace.api.profiling.RecordingInputStream;
+import io.airlift.compress.zstd.ZstdOutputStream;
 import java.io.BufferedOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -86,6 +87,7 @@ interface RetryBackoff {
   private static final int[] LZ4_MAGIC = new int[] {0x04, 0x22, 0x4D, 0x18};
   private static final int ZIP_MAGIC[] = new int[] {80, 75, 3, 4};
   private static final int GZ_MAGIC[] = new int[] {31, 139};
+  private static final int ZSTD_MAGIC[] = new int[] {0x28, 0xB5, 0x2F, 0xFD};
 
   private final InputStreamSupplier inputStreamSupplier;
   private final OutputStreamMappingFunction outputStreamMapper;
@@ -269,7 +271,24 @@ public void close() throws IOException {
    */
   static boolean isCompressed(@Nonnull final InputStream is) throws IOException {
     checkMarkSupported(is);
-    return isGzip(is) || isLz4(is) || isZip(is);
+    return isGzip(is) || isLz4(is) || isZip(is) || isZstd(is);
+  }
+
+  /**
+   * Check whether the stream represents ZSTD data
+   *
+   * @param is input stream; must support {@linkplain InputStream#mark(int)}
+   * @return {@literal true} if the stream represents ZSTD data
+   * @throws IOException
+   */
+  static boolean isZstd(@Nonnull final InputStream is) throws IOException {
+    checkMarkSupported(is);
+    is.mark(ZSTD_MAGIC.length);
+    try {
+      return hasMagic(is, ZSTD_MAGIC);
+    } finally {
+      is.reset();
+    }
   }
 
   /**
@@ -331,12 +350,11 @@ private static void checkMarkSupported(@Nonnull final InputStream is) throws IOE
 
   private static OutputStreamMappingFunction getOutputStreamMapper(
       @Nonnull CompressionType compressionType) {
-    // currently only gzip and off are supported
-    // this needs to be updated once more compression types are added
-    compressionType =
-        (Platform.isNativeImage() && compressionType != CompressionType.OFF
-            ? CompressionType.GZIP
-            : compressionType);
+    // Handle native image compatibility
+    if (Platform.isNativeImage() && compressionType != CompressionType.OFF) {
+      compressionType = CompressionType.GZIP;
+    }
+
     switch (compressionType) {
       case GZIP:
         {
@@ -346,6 +364,10 @@ private static OutputStreamMappingFunction getOutputStreamMapper(
         {
           return out -> out;
         }
+      case ZSTD:
+        {
+          return CompressingRequestBody::toZstdStream;
+        }
       case ON:
       case LZ4:
       default:
@@ -355,6 +377,12 @@ private static OutputStreamMappingFunction getOutputStreamMapper(
     }
   }
 
+  private static OutputStream toZstdStream(@Nonnull OutputStream os) throws IOException {
+    // Default compression level is 3 which provides a good balance between performance and
+    // compression ratio
+    return new ZstdOutputStream(os);
+  }
+
   private static OutputStream toLz4Stream(@Nonnull OutputStream os) throws IOException {
     return new LZ4FrameOutputStream(
         os,

dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/CompressionType.java

@@ -12,7 +12,9 @@ enum CompressionType {
   /** Lower compression ratio with less CPU overhead * */
   LZ4,
   /** Better compression ratio for the price of higher CPU usage * */
-  GZIP;
+  GZIP,
+  /** High compression ratio with reasonable CPU usage * */
+  ZSTD;
 
   private static final Logger log = LoggerFactory.getLogger(CompressionType.class);
 
@@ -30,8 +32,10 @@ static CompressionType of(String type) {
         return LZ4;
       case "gzip":
         return GZIP;
+      case "zstd":
+        return ZSTD;
       default:
-        log.warn("Unrecognizable compression type: {}. Defaulting to 'on'.", type);
+        log.warn("Unrecognizable compression type: {}. Defaulting to 'lz4'.", type);
         return ON;
     }
   }

dd-java-agent/agent-profiling/profiling-uploader/src/test/java/com/datadog/profiling/uploader/CompressingRequestBodyTest.java

@@ -6,6 +6,8 @@
 import static org.mockito.Mockito.when;
 
 import datadog.trace.api.profiling.RecordingInputStream;
+import io.airlift.compress.zstd.ZstdInputStream;
+import io.airlift.compress.zstd.ZstdOutputStream;
 import java.io.BufferedInputStream;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -186,6 +188,15 @@ void writeTo(CompressionType compressionType) throws IOException {
           assertEquals(compressed.length, instance.getWrittenBytes());
           break;
         }
+      case ZSTD:
+        {
+          assertTrue(CompressingRequestBody.isZstd(compressedStream));
+          byte[] uncompressed = IOUtils.toByteArray(new ZstdInputStream(compressedStream));
+          assertArrayEquals(recordingData, uncompressed);
+          assertEquals(recordingData.length, instance.getReadBytes());
+          assertEquals(compressed.length, instance.getWrittenBytes());
+          break;
+        }
     }
   }
 
@@ -210,6 +221,11 @@ void writeToRecompression(CompressionType targetType) throws IOException {
             compressedStream = new GZIPOutputStream(baos);
             break;
           }
+        case ZSTD:
+          {
+            compressedStream = new ZstdOutputStream(baos);
+            break;
+          }
       }
       assertNotNull(compressedStream);
 

dd-java-agent/build.gradle

@@ -38,7 +38,7 @@ ext.generalShadowJarConfig = {
   // used to report our own dependencies, but we should remove the top-level metadata
   // of vendored packages because those could trigger unwanted framework checks.
   exclude '/META-INF/maven/org.slf4j/**'
-  exclude '/META-INF/maven/org.yaml/**'
+  exclude '/META-INF/maven/org.snakeyaml/**'
   exclude '**/META-INF/maven/**/pom.xml'
   exclude '**/META-INF/proguard/'
   exclude '**/META-INF/*.kotlin_module'
@@ -66,7 +66,7 @@ ext.generalShadowJarConfig = {
 
   // Prevents conflict with other instances, but doesn't relocate instrumentation
   if (!projectName.equals('instrumentation')) {
-    relocate 'org.yaml.snakeyaml', 'datadog.snakeyaml'
+    relocate 'org.snakeyaml.engine', 'datadog.snakeyaml.engine'
     relocate 'okhttp3', 'datadog.okhttp3'
     relocate 'okio',  'datadog.okio'
   }
@@ -306,7 +306,7 @@ tasks.register('checkAgentJarSize').configure {
   doLast {
     // Arbitrary limit to prevent unintentional increases to the agent jar size
     // Raise or lower as required
-    assert shadowJar.archiveFile.get().getAsFile().length() <= 31 * 1024 * 1024
+    assert shadowJar.archiveFile.get().getAsFile().length() <= 32 * 1024 * 1024
   }
 
   dependsOn "shadowJar"

dd-java-agent/instrumentation/build.gradle

@@ -60,7 +60,7 @@ subprojects { Project subProj ->
       jdkCompile = "main_${name}Implementation"
     }
     configurations.muzzleBootstrap {
-      exclude group: 'org.yaml', module : 'snakeyaml' // we vendor this in the agent jar
+      exclude group: 'org.snakeyaml', module : 'snakeyaml-engine' // we vendor this in the agent jar
     }
     dependencies {
       // Apply common dependencies for instrumentation.