Merge branch 'master' into daniel.mohedano/maven-smoke-feedback

Author: daniel-mohedano

.gitlab-ci.yml

@@ -25,10 +25,10 @@ variables:
   BUILD_JOB_NAME: "build"
   DEPENDENCY_CACHE_POLICY: pull
   BUILD_CACHE_POLICY: pull
-  GRADLE_VERSION: "8.5" # must match gradle-wrapper.properties
+  GRADLE_VERSION: "8.14.3" # must match gradle-wrapper.properties
   MAVEN_REPOSITORY_PROXY: "http://artifactual.artifactual.all-clusters.local-dc.fabric.dog:8081/repository/maven-central/"
   GRADLE_PLUGIN_PROXY: "http://artifactual.artifactual.all-clusters.local-dc.fabric.dog:8081/repository/gradle-plugin-portal-proxy/"
-  BUILDER_IMAGE_VERSION_PREFIX: "v25.06-" # use either an empty string (e.g. "") for latest images or a version followed by a hyphen (e.g. "v25.05-")
+  BUILDER_IMAGE_VERSION_PREFIX: "v25.07-" # use either an empty string (e.g. "") for latest images or a version followed by a hyphen (e.g. "v25.05-")
   REPO_NOTIFICATION_CHANNEL: "#apm-java-escalations"
   DEFAULT_TEST_JVMS: /^(8|11|17|21|stable)$/
   PROFILE_TESTS:
@@ -137,9 +137,12 @@ default:
     KUBERNETES_MEMORY_REQUEST: 8Gi
     KUBERNETES_MEMORY_LIMIT: 8Gi
     CACHE_TYPE: lib #default
+    FF_USE_FASTZIP: "true"
+    CACHE_COMPRESSION_LEVEL: "slowest"
+
     RUNTIME_AVAILABLE_PROCESSORS_OVERRIDE: 4 # Runtime.getRuntime().availableProcessors() returns incorrect or very high values in Kubernetes
   cache:
-    - key: '$CI_SERVER_VERSION-$CACHE_TYPE' # Dependencies cache. Reset the cache every time gitlab is upgraded.  ~Every couple months
+    - key: dependency-$CACHE_TYPE # Dependencies cache
       paths:
         # Cached dependencies and wrappers for gradle
         - .gradle/wrapper
@@ -148,8 +151,8 @@ default:
       policy: $DEPENDENCY_CACHE_POLICY
       unprotect: true
       fallback_keys: # Use fallback keys because all cache types are not populated. See note under: populate_dep_cache
-        - '$CI_SERVER_VERSION-base'
-        - '$CI_SERVER_VERSION-lib'
+        - 'dependency-base'
+        - 'dependency-lib'
     - key: $CI_PIPELINE_ID-$CACHE_TYPE # Incremental build cache. Shared by all jobs in the pipeline of the same type
       paths:
         - .gradle/caches/$GRADLE_VERSION
@@ -837,7 +840,7 @@ package-oci:
 
 configure_system_tests:
   variables:
-    SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,docker-ssi,lib-injection"
+    SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,simple_onboarding_appsec,docker-ssi,lib-injection"
 
 create_key:
   stage: generate-signing-key

.gitlab/one-pipeline.locked.yml

@@ -1,4 +1,4 @@
 # DO NOT EDIT THIS FILE MANUALLY
 # This file is auto-generated by automation.
 include:
-  - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/d44e89797a5a47c43cf712193abefe2178a004176606f7e01b77d1ec49a3ef5e/one-pipeline.yml
+  - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/a0486057161f85a77e39ad2aa60ac66bb52414696d9b3dd87177df1057b11295/one-pipeline.yml

.gitlab/upload_ciapp.sh

@@ -35,5 +35,20 @@ junit_upload() {
         ./results
 }
 
+# Upload code coverage results to Datadog
+coverage_upload() {
+    DD_API_KEY=$1 \
+    [email protected]:DataDog/dd-trace-java.git \
+        datadog-ci coverage upload --ignored-paths=./test-published-dependencies .
+}
+
 # Upload test results to production environment like all other CI jobs
 junit_upload "$DATADOG_API_KEY_PROD"
+junit_upload_status=$?
+
+coverage_upload "$DATADOG_API_KEY_PROD"
+coverage_upload_status=$?
+
+if [[ $junit_upload_status -ne 0 || $coverage_upload_status -ne 0 ]]; then
+  exit 1
+fi

buildSrc/build.gradle.kts

@@ -46,7 +46,7 @@ dependencies {
   implementation("org.ow2.asm", "asm", "9.8")
   implementation("org.ow2.asm", "asm-tree", "9.8")
 
-  testImplementation("org.spockframework", "spock-core", "2.2-groovy-3.0")
+  testImplementation(libs.spock.core)
   testImplementation("org.codehaus.groovy", "groovy-all", "3.0.17")
 }
 

buildSrc/call-site-instrumentation-plugin/build.gradle.kts

@@ -37,7 +37,7 @@ dependencies {
   implementation("com.github.javaparser", "javaparser-symbol-solver-core", "3.24.4")
 
   testImplementation("net.bytebuddy", "byte-buddy", "1.17.5")
-  testImplementation("org.spockframework", "spock-core", "2.0-groovy-3.0")
+  testImplementation(libs.spock.core)
   testImplementation("org.objenesis", "objenesis", "3.0.1")
   testImplementation("org.codehaus.groovy", "groovy-all", "3.0.17")
   testImplementation("javax.servlet", "javax.servlet-api", "3.0.1")
@@ -68,7 +68,7 @@ val copyCallSiteSources = tasks.register<Copy>("copyCallSiteSources") {
 }
 
 tasks {
-  withType<AbstractCompile>() {
+  withType<AbstractCompile> {
     dependsOn(copyCallSiteSources)
   }
 }

buildSrc/settings.gradle.kts

@@ -1 +1,9 @@
 include(":call-site-instrumentation-plugin")
+
+dependencyResolutionManagement {
+  versionCatalogs {
+    create("libs") {
+      from(files("../gradle/libs.versions.toml"))
+    }
+  }
+}

communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java

@@ -1,11 +1,16 @@
 package datadog.communication.ddagent;
 
+import static datadog.communication.http.OkHttpUtils.DATADOG_CONTAINER_ID;
+import static datadog.communication.http.OkHttpUtils.DATADOG_CONTAINER_TAGS_HASH;
 import static datadog.communication.serialization.msgpack.MsgPackWriter.FIXARRAY;
+import static java.util.Collections.emptySet;
 import static java.util.Collections.singletonList;
+import static java.util.Collections.unmodifiableSet;
 
 import com.squareup.moshi.JsonAdapter;
 import com.squareup.moshi.Moshi;
 import com.squareup.moshi.Types;
+import datadog.common.container.ContainerInfo;
 import datadog.communication.http.OkHttpUtils;
 import datadog.communication.monitor.DDAgentStatsDClientManager;
 import datadog.communication.monitor.Monitoring;
@@ -86,6 +91,8 @@ public class DDAgentFeaturesDiscovery implements DroppingPolicy {
   private volatile String evpProxyEndpoint;
   private volatile String version;
   private volatile String telemetryProxyEndpoint;
+  private volatile Set<String> peerTags = emptySet();
+  private volatile Set<String> spanKindsToComputedStats = emptySet();
 
   private long lastTimeDiscovered;
 
@@ -119,6 +126,8 @@ private void reset() {
     version = null;
     lastTimeDiscovered = 0;
     telemetryProxyEndpoint = null;
+    peerTags = emptySet();
+    spanKindsToComputedStats = emptySet();
   }
 
   /** Run feature discovery, unconditionally. */
@@ -151,11 +160,15 @@ private void doDiscovery() {
     // 3. fallback if the endpoint couldn't be found or the response couldn't be parsed
     try (Recording recording = discoveryTimer.start()) {
       boolean fallback = true;
-      try (Response response =
-          client
-              .newCall(new Request.Builder().url(agentBaseUrl.resolve("info").url()).build())
-              .execute()) {
+      final Request.Builder requestBuilder =
+          new Request.Builder().url(agentBaseUrl.resolve("info").url());
+      final String containerId = ContainerInfo.get().getContainerId();
+      if (containerId != null) {
+        requestBuilder.header(DATADOG_CONTAINER_ID, containerId);
+      }
+      try (Response response = client.newCall(requestBuilder.build()).execute()) {
         if (response.isSuccessful()) {
+          processInfoResponseHeaders(response);
           fallback = !processInfoResponse(response.body().string());
         }
       } catch (Throwable error) {
@@ -216,6 +229,10 @@ private String probeTracesEndpoint(String[] endpoints) {
     return V3_ENDPOINT;
   }
 
+  private void processInfoResponseHeaders(Response response) {
+    ContainerInfo.get().setContainerTagsHash(response.header(DATADOG_CONTAINER_TAGS_HASH));
+  }
+
   @SuppressWarnings("unchecked")
   private boolean processInfoResponse(String response) {
     try {
@@ -287,6 +304,18 @@ private boolean processInfoResponse(String response) {
             null != canDrop
                 && ("true".equalsIgnoreCase(String.valueOf(canDrop))
                     || Boolean.TRUE.equals(canDrop));
+
+        Object peer_tags = map.get("peer_tags");
+        peerTags =
+            peer_tags instanceof List
+                ? unmodifiableSet(new HashSet<>((List<String>) peer_tags))
+                : emptySet();
+
+        Object span_kinds = map.get("span_kinds_stats_computed");
+        spanKindsToComputedStats =
+            span_kinds instanceof List
+                ? unmodifiableSet(new HashSet<>((List<String>) span_kinds))
+                : emptySet();
       }
       try {
         state = Strings.sha256(response);
@@ -344,6 +373,14 @@ public boolean supportsLongRunning() {
     return supportsLongRunning;
   }
 
+  public Set<String> peerTags() {
+    return peerTags;
+  }
+
+  public Set<String> spanKindsToComputedStats() {
+    return spanKindsToComputedStats;
+  }
+
   public String getMetricsEndpoint() {
     return metricsEndpoint;
   }

communication/src/main/java/datadog/communication/ddagent/SharedCommunicationObjects.java

@@ -30,7 +30,7 @@ public class SharedCommunicationObjects {
   public OkHttpClient okHttpClient;
   public HttpUrl agentUrl;
   public Monitoring monitoring;
-  private DDAgentFeaturesDiscovery featuresDiscovery;
+  private volatile DDAgentFeaturesDiscovery featuresDiscovery;
   private ConfigurationPoller configurationPoller;
 
   public SharedCommunicationObjects() {
@@ -139,28 +139,34 @@ public void setFeaturesDiscovery(DDAgentFeaturesDiscovery featuresDiscovery) {
   }
 
   public DDAgentFeaturesDiscovery featuresDiscovery(Config config) {
-    if (featuresDiscovery == null) {
-      createRemaining(config);
-      featuresDiscovery =
-          new DDAgentFeaturesDiscovery(
-              okHttpClient,
-              monitoring,
-              agentUrl,
-              config.isTraceAgentV05Enabled(),
-              config.isTracerMetricsEnabled());
-
-      if (paused) {
-        // defer remote discovery until remote I/O is allowed
-      } else {
-        if (AGENT_THREAD_GROUP.equals(Thread.currentThread().getThreadGroup())) {
-          featuresDiscovery.discover(); // safe to run on same thread
-        } else {
-          // avoid performing blocking I/O operation on application thread
-          AgentTaskScheduler.INSTANCE.execute(featuresDiscovery::discover);
+    DDAgentFeaturesDiscovery ret = featuresDiscovery;
+    if (ret == null) {
+      synchronized (this) {
+        if (featuresDiscovery == null) {
+          createRemaining(config);
+          ret =
+              new DDAgentFeaturesDiscovery(
+                  okHttpClient,
+                  monitoring,
+                  agentUrl,
+                  config.isTraceAgentV05Enabled(),
+                  config.isTracerMetricsEnabled());
+
+          if (paused) {
+            // defer remote discovery until remote I/O is allowed
+          } else {
+            if (AGENT_THREAD_GROUP.equals(Thread.currentThread().getThreadGroup())) {
+              ret.discover(); // safe to run on same thread
+            } else {
+              // avoid performing blocking I/O operation on application thread
+              AgentTaskScheduler.INSTANCE.execute(ret::discoverIfOutdated);
+            }
+          }
+          featuresDiscovery = ret;
         }
       }
     }
-    return featuresDiscovery;
+    return ret;
   }
 
   private static final class FixedConfigUrlSupplier implements Supplier<String> {

communication/src/main/java/datadog/communication/http/OkHttpUtils.java

@@ -43,8 +43,9 @@ public final class OkHttpUtils {
   private static final String DATADOG_META_LANG_INTERPRETER = "Datadog-Meta-Lang-Interpreter";
   private static final String DATADOG_META_LANG_INTERPRETER_VENDOR =
       "Datadog-Meta-Lang-Interpreter-Vendor";
-  private static final String DATADOG_CONTAINER_ID = "Datadog-Container-ID";
+  public static final String DATADOG_CONTAINER_ID = "Datadog-Container-ID";
   private static final String DATADOG_ENTITY_ID = "Datadog-Entity-ID";
+  public static final String DATADOG_CONTAINER_TAGS_HASH = "Datadog-Container-Tags-Hash";
 
   private static final String DD_API_KEY = "DD-API-KEY";
 

communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy

@@ -1,9 +1,11 @@
 package datadog.communication.ddagent
 
+import datadog.common.container.ContainerInfo
 import datadog.communication.monitor.Monitoring
 import datadog.trace.test.util.DDSpecification
 import datadog.trace.util.Strings
 import okhttp3.Call
+import okhttp3.Headers
 import okhttp3.HttpUrl
 import okhttp3.MediaType
 import okhttp3.OkHttpClient
@@ -19,6 +21,8 @@ import java.nio.file.Paths
 import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V01_DATASTREAMS_ENDPOINT
 import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V6_METRICS_ENDPOINT
 import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V7_CONFIG_ENDPOINT
+import static datadog.communication.http.OkHttpUtils.DATADOG_CONTAINER_ID
+import static datadog.communication.http.OkHttpUtils.DATADOG_CONTAINER_TAGS_HASH
 
 class DDAgentFeaturesDiscoveryTest extends DDSpecification {
 
@@ -30,6 +34,8 @@ class DDAgentFeaturesDiscoveryTest extends DDSpecification {
 
   static final String INFO_RESPONSE = loadJsonFile("agent-info.json")
   static final String INFO_STATE = Strings.sha256(INFO_RESPONSE)
+  static final String INFO_WITH_PEER_TAG_BACK_PROPAGATION_RESPONSE = loadJsonFile("agent-info-with-peer-tag-back-propagation.json")
+  static final String INFO_WITH_PEER_TAG_BACK_PROPAGATION_STATE = Strings.sha256(INFO_WITH_PEER_TAG_BACK_PROPAGATION_RESPONSE)
   static final String INFO_WITH_CLIENT_DROPPING_RESPONSE = loadJsonFile("agent-info-with-client-dropping.json")
   static final String INFO_WITH_CLIENT_DROPPING_STATE = Strings.sha256(INFO_WITH_CLIENT_DROPPING_RESPONSE)
   static final String INFO_WITHOUT_METRICS_RESPONSE = loadJsonFile("agent-info-without-metrics.json")
@@ -424,13 +430,77 @@ class DDAgentFeaturesDiscoveryTest extends DDSpecification {
     !features.supportsContentEncodingHeadersWithEvpProxy()
   }
 
-  def infoResponse(Request request, String json) {
+  def "test parse /info response with peer tag back propagation"() {
+    setup:
+    OkHttpClient client = Mock(OkHttpClient)
+    DDAgentFeaturesDiscovery features = new DDAgentFeaturesDiscovery(client, monitoring, agentUrl, true, true)
+
+    when: "/info available"
+    features.discover()
+
+    then:
+    1 * client.newCall(_) >> { Request request -> infoResponse(request, INFO_RESPONSE) }
+
+    when: "/info available with peer tag back propagation"
+    features.discover()
+
+    then:
+    1 * client.newCall(_) >> { Request request -> infoResponse(request, INFO_WITH_PEER_TAG_BACK_PROPAGATION_RESPONSE) }
+    features.state() == INFO_WITH_PEER_TAG_BACK_PROPAGATION_STATE
+    features.supportsDropping()
+    features.peerTags().containsAll(
+      "_dd.base_service",
+      "active_record.db.vendor",
+      "amqp.destination",
+      "amqp.exchange",
+      "amqp.queue",
+      "grpc.host",
+      "hostname",
+      "http.host",
+      "http.server_name",
+      "streamname",
+      "tablename",
+      "topicname"
+      )
+    features.spanKindsToComputedStats().containsAll(
+      "client",
+      "consumer",
+      "producer",
+      "server"
+      )
+  }
+
+  def "should send container id as header on the info request and parse the hash in the response"() {
+    setup:
+    OkHttpClient client = Mock(OkHttpClient)
+    DDAgentFeaturesDiscovery features = new DDAgentFeaturesDiscovery(client, monitoring, agentUrl, true, true)
+    def oldContainerId = ContainerInfo.get().getContainerId()
+    def oldContainerTagsHash = ContainerInfo.get().getContainerTagsHash()
+    ContainerInfo.get().setContainerId("test")
+
+    when: "/info requested"
+    features.discover()
+
+    then:
+    1 * client.newCall(_) >> { Request request ->
+      assert request.header(DATADOG_CONTAINER_ID) == "test"
+      infoResponse(request, INFO_RESPONSE, Headers.of(DATADOG_CONTAINER_TAGS_HASH, "test-hash"))
+    }
+    and:
+    assert ContainerInfo.get().getContainerTagsHash() == "test-hash"
+    cleanup:
+    ContainerInfo.get().setContainerId(oldContainerId)
+    ContainerInfo.get().setContainerTagsHash(oldContainerTagsHash)
+  }
+
+  def infoResponse(Request request, String json, Headers headers = new Headers.Builder().build()) {
     return Mock(Call) {
       it.execute() >> new Response.Builder()
         .code(200)
         .request(request)
         .protocol(Protocol.HTTP_1_1)
         .message("")
+        .headers(headers)
         .body(ResponseBody.create(MediaType.get("application/json"), json))
         .build()
     }