Merge branch 'master' into kr-igor/kafka-lag-spark-streaming

Author: kr-igor

.circleci/config.continue.yml.j2

@@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
-default_system_tests_commit: &default_system_tests_commit 20b61e9be1e8c922804f8e5ddc80ba4a1f4cdd5b
+default_system_tests_commit: &default_system_tests_commit 1c9542783eedfed1b995d976963fbeb14ba772b9
 
 parameters:
   nightly:

.github/CODEOWNERS

@@ -26,7 +26,8 @@ dd-java-agent/instrumentation/karate/      @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/scalatest/   @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/selenium/    @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/testng/      @DataDog/ci-app-libraries-java
-dd-java-agent/instrumentation/gradle/      @DataDog/ci-app-libraries-java
+dd-java-agent/instrumentation/gradle-3.0/  @DataDog/ci-app-libraries-java
+dd-java-agent/instrumentation/gradle-8.3/  @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/maven-3.2.1/ @DataDog/ci-app-libraries-java
 dd-smoke-tests/gradle/                     @DataDog/ci-app-libraries-java
 dd-smoke-tests/maven/                      @DataDog/ci-app-libraries-java

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/utils/ProcessHierarchyUtils.java

@@ -38,7 +38,11 @@ private static boolean isMavenParent() {
   }
 
   private static boolean isGradleDaemon() {
-    return ClassLoader.getSystemClassLoader().getResource("org/gradle/launcher/daemon/") != null;
+    return ClassLoader.getSystemClassLoader()
+                .getResource("org/gradle/launcher/daemon/bootstrap/GradleDaemon.class")
+            != null
+        // double-check this is not a Gradle Worker
+        && System.getProperties().getProperty("org.gradle.internal.worker.tmpdir") == null;
   }
 
   public static long getParentSessionId() {

dd-java-agent/agent-debugger/debugger-bootstrap/src/main/java/datadog/trace/bootstrap/debugger/util/Redaction.java

@@ -110,10 +110,15 @@ public class Redaction {
   private static List<String> redactedPackages;
 
   static {
+    initKeywords();
+  }
+
+  static void initKeywords() {
     /*
      * based on sentry list: https://github.com/getsentry/sentry-python/blob/fefb454287b771ac31db4e30fa459d9be2f977b8/sentry_sdk/scrubber.py#L17-L58
      */
     KEYWORDS.addAll(PREDEFINED_KEYWORDS);
+    KEYWORDS.removeAll(Config.get().getDebuggerRedactionExcludedIdentifiers());
   }
 
   public static void addUserDefinedKeywords(Config config) {

dd-java-agent/agent-debugger/debugger-bootstrap/src/test/java/datadog/trace/bootstrap/debugger/util/RedactionTest.java

@@ -4,6 +4,9 @@
 
 import datadog.trace.api.Config;
 import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
 import org.junit.jupiter.api.Test;
 
 class RedactionTest {
@@ -48,6 +51,20 @@ public void userDefinedTypes() {
     }
   }
 
+  @Test
+  public void exclusions() {
+    Config config = Config.get();
+    setFieldInConfig(
+        config, "debuggerRedactionExcludedIdentifiers", new HashSet<>(Arrays.asList("password")));
+    Redaction.initKeywords();
+    try {
+      assertFalse(Redaction.isRedactedKeyword("password"));
+    } finally {
+      setFieldInConfig(config, "debuggerRedactionExcludedIdentifiers", Collections.emptySet());
+      Redaction.initKeywords();
+    }
+  }
+
   private static void setFieldInConfig(Config config, String fieldName, Object value) {
     try {
       Field field = config.getClass().getDeclaredField(fieldName);

dd-java-agent/agent-debugger/debugger-el/src/main/java/com/datadog/debugger/el/expressions/ComparisonOperator.java

@@ -21,11 +21,52 @@ public Boolean apply(Value<?> left, Value<?> right) {
         }
         return compare(leftNumber, rightNumber) == 0;
       }
+      if (left.getValue() instanceof Enum || right.getValue() instanceof Enum) {
+        return applyEqualityForEnum(left, right);
+      }
       if (left.getValue().getClass() == right.getValue().getClass()) {
         return Objects.equals(left.getValue(), right.getValue());
       }
       return Boolean.FALSE;
     }
+
+    private Boolean applyEqualityForEnum(Value<?> left, Value<?> right) {
+      if (left.getValue() instanceof Enum && right instanceof StringValue) {
+        return doApplyEqualityForEnum(left, right);
+      }
+      if (left instanceof StringValue && right.getValue() instanceof Enum) {
+        return doApplyEqualityForEnum(right, left);
+      }
+      throw new EvaluationException(
+          "Equality operator is not supported for the given types: "
+              + left.getValue().getClass().getName()
+              + " and "
+              + right.getValue().getClass().getName(),
+          null);
+    }
+
+    private Boolean doApplyEqualityForEnum(Value<?> enumExpr, Value<?> enumValueExpr) {
+      Enum<?> enumValue = (Enum<?>) enumExpr.getValue();
+      String enumValueStr = (String) enumValueExpr.getValue();
+      Class<? extends Enum> enumClass = enumValue.getClass();
+      Enum[] enumConstants = enumClass.getEnumConstants();
+      for (Enum<?> enumConstant : enumConstants) {
+        // Check if string constant as value expression matches for enum constant
+        // the endsWith allow to match either:
+        // - the full enum constant name (com.datadog.debugger.MyEnum.ONE)
+        // - the simple name with enum class name (MyEnum.ONE)
+        // - the simple name (ONE)
+        // The second check against enumValue is to ensure the instance filtered based on the
+        // name is still correct because the name can partially match (CLOSE in OPENCLOSE)
+        // with an enum defined like (OPEN, CLOSE, OPENCLOSE)
+        if (enumValueStr.endsWith(enumConstant.name())) {
+          if (enumValue.equals(enumConstant)) {
+            return Boolean.TRUE;
+          }
+        }
+      }
+      return Boolean.FALSE;
+    }
   },
   GE(">=") {
     @Override

dd-java-agent/agent-debugger/debugger-el/src/test/java/com/datadog/debugger/el/expressions/ComparisonExpressionTest.java

@@ -16,6 +16,7 @@
 import com.datadog.debugger.el.values.StringValue;
 import datadog.trace.bootstrap.debugger.el.ValueReferenceResolver;
 import java.math.BigDecimal;
+import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.Random;
 import java.util.stream.Stream;
@@ -164,7 +165,37 @@ private static Stream<Arguments> expressions() {
             new StringValue("java.lang.Double"),
             INSTANCEOF,
             true,
-            "1.0 instanceof \"java.lang.Double\""));
+            "1.0 instanceof \"java.lang.Double\""),
+        Arguments.of(
+            new ObjectValue(StandardOpenOption.READ),
+            new StringValue("READ"),
+            EQ,
+            true,
+            "java.nio.file.StandardOpenOption == \"READ\""),
+        Arguments.of(
+            new ObjectValue(StandardOpenOption.READ),
+            new StringValue("StandardOpenOption.READ"),
+            EQ,
+            true,
+            "java.nio.file.StandardOpenOption == \"StandardOpenOption.READ\""),
+        Arguments.of(
+            new ObjectValue(StandardOpenOption.READ),
+            new StringValue("java.nio.file.StandardOpenOption.READ"),
+            EQ,
+            true,
+            "java.nio.file.StandardOpenOption == \"java.nio.file.StandardOpenOption.READ\""),
+        Arguments.of(
+            new ObjectValue(StandardOpenOption.CREATE),
+            new StringValue("READ"),
+            EQ,
+            false,
+            "java.nio.file.StandardOpenOption == \"READ\""),
+        Arguments.of(
+            new StringValue("READ"),
+            new ObjectValue(StandardOpenOption.READ),
+            EQ,
+            true,
+            "\"READ\" == java.nio.file.StandardOpenOption"));
   }
 
   @ParameterizedTest

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/probe/LogProbe.java

@@ -553,9 +553,6 @@ public void commit(CapturedContext lineContext, int line) {
     if (status.shouldSend()) {
       snapshot.setTraceId(lineContext.getTraceId());
       snapshot.setSpanId(lineContext.getSpanId());
-      if (isCaptureSnapshot()) {
-        snapshot.addLine(lineContext, line);
-      }
       snapshot.setMessage(status.getMessage());
       shouldCommit = true;
     }
@@ -564,9 +561,12 @@ public void commit(CapturedContext lineContext, int line) {
       shouldCommit = true;
     }
     if (shouldCommit) {
-      // freeze context just before commit because line probes have only one context
-      Duration timeout = Duration.of(Config.get().getDebuggerCaptureTimeout(), ChronoUnit.MILLIS);
-      lineContext.freeze(new TimeoutChecker(timeout));
+      if (isCaptureSnapshot()) {
+        // freeze context just before commit because line probes have only one context
+        Duration timeout = Duration.of(Config.get().getDebuggerCaptureTimeout(), ChronoUnit.MILLIS);
+        lineContext.freeze(new TimeoutChecker(timeout));
+        snapshot.addLine(lineContext, line);
+      }
       commitSnapshot(snapshot, sink);
       return;
     }

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/sink/SymbolSink.java

@@ -3,14 +3,12 @@
 import com.datadog.debugger.symbol.Scope;
 import com.datadog.debugger.symbol.ServiceVersion;
 import com.datadog.debugger.uploader.BatchUploader;
-import com.datadog.debugger.util.ExceptionHelper;
 import com.datadog.debugger.util.MoshiHelper;
 import com.squareup.moshi.JsonAdapter;
 import datadog.trace.api.Config;
 import datadog.trace.util.TagsHelper;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.BlockingQueue;
@@ -21,7 +19,7 @@
 public class SymbolSink {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(SymbolSink.class);
-  private static final int CAPACITY = 1024;
+  static final int CAPACITY = 1024;
   private static final JsonAdapter<ServiceVersion> SERVICE_VERSION_ADAPTER =
       MoshiHelper.createMoshiSymbol().adapter(ServiceVersion.class);
   private static final String EVENT_FORMAT =
@@ -36,7 +34,7 @@ public class SymbolSink {
   private final String version;
   private final BatchUploader symbolUploader;
   private final BatchUploader.MultiPartContent event;
-  private final BlockingQueue<ServiceVersion> scopes = new ArrayBlockingQueue<>(CAPACITY);
+  private final BlockingQueue<Scope> scopes = new ArrayBlockingQueue<>(CAPACITY);
   private final Stats stats = new Stats();
 
   public SymbolSink(Config config) {
@@ -59,37 +57,47 @@ public void stop() {
     symbolUploader.shutdown();
   }
 
-  public boolean addScope(Scope jarScope) {
-    ServiceVersion serviceVersion =
-        new ServiceVersion(serviceName, env, version, "JAVA", Collections.singletonList(jarScope));
-    return scopes.offer(serviceVersion);
+  public void addScope(Scope jarScope) {
+    boolean added = scopes.offer(jarScope);
+    int retries = 10;
+    while (!added) {
+      // Q is full, flushing synchronously
+      flush();
+      added = scopes.offer(jarScope);
+      retries--;
+      if (retries < 0) {
+        throw new IllegalStateException("Scope cannot be enqueued after 10 retries" + jarScope);
+      }
+    }
   }
 
   public void flush() {
     if (scopes.isEmpty()) {
       return;
     }
-    List<ServiceVersion> scopesToSerialize = new ArrayList<>();
+    List<Scope> scopesToSerialize = new ArrayList<>();
+    // ArrayBlockingQueue makes drainTo atomic, so it is safe to call flush from different and
+    // concurrent threads
     scopes.drainTo(scopesToSerialize);
-    LOGGER.debug("Sending {} scopes", scopesToSerialize.size());
-    for (ServiceVersion serviceVersion : scopesToSerialize) {
-      try {
-        String json = SERVICE_VERSION_ADAPTER.toJson(serviceVersion);
-        LOGGER.debug(
-            "Sending scope: {}, size={}",
-            serviceVersion.getScopes().get(0).getName(),
-            json.length());
-        List<Scope> classScopes = serviceVersion.getScopes().get(0).getScopes();
-        int classScopeCount = classScopes != null ? classScopes.size() : 0;
-        stats.updateStats(classScopeCount, json.length());
-        symbolUploader.uploadAsMultipart(
-            "",
-            event,
-            new BatchUploader.MultiPartContent(
-                json.getBytes(StandardCharsets.UTF_8), "file", "file.json"));
-      } catch (Exception e) {
-        ExceptionHelper.logException(LOGGER, e, "Error during scope serialization:");
-      }
+    // concurrent calls to flush can result in empty scope to send, we don't want to send empty
+    if (scopesToSerialize.isEmpty()) {
+      return;
+    }
+    String json =
+        SERVICE_VERSION_ADAPTER.toJson(
+            new ServiceVersion(serviceName, env, version, "JAVA", scopesToSerialize));
+    LOGGER.debug("Sending {} jar scopes size={}", scopesToSerialize.size(), json.length());
+    updateStats(scopesToSerialize, json);
+    symbolUploader.uploadAsMultipart(
+        "",
+        event,
+        new BatchUploader.MultiPartContent(
+            json.getBytes(StandardCharsets.UTF_8), "file", "file.json"));
+  }
+
+  private void updateStats(List<Scope> scopesToSerialize, String json) {
+    for (Scope scope : scopesToSerialize) {
+      stats.updateStats(scope.getScopes() != null ? scope.getScopes().size() : 0, json.length());
     }
   }
 

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/uploader/BatchUploader.java

@@ -174,7 +174,8 @@ private void doUpload(Runnable makeRequest) {
         debuggerMetrics.count("batch.uploaded", 1);
       } else {
         debuggerMetrics.count("request.queue.full", 1);
-        log.warn("Cannot upload batch data: too many enqueued requests!");
+        ratelimitedLogger.warn(
+            "Cannot upload batch data to {}: too many enqueued requests!", urlBase);
       }
     } catch (Exception ex) {
       debuggerMetrics.count("batch.upload.error", 1);