Probe for existence of IBMSASL or ACCP security providers.

If they exist then complete registration of other products on a different thread after premain to avoid eagerly loading 'java.util.logging' before main. A longer delay is added when we know a custom logging manager will be installed.

Includes a fix for a race condition on IBM Java 8 where resuming Appsec/IAST could affect initial remote discovery.

Author: mcculls

communication/src/main/java/datadog/communication/ddagent/SharedCommunicationObjects.java

@@ -11,6 +11,7 @@
 import datadog.remoteconfig.DefaultConfigurationPoller;
 import datadog.trace.api.Config;
 import datadog.trace.util.AgentTaskScheduler;
+import java.security.Security;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
@@ -59,6 +60,7 @@ public void createRemaining(Config config) {
     }
   }
 
+  /** Registers a callback to be called when remote communications resume. */
   public void whenReady(Runnable callback) {
     if (paused) {
       synchronized (pausedComponents) {
@@ -71,8 +73,15 @@ public void whenReady(Runnable callback) {
     callback.run(); // not paused, run immediately
   }
 
+  /** Resumes remote communications including any paused callbacks. */
   public void resume() {
     paused = false;
+    // attempt discovery first to avoid potential race condition on IBM Java8
+    if (null != featuresDiscovery) {
+      featuresDiscovery.discoverIfOutdated();
+    } else {
+      Security.getProviders(); // fallback to preloading provider extensions
+    }
     synchronized (pausedComponents) {
       for (Runnable callback : pausedComponents) {
         try {

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java

@@ -1,7 +1,6 @@
 package datadog.trace.bootstrap;
 
 import static datadog.trace.api.ConfigDefaults.DEFAULT_STARTUP_LOGS_ENABLED;
-import static datadog.trace.api.Platform.getRuntimeVendor;
 import static datadog.trace.api.Platform.isJavaVersionAtLeast;
 import static datadog.trace.api.Platform.isOracleJDK8;
 import static datadog.trace.bootstrap.Library.WILDFLY;
@@ -329,7 +328,7 @@ public void run() {
       if (appUsingCustomJMXBuilder) {
         log.debug("Custom JMX builder detected. Delaying JMXFetch initialization.");
         registerMBeanServerBuilderCallback(new StartJmxCallback(jmxStartDelay));
-        // one minute fail-safe in case nothing touches JMX and and callback isn't triggered
+        // one minute fail-safe in case nothing touches JMX and callback isn't triggered
         scheduleJmxStart(60 + jmxStartDelay);
       } else if (appUsingCustomLogManager) {
         log.debug("Custom logger detected. Delaying JMXFetch initialization.");
@@ -339,20 +338,31 @@ public void run() {
       }
     }
 
-    boolean delayOkHttp = appUsingCustomLogManager && okHttpMayIndirectlyLoadJUL();
-
     /*
      * Similar thing happens with DatadogTracer on (at least) zulu-8 because it uses OkHttp which indirectly loads JFR
      * events which in turn loads LogManager. This is not a problem on newer JDKs because there JFR uses different
      * logging facility. Likewise on IBM JDKs OkHttp may indirectly load 'IBMSASL' which in turn loads LogManager.
      */
+    boolean delayOkHttp = !ciVisibilityEnabled && okHttpMayIndirectlyLoadJUL();
+    boolean waitForJUL = appUsingCustomLogManager && delayOkHttp;
+    int okHttpDelayMillis;
+    if (waitForJUL) {
+      okHttpDelayMillis = 1_000;
+    } else if (delayOkHttp) {
+      okHttpDelayMillis = 100;
+    } else {
+      okHttpDelayMillis = 0;
+    }
+
     InstallDatadogTracerCallback installDatadogTracerCallback =
-        new InstallDatadogTracerCallback(initTelemetry, inst, delayOkHttp);
-    if (delayOkHttp) {
+        new InstallDatadogTracerCallback(initTelemetry, inst, okHttpDelayMillis);
+    if (waitForJUL) {
       log.debug("Custom logger detected. Delaying Datadog Tracer initialization.");
       registerLogManagerCallback(installDatadogTracerCallback);
+    } else if (okHttpDelayMillis > 0) {
+      installDatadogTracerCallback.run(); // complete on different thread (after premain)
     } else {
-      installDatadogTracerCallback.execute();
+      installDatadogTracerCallback.execute(); // complete on primordial thread in premain
     }
 
     /*
@@ -362,7 +372,7 @@ public void run() {
     if (profilingEnabled && !isOracleJDK8()) {
       StaticEventLogger.begin("Profiling");
 
-      if (delayOkHttp) {
+      if (waitForJUL) {
         log.debug("Custom logger detected. Delaying Profiling initialization.");
         registerLogManagerCallback(new StartProfilingAgentCallback(inst));
       } else {
@@ -499,18 +509,18 @@ protected static class InstallDatadogTracerCallback extends ClassLoadCallBack {
     private final Instrumentation instrumentation;
     private final Object sco;
     private final Class<?> scoClass;
-    private final boolean delayOkHttp;
+    private final int okHttpDelayMillis;
 
     public InstallDatadogTracerCallback(
         InitializationTelemetry initTelemetry,
         Instrumentation instrumentation,
-        boolean delayOkHttp) {
-      this.delayOkHttp = delayOkHttp;
+        int okHttpDelayMillis) {
+      this.okHttpDelayMillis = okHttpDelayMillis;
       this.instrumentation = instrumentation;
       try {
         scoClass =
             AGENT_CLASSLOADER.loadClass("datadog.communication.ddagent.SharedCommunicationObjects");
-        sco = scoClass.getConstructor(boolean.class).newInstance(delayOkHttp);
+        sco = scoClass.getConstructor(boolean.class).newInstance(okHttpDelayMillis > 0);
       } catch (ClassNotFoundException
           | NoSuchMethodException
           | InstantiationException
@@ -530,7 +540,7 @@ public AgentThread agentThread() {
 
     @Override
     public void execute() {
-      if (delayOkHttp) {
+      if (okHttpDelayMillis > 0) {
         resumeRemoteComponents();
       }
 
@@ -550,7 +560,7 @@ private void resumeRemoteComponents() {
       try {
         // remote components were paused for custom log-manager/jmx-builder
         // add small delay before resuming remote I/O to help stabilization
-        Thread.sleep(1_000);
+        Thread.sleep(okHttpDelayMillis);
         scoClass.getMethod("resume").invoke(sco);
       } catch (InterruptedException ignore) {
       } catch (Throwable e) {
@@ -1339,15 +1349,25 @@ private static String ddGetEnv(final String sysProp) {
   }
 
   private static boolean okHttpMayIndirectlyLoadJUL() {
-    if ("IBM Corporation".equals(getRuntimeVendor())) {
-      return true; // IBM JDKs ship with 'IBMSASL' which will load JUL when OkHttp accesses TLS
+    if (isIBMSASLInstalled() || isACCPInstalled()) {
+      return true; // 'IBMSASL' and 'ACCP' crypto providers can load JUL when OkHttp accesses TLS
     }
     if (isJavaVersionAtLeast(9)) {
       return false; // JDKs since 9 have reworked JFR to use a different logging facility, not JUL
     }
     return isJFRSupported(); // assume OkHttp will indirectly load JUL via its JFR events
   }
 
+  private static boolean isIBMSASLInstalled() {
+    return ClassLoader.getSystemResource("com/ibm/security/sasl/IBMSASL.class") != null;
+  }
+
+  private static boolean isACCPInstalled() {
+    return ClassLoader.getSystemResource(
+            "com/amazon/corretto/crypto/provider/AmazonCorrettoCryptoProvider.class")
+        != null;
+  }
+
   private static boolean isJFRSupported() {
     // FIXME: this is quite a hack because there maybe jfr classes on classpath somehow that have
     // nothing to do with JDK - but this should be safe because only thing this does is to delay