Rework crash-tracking to allow auto-injection of JVM arguments for error log capture

Author: jbachorik

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java

@@ -3,6 +3,7 @@
 import static datadog.environment.JavaVirtualMachine.isJavaVersionAtLeast;
 import static datadog.environment.JavaVirtualMachine.isOracleJDK8;
 import static datadog.trace.api.ConfigDefaults.DEFAULT_STARTUP_LOGS_ENABLED;
+import static datadog.trace.api.telemetry.LogCollector.SEND_TELEMETRY;
 import static datadog.trace.bootstrap.Library.WILDFLY;
 import static datadog.trace.bootstrap.Library.detectLibraries;
 import static datadog.trace.util.AgentThreadFactory.AgentThread.JMX_STARTUP;
@@ -21,6 +22,7 @@
 import datadog.trace.api.appsec.AppSecEventTracker;
 import datadog.trace.api.config.AppSecConfig;
 import datadog.trace.api.config.CiVisibilityConfig;
+import datadog.trace.api.config.CrashTrackingConfig;
 import datadog.trace.api.config.CwsConfig;
 import datadog.trace.api.config.DebuggerConfig;
 import datadog.trace.api.config.GeneralConfig;
@@ -38,6 +40,7 @@
 import datadog.trace.api.profiling.ProfilingEnablement;
 import datadog.trace.api.scopemanager.ScopeListener;
 import datadog.trace.bootstrap.benchmark.StaticEventLogger;
+import datadog.trace.bootstrap.config.provider.ConfigProvider;
 import datadog.trace.bootstrap.config.provider.StableConfigSource;
 import datadog.trace.bootstrap.instrumentation.api.AgentTracer;
 import datadog.trace.bootstrap.instrumentation.api.AgentTracer.TracerAPI;
@@ -54,6 +57,7 @@
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.security.CodeSource;
+import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -95,6 +99,9 @@ private enum AgentFeature {
     TRACING(TraceInstrumentationConfig.TRACE_ENABLED, true),
     JMXFETCH(JmxFetchConfig.JMX_FETCH_ENABLED, true),
     STARTUP_LOGS(GeneralConfig.STARTUP_LOGS_ENABLED, DEFAULT_STARTUP_LOGS_ENABLED),
+    CRASH_TRACKING(
+        CrashTrackingConfig.CRASH_TRACKING_ENABLED,
+        CrashTrackingConfig.CRASH_TRACKING_ENABLED_DEFAULT),
     PROFILING(ProfilingConfig.PROFILING_ENABLED, false),
     APPSEC(AppSecConfig.APPSEC_ENABLED, false),
     IAST(IastConfig.IAST_ENABLED, false),
@@ -146,9 +153,11 @@ public boolean isEnabledByDefault() {
   private static ClassLoader AGENT_CLASSLOADER = null;
 
   private static volatile Runnable PROFILER_INIT_AFTER_JMX = null;
+  private static volatile Runnable CRASHTRACKER_INIT_AFTER_JMX = null;
 
   private static boolean jmxFetchEnabled = true;
   private static boolean profilingEnabled = false;
+  private static boolean crashTrackingEnabled = false;
   private static boolean appSecEnabled;
   private static boolean appSecFullyDisabled;
   private static boolean remoteConfigEnabled = true;
@@ -276,6 +285,7 @@ public static void start(
 
     jmxFetchEnabled = isFeatureEnabled(AgentFeature.JMXFETCH);
     profilingEnabled = isFeatureEnabled(AgentFeature.PROFILING);
+    crashTrackingEnabled = isFeatureEnabled(AgentFeature.CRASH_TRACKING);
     usmEnabled = isFeatureEnabled(AgentFeature.USM);
     appSecEnabled = isFeatureEnabled(AgentFeature.APPSEC);
     appSecFullyDisabled = isFullyDisabled(AgentFeature.APPSEC);
@@ -303,13 +313,7 @@ public static void start(
         // Profiling can not run early on Oracle JDK 8 because it will cause JFR initialization
         // deadlock.
         // Oracle JDK 8 JFR controller requires JMX so register an 'after-jmx-initialized' callback.
-        PROFILER_INIT_AFTER_JMX =
-            new Runnable() {
-              @Override
-              public void run() {
-                startProfilingAgent(false, inst);
-              }
-            };
+        PROFILER_INIT_AFTER_JMX = () -> startProfilingAgent(false, inst);
       }
     }
 
@@ -322,6 +326,12 @@ public void run() {
      * when it will happen after the class transformers were added.
      */
     AgentTaskScheduler.initialize();
+
+    // We need to run the crashtracking initialization after all the config has been resolved and
+    // task scheduler initialized
+    if (crashTrackingEnabled) {
+      startCrashTracking();
+    }
     startDatadogAgent(initTelemetry, inst);
 
     final EnumSet<Library> libraries = detectLibraries(log);
@@ -715,6 +725,26 @@ private static synchronized void installDatadogTracer(
     StaticEventLogger.end("GlobalTracer");
   }
 
+  private static void startCrashTracking() {
+    if (Platform.isJavaVersionAtLeast(9)) {
+      // it is safe to initialize crashtracking early
+      // since it can take 100ms+ to initialize the native library we will defer the initialization
+      // ... unless we request early start with the debug config flag
+      if (ConfigProvider.getInstance()
+          .getBoolean(
+              CrashTrackingConfig.CRASH_TRACKING_START_EARLY,
+              CrashTrackingConfig.CRASH_TRACKING_START_EARLY_DEFAULT)) {
+        initializeCrashTrackingDefault();
+      } else {
+        AgentTaskScheduler.INSTANCE.execute(Agent::initializeCrashTrackingDefault);
+      }
+    } else {
+      // for Java 8 we are relying on JMX to give us the process PID
+      // we need to delay the crash tracking initialization until JMX is available
+      CRASHTRACKER_INIT_AFTER_JMX = Agent::initializeDelayedCrashTracking;
+    }
+  }
+
   private static void scheduleJmxStart(final int jmxStartDelay) {
     if (jmxStartDelay > 0) {
       AgentTaskScheduler.INSTANCE.scheduleWithJitter(
@@ -738,25 +768,33 @@ private static synchronized void startJmx() {
     if (jmxStarting.getAndSet(true)) {
       return; // another thread is already in startJmx
     }
-    // error tracking initialization relies on JMX being available
-    initializeErrorTracking();
     if (jmxFetchEnabled) {
       startJmxFetch();
     }
     initializeJmxSystemAccessProvider(AGENT_CLASSLOADER);
+    if (crashTrackingEnabled && CRASHTRACKER_INIT_AFTER_JMX != null) {
+      try {
+        CRASHTRACKER_INIT_AFTER_JMX.run();
+      } finally {
+        CRASHTRACKER_INIT_AFTER_JMX = null;
+      }
+    }
     if (profilingEnabled) {
       registerDeadlockDetectionEvent();
       registerSmapEntryEvent();
       if (PROFILER_INIT_AFTER_JMX != null) {
-        if (getJmxStartDelay() == 0) {
-          log.debug("Waiting for profiler initialization");
-          AgentTaskScheduler.INSTANCE.scheduleWithJitter(
-              PROFILER_INIT_AFTER_JMX, 500, TimeUnit.MILLISECONDS);
-        } else {
-          log.debug("Initializing profiler");
-          PROFILER_INIT_AFTER_JMX.run();
+        try {
+          if (getJmxStartDelay() == 0) {
+            log.debug("Waiting for profiler initialization");
+            AgentTaskScheduler.INSTANCE.scheduleWithJitter(
+                PROFILER_INIT_AFTER_JMX, 500, TimeUnit.MILLISECONDS);
+          } else {
+            log.debug("Initializing profiler");
+            PROFILER_INIT_AFTER_JMX.run();
+          }
+        } finally {
+          PROFILER_INIT_AFTER_JMX = null;
         }
-        PROFILER_INIT_AFTER_JMX = null;
       }
     }
   }
@@ -998,16 +1036,55 @@ private static void stopTelemetry() {
     }
   }
 
-  private static void initializeErrorTracking() {
+  private static void initializeDelayedCrashTracking() {
+    initializeCrashTracking(true, isCrashTrackingAutoconfigEnabled());
+  }
+
+  private static void initializeDelayedCrashTrackingOnlyJmx() {
+    initializeCrashTracking(true, false);
+  }
+
+  private static void initializeCrashTrackingDefault() {
+    initializeCrashTracking(false, isCrashTrackingAutoconfigEnabled());
+  }
+
+  private static boolean isCrashTrackingAutoconfigEnabled() {
+    return !ConfigProvider.getInstance()
+        .getBoolean(
+            CrashTrackingConfig.CRASH_TRACKING_DISABLE_AUTOCONFIG,
+            CrashTrackingConfig.CRASH_TRACKING_DISABLE_AUTOCONFIG_DEFAULT);
+  }
+
+  private static void initializeCrashTracking(boolean delayed, boolean checkNative) {
     if (JavaVirtualMachine.isJ9()) {
       // TODO currently crash tracking is supported only for HotSpot based JVMs
       return;
     }
+    log.debug("Initializing crashtracking");
     try {
-      Class<?> clz = AGENT_CLASSLOADER.loadClass("com.datadog.crashtracking.ScriptInitializer");
-      clz.getMethod("initialize").invoke(null);
+      Class<?> clz = AGENT_CLASSLOADER.loadClass("datadog.crashtracking.Initializer");
+      // first try to use the JVMAccess using the native library; unless `checkNative` is false
+      boolean rslt =
+          checkNative && (boolean) clz.getMethod("initialize", boolean.class).invoke(null, false);
+      if (!rslt) {
+        if (delayed) {
+          // already delayed initialization, so no need to reschedule it again
+          // just call initialize and force JMX
+          rslt = (boolean) clz.getMethod("initialize", boolean.class).invoke(null, true);
+        } else {
+          // delayed initialization, so we need to reschedule it and mark as delayed but do not
+          // re-check the native library
+          CRASHTRACKER_INIT_AFTER_JMX = Agent::initializeDelayedCrashTrackingOnlyJmx;
+        }
+      }
+      if (rslt) {
+        log.debug("Crashtracking initialized");
+      } else {
+        log.debug(
+            SEND_TELEMETRY, "Crashtracking failed to initialize. No additional details available.");
+      }
     } catch (Throwable t) {
-      log.debug("Unable to initialize crash uploader", t);
+      log.debug(SEND_TELEMETRY, "Unable to initialize crashtracking", t);
     }
   }
 
@@ -1106,8 +1183,11 @@ public void withTracer(TracerAPI tracer) {
               }
             });
       }
-    } catch (final Throwable ex) {
-      log.error("Throwable thrown while starting profiling agent", ex);
+    } catch (final Throwable t) {
+      log.error(
+          SEND_TELEMETRY,
+          "Throwable thrown while starting profiling agent "
+              + Arrays.toString(t.getCause().getStackTrace()));
     } finally {
       Thread.currentThread().setContextClassLoader(contextLoader);
     }