Automatically register crashtracking via native extensions (#8851)

Co-authored-by: Jaroslav Bachorik <[email protected]>

Author: MattAlp

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java

@@ -3,6 +3,7 @@
 import static datadog.environment.JavaVirtualMachine.isJavaVersionAtLeast;
 import static datadog.environment.JavaVirtualMachine.isOracleJDK8;
 import static datadog.trace.api.ConfigDefaults.DEFAULT_STARTUP_LOGS_ENABLED;
+import static datadog.trace.api.telemetry.LogCollector.SEND_TELEMETRY;
 import static datadog.trace.bootstrap.Library.WILDFLY;
 import static datadog.trace.bootstrap.Library.detectLibraries;
 import static datadog.trace.util.AgentThreadFactory.AgentThread.JMX_STARTUP;
@@ -21,6 +22,7 @@
 import datadog.trace.api.appsec.AppSecEventTracker;
 import datadog.trace.api.config.AppSecConfig;
 import datadog.trace.api.config.CiVisibilityConfig;
+import datadog.trace.api.config.CrashTrackingConfig;
 import datadog.trace.api.config.CwsConfig;
 import datadog.trace.api.config.DebuggerConfig;
 import datadog.trace.api.config.GeneralConfig;
@@ -56,6 +58,7 @@
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.security.CodeSource;
+import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -97,6 +100,9 @@ private enum AgentFeature {
     TRACING(TraceInstrumentationConfig.TRACE_ENABLED, true),
     JMXFETCH(JmxFetchConfig.JMX_FETCH_ENABLED, true),
     STARTUP_LOGS(GeneralConfig.STARTUP_LOGS_ENABLED, DEFAULT_STARTUP_LOGS_ENABLED),
+    CRASH_TRACKING(
+        CrashTrackingConfig.CRASH_TRACKING_ENABLED,
+        CrashTrackingConfig.CRASH_TRACKING_ENABLED_DEFAULT),
     PROFILING(ProfilingConfig.PROFILING_ENABLED, false),
     APPSEC(AppSecConfig.APPSEC_ENABLED, false),
     IAST(IastConfig.IAST_ENABLED, false),
@@ -150,9 +156,11 @@ public boolean isEnabledByDefault() {
   private static ClassLoader AGENT_CLASSLOADER = null;
 
   private static volatile Runnable PROFILER_INIT_AFTER_JMX = null;
+  private static volatile Runnable CRASHTRACKER_INIT_AFTER_JMX = null;
 
   private static boolean jmxFetchEnabled = true;
   private static boolean profilingEnabled = false;
+  private static boolean crashTrackingEnabled = false;
   private static boolean appSecEnabled;
   private static boolean appSecFullyDisabled;
   private static boolean remoteConfigEnabled = true;
@@ -282,6 +290,7 @@ public static void start(
 
     jmxFetchEnabled = isFeatureEnabled(AgentFeature.JMXFETCH);
     profilingEnabled = isFeatureEnabled(AgentFeature.PROFILING);
+    crashTrackingEnabled = isFeatureEnabled(AgentFeature.CRASH_TRACKING);
     usmEnabled = isFeatureEnabled(AgentFeature.USM);
     appSecEnabled = isFeatureEnabled(AgentFeature.APPSEC);
     appSecFullyDisabled = isFullyDisabled(AgentFeature.APPSEC);
@@ -328,13 +337,7 @@ public static void start(
         // Profiling can not run early on Oracle JDK 8 because it will cause JFR initialization
         // deadlock.
         // Oracle JDK 8 JFR controller requires JMX so register an 'after-jmx-initialized' callback.
-        PROFILER_INIT_AFTER_JMX =
-            new Runnable() {
-              @Override
-              public void run() {
-                startProfilingAgent(false, inst);
-              }
-            };
+        PROFILER_INIT_AFTER_JMX = () -> startProfilingAgent(false, inst);
       }
     }
 
@@ -347,6 +350,14 @@ public void run() {
      * when it will happen after the class transformers were added.
      */
     AgentTaskScheduler.initialize();
+
+    // We need to run the crashtracking initialization after all the config has been resolved and
+    // task scheduler initialized
+    if (crashTrackingEnabled) {
+      StaticEventLogger.begin("crashtracking");
+      startCrashTracking();
+      StaticEventLogger.end("crashtracking");
+    }
     startDatadogAgent(initTelemetry, inst);
 
     final EnumSet<Library> libraries = detectLibraries(log);
@@ -741,6 +752,29 @@ private static synchronized void installDatadogTracer(
     StaticEventLogger.end("GlobalTracer");
   }
 
+  private static void startCrashTracking() {
+    if (isJavaVersionAtLeast(9)) {
+      // it is safe to initialize crashtracking early
+      // since it can take 100ms+ to initialize the native library we will defer the initialization
+      // ... unless we request early start with the debug config flag
+      boolean forceEarlyStart = CrashTrackingConfig.CRASH_TRACKING_START_EARLY_DEFAULT;
+      String forceEarlyStartStr =
+          ddGetProperty("dd." + CrashTrackingConfig.CRASH_TRACKING_START_EARLY);
+      if (forceEarlyStartStr != null) {
+        forceEarlyStart = Boolean.parseBoolean(forceEarlyStartStr);
+      }
+      if (forceEarlyStart) {
+        initializeCrashTrackingDefault();
+      } else {
+        AgentTaskScheduler.INSTANCE.execute(Agent::initializeCrashTrackingDefault);
+      }
+    } else {
+      // for Java 8 we are relying on JMX to give us the process PID
+      // we need to delay the crash tracking initialization until JMX is available
+      CRASHTRACKER_INIT_AFTER_JMX = Agent::initializeDelayedCrashTracking;
+    }
+  }
+
   private static void scheduleJmxStart(final int jmxStartDelay) {
     if (jmxStartDelay > 0) {
       AgentTaskScheduler.INSTANCE.scheduleWithJitter(
@@ -764,25 +798,41 @@ private static synchronized void startJmx() {
     if (jmxStarting.getAndSet(true)) {
       return; // another thread is already in startJmx
     }
-    // error tracking initialization relies on JMX being available
-    initializeErrorTracking();
     if (jmxFetchEnabled) {
       startJmxFetch();
     }
     initializeJmxSystemAccessProvider(AGENT_CLASSLOADER);
+    if (crashTrackingEnabled && CRASHTRACKER_INIT_AFTER_JMX != null) {
+      try {
+        CRASHTRACKER_INIT_AFTER_JMX.run();
+      } finally {
+        CRASHTRACKER_INIT_AFTER_JMX = null;
+      }
+    }
     if (profilingEnabled) {
       registerDeadlockDetectionEvent();
       registerSmapEntryEvent();
       if (PROFILER_INIT_AFTER_JMX != null) {
-        if (getJmxStartDelay() == 0) {
-          log.debug("Waiting for profiler initialization");
-          AgentTaskScheduler.INSTANCE.scheduleWithJitter(
-              PROFILER_INIT_AFTER_JMX, 500, TimeUnit.MILLISECONDS);
-        } else {
-          log.debug("Initializing profiler");
-          PROFILER_INIT_AFTER_JMX.run();
+        try {
+          /*
+          When getJmxStartDelay() is set to 0 we will attempt to initialize the JMX subsystem as soon as available.
+          But, this can cause issues with JFR as it needs some 'grace period' after JMX is ready. That's why we are
+          re-scheduling the profiler initialization code just a tad later.
+
+          If the jmx start delay is set, we are already delayed relative to the jmx init so we can just plainly
+          run the initialization code.
+          */
+          if (getJmxStartDelay() == 0) {
+            log.debug("Waiting for profiler initialization");
+            AgentTaskScheduler.INSTANCE.scheduleWithJitter(
+                PROFILER_INIT_AFTER_JMX, 500, TimeUnit.MILLISECONDS);
+          } else {
+            log.debug("Initializing profiler");
+            PROFILER_INIT_AFTER_JMX.run();
+          }
+        } finally {
+          PROFILER_INIT_AFTER_JMX = null;
         }
-        PROFILER_INIT_AFTER_JMX = null;
       }
     }
   }
@@ -1042,16 +1092,63 @@ private static void stopTelemetry() {
     }
   }
 
-  private static void initializeErrorTracking() {
+  private static void initializeDelayedCrashTracking() {
+    initializeCrashTracking(true, isCrashTrackingAutoconfigEnabled());
+  }
+
+  private static void initializeDelayedCrashTrackingOnlyJmx() {
+    initializeCrashTracking(true, false);
+  }
+
+  private static void initializeCrashTrackingDefault() {
+    initializeCrashTracking(false, isCrashTrackingAutoconfigEnabled());
+  }
+
+  private static boolean isCrashTrackingAutoconfigEnabled() {
+    String enabledVal = ddGetProperty("dd." + CrashTrackingConfig.CRASH_TRACKING_ENABLE_AUTOCONFIG);
+    boolean enabled = CrashTrackingConfig.CRASH_TRACKING_ENABLE_AUTOCONFIG_DEFAULT;
+    if (enabledVal != null) {
+      enabled = Boolean.parseBoolean(enabledVal);
+    } else {
+      // If the property is not set, then we check if profiling is enabled
+      enabled = profilingEnabled;
+    }
+    return enabled;
+  }
+
+  private static void initializeCrashTracking(boolean delayed, boolean checkNative) {
     if (JavaVirtualMachine.isJ9()) {
       // TODO currently crash tracking is supported only for HotSpot based JVMs
       return;
     }
+    log.debug("Initializing crashtracking");
     try {
-      Class<?> clz = AGENT_CLASSLOADER.loadClass("com.datadog.crashtracking.ScriptInitializer");
-      clz.getMethod("initialize").invoke(null);
+      Class<?> clz = AGENT_CLASSLOADER.loadClass("datadog.crashtracking.Initializer");
+      // first try to use the JVMAccess using the native library; unless `checkNative` is false
+      Boolean rslt =
+          checkNative && (Boolean) clz.getMethod("initialize", boolean.class).invoke(null, false);
+      if (!rslt) {
+        if (delayed) {
+          // already delayed initialization, so no need to reschedule it again
+          // just call initialize and force JMX
+          rslt = (Boolean) clz.getMethod("initialize", boolean.class).invoke(null, true);
+        } else {
+          // delayed initialization, so we need to reschedule it and mark as delayed but do not
+          // re-check the native library
+          CRASHTRACKER_INIT_AFTER_JMX = Agent::initializeDelayedCrashTrackingOnlyJmx;
+          rslt = null; // we will initialize it later
+        }
+      }
+      if (rslt == null) {
+        log.debug("Crashtracking initialization delayed until JMX is available");
+      } else if (rslt) {
+        log.debug("Crashtracking initialized");
+      } else {
+        log.debug(
+            SEND_TELEMETRY, "Crashtracking failed to initialize. No additional details available.");
+      }
     } catch (Throwable t) {
-      log.debug("Unable to initialize crash uploader", t);
+      log.debug(SEND_TELEMETRY, "Unable to initialize crashtracking", t);
     }
   }
 
@@ -1150,8 +1247,11 @@ public void withTracer(TracerAPI tracer) {
               }
             });
       }
-    } catch (final Throwable ex) {
-      log.error("Throwable thrown while starting profiling agent", ex);
+    } catch (final Throwable t) {
+      log.error(
+          SEND_TELEMETRY,
+          "Throwable thrown while starting profiling agent "
+              + Arrays.toString(t.getCause().getStackTrace()));
     } finally {
       Thread.currentThread().setContextClassLoader(contextLoader);
     }

dd-java-agent/agent-crashtracking/build.gradle

@@ -16,6 +16,7 @@ dependencies {
   implementation project(':internal-api')
   implementation project(':utils:container-utils')
   implementation project(':utils:version-utils')
+  implementation project(path: ':dd-java-agent:ddprof-lib', configuration: 'shadow')
 
   implementation libs.okhttp
   implementation libs.moshi