Experimental support for CRaC (Coordinated Restore at Checkpoint) - not for merging

Author: mcculls

dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/TracerInstaller.java

@@ -24,7 +24,7 @@ public static synchronized void installGlobalTracer(
                 .pollForTracerFlareRequests()
                 .pollForTracingConfiguration()
                 .build();
-        installGlobalTracer(tracer);
+        forceInstallGlobalTracer(tracer);
       } else {
         log.debug("GlobalTracer already registered.");
       }

dd-java-agent/instrumentation/crac/build.gradle

@@ -0,0 +1,13 @@
+muzzle {
+  pass {
+    group = "org.crac"
+    module = "crac"
+    versions = "[1.3,)"
+  }
+}
+
+apply from: "$rootDir/gradle/java.gradle"
+
+dependencies {
+  compileOnly group: 'org.crac', name: 'crac', version: '1.3.0'
+}

dd-java-agent/instrumentation/crac/src/main/java/trace/instrumentation/crac/CracRuntimeInstrumentation.java

@@ -0,0 +1,47 @@
+package trace.instrumentation.crac;
+
+import static net.bytebuddy.matcher.ElementMatchers.isTypeInitializer;
+
+import com.google.auto.service.AutoService;
+import datadog.trace.agent.tooling.Instrumenter;
+import datadog.trace.agent.tooling.InstrumenterModule;
+import datadog.trace.api.Platform;
+import net.bytebuddy.asm.Advice;
+import org.crac.Core;
+
+@AutoService(InstrumenterModule.class)
+public final class CracRuntimeInstrumentation extends InstrumenterModule.Tracing
+    implements Instrumenter.ForSingleType {
+
+  public CracRuntimeInstrumentation() {
+    super("crac");
+  }
+
+  @Override
+  protected boolean defaultEnabled() {
+    return !Platform.isNativeImageBuilder();
+  }
+
+  @Override
+  public String instrumentedType() {
+    return "org.crac.Core";
+  }
+
+  @Override
+  public String[] helperClassNames() {
+    return new String[] {packageName + ".TracerResource"};
+  }
+
+  @Override
+  public void methodAdvice(Instrumenter.MethodTransformer transformer) {
+    transformer.applyAdvice(
+        isTypeInitializer(), CracRuntimeInstrumentation.class.getName() + "$TracerResourceAdvice");
+  }
+
+  public static class TracerResourceAdvice {
+    @Advice.OnMethodExit(suppress = Throwable.class)
+    public static void onExit() {
+      Core.getGlobalContext().register(new TracerResource());
+    }
+  }
+}

dd-java-agent/instrumentation/crac/src/main/java/trace/instrumentation/crac/TracerResource.java

@@ -0,0 +1,17 @@
+package trace.instrumentation.crac;
+
+import datadog.trace.api.GlobalTracer;
+import datadog.trace.bootstrap.instrumentation.api.AgentTracer;
+import org.crac.Context;
+import org.crac.Resource;
+
+public final class TracerResource implements Resource {
+
+  @Override
+  public void beforeCheckpoint(Context<? extends Resource> context) {
+    ((AgentTracer.TracerAPI) GlobalTracer.get()).pause();
+  }
+
+  @Override
+  public void afterRestore(Context<? extends Resource> context) {}
+}

dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java

@@ -152,6 +152,8 @@ public static CoreTracerBuilder builder() {
   /** Nanosecond offset to counter clock drift */
   private volatile long counterDrift;
 
+  private final SharedCommunicationObjects sco;
+
   private final TracerFlarePoller tracerFlarePoller;
 
   private final TracingConfigPoller tracingConfigPoller;
@@ -652,6 +654,7 @@ private CoreTracer(
     }
     sharedCommunicationObjects.monitoring = monitoring;
     sharedCommunicationObjects.createRemaining(config);
+    sco = sharedCommunicationObjects;
 
     tracerFlarePoller = new TracerFlarePoller(dynamicConfig);
     if (pollForTracerFlareRequests) {
@@ -1148,6 +1151,11 @@ public void close() {
     tracerFlarePoller.stop();
   }
 
+  @Override
+  public void pause() {
+    sco.okHttpClient.connectionPool().evictAll();
+  }
+
   @Override
   public void addScopeListener(
       Runnable afterScopeActivatedCallback, Runnable afterScopeClosedCallback) {

internal-api/src/main/java/datadog/trace/api/Config.java

@@ -993,7 +993,7 @@ private Config(final ConfigProvider configProvider, final InstrumenterConfig ins
     statsDClientSocketBuffer = configProvider.getInteger(STATSD_CLIENT_SOCKET_BUFFER);
     statsDClientSocketTimeout = configProvider.getInteger(STATSD_CLIENT_SOCKET_TIMEOUT);
 
-    runtimeMetricsEnabled = configProvider.getBoolean(RUNTIME_METRICS_ENABLED, true);
+    runtimeMetricsEnabled = configProvider.getBoolean(RUNTIME_METRICS_ENABLED, false);
 
     jmxFetchEnabled =
         runtimeMetricsEnabled
@@ -1231,9 +1231,7 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment())
                 TELEMETRY_LOG_COLLECTION_ENABLED, DEFAULT_TELEMETRY_LOG_COLLECTION_ENABLED);
 
     isTelemetryDependencyServiceEnabled =
-        configProvider.getBoolean(
-            TELEMETRY_DEPENDENCY_COLLECTION_ENABLED,
-            DEFAULT_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED);
+        configProvider.getBoolean(TELEMETRY_DEPENDENCY_COLLECTION_ENABLED, false);
     telemetryDependencyResolutionQueueSize =
         configProvider.getInteger(
             TELEMETRY_DEPENDENCY_RESOLUTION_QUEUE_SIZE,

internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentTracer.java

@@ -256,6 +256,8 @@ default SpanBuilder buildSpan(CharSequence spanName) {
 
     void close();
 
+    void pause();
+
     /**
      * Attach a scope listener to the global scope manager
      *
@@ -412,6 +414,9 @@ public SpanBuilder buildSpan(final String instrumentationName, final CharSequenc
     @Override
     public void close() {}
 
+    @Override
+    public void pause() {}
+
     @Override
     public void addScopeListener(
         Runnable afterScopeActivatedCallback, Runnable afterScopeClosedCallback) {}

settings.gradle

@@ -218,6 +218,7 @@ include ':dd-java-agent:instrumentation:couchbase:couchbase-2.0'
 include ':dd-java-agent:instrumentation:couchbase:couchbase-2.6'
 include ':dd-java-agent:instrumentation:couchbase:couchbase-3.1'
 include ':dd-java-agent:instrumentation:couchbase:couchbase-3.2'
+include ':dd-java-agent:instrumentation:crac'
 include ':dd-java-agent:instrumentation:cucumber'
 include ':dd-java-agent:instrumentation:cxf-2.1'
 include ':dd-java-agent:instrumentation:datanucleus-4'