Skip to content

Commit d314908

Browse files
jbachorikjbachorik
committed
Fix the stackdepth setting propagation to JFR
1 parent 887ea39 commit d314908

File tree

11 files changed

+150
-69
lines changed

11 files changed

+150
-69
lines changed

‎dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java‎

Lines changed: 41 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -325,19 +325,20 @@ public static void start(
325325
}
326326
}
327327

328+
boolean retryProfilerStart = false;
328329
if (profilingEnabled) {
329330
if (!isOracleJDK8()) {
330331
// Profiling agent startup code is written in a way to allow `startProfilingAgent` be called
331332
// multiple times
332333
// If early profiling is enabled then this call will start profiling.
333334
// If early profiling is disabled then later call will do this.
334-
startProfilingAgent(true, inst);
335+
retryProfilerStart = startProfilingAgent(true, true, inst);
335336
} else {
336337
log.debug("Oracle JDK 8 detected. Delaying profiler initialization.");
337338
// Profiling can not run early on Oracle JDK 8 because it will cause JFR initialization
338339
// deadlock.
339340
// Oracle JDK 8 JFR controller requires JMX so register an 'after-jmx-initialized' callback.
340-
PROFILER_INIT_AFTER_JMX = () -> startProfilingAgent(false, inst);
341+
PROFILER_INIT_AFTER_JMX = () -> startProfilingAgent(false, true, inst);
341342
}
342343
}
343344

@@ -433,7 +434,7 @@ public static void start(
433434
log.debug("Custom logger detected. Delaying Profiling initialization.");
434435
registerLogManagerCallback(new StartProfilingAgentCallback(inst));
435436
} else {
436-
startProfilingAgent(false, inst);
437+
startProfilingAgent(false, retryProfilerStart, inst);
437438
// only enable instrumentation based profilers when we know JFR is ready
438439
InstrumentationBasedProfiling.enableInstrumentationBasedProfiling();
439440
}
@@ -652,7 +653,7 @@ public AgentThread agentThread() {
652653

653654
@Override
654655
public void execute() {
655-
startProfilingAgent(false, inst);
656+
startProfilingAgent(false, true, inst);
656657
// only enable instrumentation based profilers when we know JFR is ready
657658
InstrumentationBasedProfiling.enableInstrumentationBasedProfiling();
658659
}
@@ -1215,48 +1216,51 @@ private static ProfilingContextIntegration createProfilingContextIntegration() {
12151216
return ProfilingContextIntegration.NoOp.INSTANCE;
12161217
}
12171218

1218-
private static void startProfilingAgent(final boolean isStartingFirst, Instrumentation inst) {
1219-
StaticEventLogger.begin("ProfilingAgent");
1220-
1219+
private static boolean startProfilingAgent(
1220+
final boolean earlyStart, final boolean firstAttempt, Instrumentation inst) {
12211221
if (isAwsLambdaRuntime()) {
1222-
log.info("Profiling not supported in AWS Lambda runtimes");
1223-
return;
1222+
if (firstAttempt) {
1223+
log.info("Profiling not supported in AWS Lambda runtimes");
1224+
}
1225+
return false;
12241226
}
12251227

1226-
final ClassLoader contextLoader = Thread.currentThread().getContextClassLoader();
1227-
try {
1228-
Thread.currentThread().setContextClassLoader(AGENT_CLASSLOADER);
1229-
final Class<?> profilingAgentClass =
1230-
AGENT_CLASSLOADER.loadClass("com.datadog.profiling.agent.ProfilingAgent");
1231-
final Method profilingInstallerMethod =
1232-
profilingAgentClass.getMethod(
1233-
"run", Boolean.TYPE, ClassLoader.class, Instrumentation.class);
1234-
profilingInstallerMethod.invoke(null, isStartingFirst, AGENT_CLASSLOADER, inst);
1228+
boolean requestRetry = false;
1229+
1230+
if (firstAttempt) {
1231+
StaticEventLogger.begin("ProfilingAgent");
1232+
final ClassLoader contextLoader = Thread.currentThread().getContextClassLoader();
1233+
try {
1234+
Thread.currentThread().setContextClassLoader(AGENT_CLASSLOADER);
1235+
final Class<?> profilingAgentClass =
1236+
AGENT_CLASSLOADER.loadClass("com.datadog.profiling.agent.ProfilingAgent");
1237+
final Method profilingInstallerMethod =
1238+
profilingAgentClass.getMethod("run", Boolean.TYPE, Instrumentation.class);
1239+
requestRetry = (boolean) profilingInstallerMethod.invoke(null, earlyStart, inst);
1240+
} catch (final Throwable ex) {
1241+
log.error(SEND_TELEMETRY, "Throwable thrown while starting profiling agent", ex);
1242+
} finally {
1243+
Thread.currentThread().setContextClassLoader(contextLoader);
1244+
}
1245+
StaticEventLogger.end("ProfilingAgent");
1246+
}
1247+
if (!earlyStart) {
12351248
/*
12361249
* Install the tracer hooks only when not using 'early start'.
12371250
* The 'early start' is happening so early that most of the infrastructure has not been set up yet.
12381251
*/
1239-
if (!isStartingFirst) {
1240-
log.debug("Scheduling scope event factory registration");
1241-
WithGlobalTracer.registerOrExecute(
1242-
new WithGlobalTracer.Callback() {
1243-
@Override
1244-
public void withTracer(TracerAPI tracer) {
1245-
log.debug("Initializing profiler tracer integrations");
1246-
tracer.getProfilingContext().onStart();
1247-
}
1248-
});
1249-
}
1250-
} catch (final Throwable t) {
1251-
log.error(
1252-
SEND_TELEMETRY,
1253-
"Throwable thrown while starting profiling agent "
1254-
+ Arrays.toString(t.getCause().getStackTrace()));
1255-
} finally {
1256-
Thread.currentThread().setContextClassLoader(contextLoader);
1252+
initProfilerContext();
12571253
}
1254+
return requestRetry;
1255+
}
12581256

1259-
StaticEventLogger.end("ProfilingAgent");
1257+
private static void initProfilerContext() {
1258+
log.debug("Scheduling profiler context initialization");
1259+
WithGlobalTracer.registerOrExecute(
1260+
tracer -> {
1261+
log.debug("Initializing profiler context integration");
1262+
tracer.getProfilingContext().onStart();
1263+
});
12601264
}
12611265

12621266
private static boolean isAwsLambdaRuntime() {

‎dd-java-agent/agent-profiling/profiling-controller-ddprof/src/main/java/com/datadog/profiling/controller/ddprof/DatadogProfilerSettings.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,9 @@ public void publish() {
2727
datadogProfiler.recordSetting(PERF_EVENTS_PARANOID_KEY, perfEventsParanoid);
2828
datadogProfiler.recordSetting(NATIVE_STACKS_KEY, String.valueOf(hasNativeStacks));
2929
datadogProfiler.recordSetting(JFR_IMPLEMENTATION_KEY, "ddprof");
30-
datadogProfiler.recordSetting(STACK_DEPTH_KEY, String.valueOf(stackDepth));
30+
datadogProfiler.recordSetting(
31+
"ddprof " + STACK_DEPTH_KEY,
32+
String.valueOf(requestedStackDepth)); // ddprof-java will accept the requested stack depth
3133
datadogProfiler.recordSetting(SELINUX_STATUS_KEY, seLinuxStatus);
3234
if (serviceInstrumentationType != null) {
3335
datadogProfiler.recordSetting(SERVICE_INSTRUMENTATION_TYPE, serviceInstrumentationType);

‎dd-java-agent/agent-profiling/profiling-controller-jfr/implementation/build.gradle‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,3 +53,13 @@ idea {
5353
jdkName = '11'
5454
}
5555
}
56+
57+
project.afterEvaluate {
58+
tasks.withType(Test).configureEach {
59+
if (javaLauncher.get().metadata.languageVersion.asInt() >= 9) {
60+
jvmArgs += [
61+
'--add-opens',
62+
'jdk.jfr/jdk.jfr.internal=ALL-UNNAMED'] // JPMSJFRAccess needs access to jdk.jfr.internal package
63+
}
64+
}
65+
}

‎dd-java-agent/agent-profiling/profiling-controller-jfr/implementation/src/main/java11/com/datadog/profiling/controller/jfr/JPMSJFRAccess.java‎

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -51,18 +51,30 @@ public JPMSJFRAccess(Instrumentation inst) throws Exception {
5151

5252
jvmClass = JFRAccess.class.getClassLoader().loadClass("jdk.jfr.internal.JVM");
5353
repositoryClass = JFRAccess.class.getClassLoader().loadClass("jdk.jfr.internal.Repository");
54-
safePathClass =
55-
JFRAccess.class.getClassLoader().loadClass("jdk.jfr.internal.SecuritySupport$SafePath");
54+
safePathClass = safePathClass();
5655
Object jvm = getJvm();
5756
setStackDepthMH = getJvmMethodHandle(jvm, "setStackDepth", int.class);
5857
setRepositoryBaseMH = setRepositoryBaseMethodHandle();
5958
counterTimeMH = getJvmMethodHandle(jvm, "counterTime");
6059
getTimeConversionFactorMH = getJvmMethodHandle(jvm, "getTimeConversionFactor");
6160
}
6261

63-
private Object getJvm()
64-
throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
65-
return jvmClass.getMethod("getJVM").invoke(null);
62+
private static Class<?> safePathClass() {
63+
try {
64+
return JFRAccess.class
65+
.getClassLoader()
66+
.loadClass("jdk.jfr.internal.SecuritySupport$SafePath");
67+
} catch (ClassNotFoundException e) {
68+
return Path.class; // no SafePath with SecurityManager gone
69+
}
70+
}
71+
72+
private Object getJvm() {
73+
try {
74+
return jvmClass.getMethod("getJVM").invoke(null);
75+
} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException ignored) {
76+
}
77+
return null;
6678
}
6779

6880
private MethodHandle getJvmMethodHandle(Object jvm, String method, Class... args)
@@ -97,6 +109,11 @@ private MethodHandle setRepositoryBaseMethodHandle()
97109
}
98110

99111
private static void patchModuleAccess(Instrumentation inst) {
112+
if (inst == null) {
113+
// used in testing; we don't have instrumentation and will patch the module access in the test
114+
// task
115+
return;
116+
}
100117
Module unnamedModule = JFRAccess.class.getClassLoader().getUnnamedModule();
101118
Module targetModule = Event.class.getModule();
102119

@@ -126,7 +143,10 @@ public boolean setStackDepth(int depth) {
126143
@Override
127144
public boolean setBaseLocation(String location) {
128145
try {
129-
Object safePath = safePathClass.getConstructor(Path.class).newInstance(Paths.get(location));
146+
Object safePath =
147+
Path.class.isAssignableFrom(safePathClass)
148+
? Paths.get(location)
149+
: safePathClass.getConstructor(Path.class).newInstance(Paths.get(location));
130150
setRepositoryBaseMH.invoke(safePath);
131151
return true;
132152
} catch (Throwable throwable) {

‎dd-java-agent/agent-profiling/profiling-controller-jfr/implementation/src/test/java/com/datadog/profiling/controller/jfr/JFRAccessTest.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import static datadog.environment.JavaVirtualMachine.isJ9;
44
import static datadog.environment.JavaVirtualMachine.isJavaVersion;
5+
import static datadog.environment.JavaVirtualMachine.isJavaVersionAtLeast;
56
import static datadog.environment.JavaVirtualMachine.isOracleJDK8;
67
import static org.junit.jupiter.api.Assertions.assertEquals;
78
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -22,6 +23,18 @@ void testJava8JFRAccess() {
2223
assertTrue(jfrAccess.setStackDepth(42));
2324
}
2425

26+
@Test
27+
void testJPMSJFRAccess() throws Exception {
28+
// For Java 9 and above, the JFR access requires instrumentation in order to patch the module
29+
// access
30+
assumeTrue(isJavaVersionAtLeast(9) && !isJ9() && !isOracleJDK8());
31+
32+
// just do a sanity check that it is possible to instantiate the class and call
33+
// 'setStackDepth()'
34+
JPMSJFRAccess jfrAccess = new JPMSJFRAccess(null);
35+
assertTrue(jfrAccess.setStackDepth(42));
36+
}
37+
2538
@Test
2639
void testJ9JFRAccess() {
2740
assumeTrue(isJ9());

‎dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/JfrProfilerSettings.java‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package com.datadog.profiling.controller.openjdk;
22

3+
import com.datadog.profiling.controller.ControllerContext;
34
import com.datadog.profiling.controller.ProfilerSettingsSupport;
45
import com.datadog.profiling.controller.openjdk.events.ProfilerSettingEvent;
56
import datadog.environment.JavaVirtualMachine;
@@ -14,16 +15,18 @@ final class JfrProfilerSettings extends ProfilerSettingsSupport {
1415
private static final String EXCEPTION_HISTO_REPORT_LIMIT_KEY = "Exception Histo Report Limit";
1516
private static final String EXCEPTION_HISTO_SIZE_LIMIT_KEY = "Exception Histo Size Limit";
1617
private final String jfrImplementation;
18+
private final boolean isDdprofActive;
1719

1820
public JfrProfilerSettings(
1921
ConfigProvider configProvider,
20-
String ddprofUnavailableReason,
22+
ControllerContext.Snapshot context,
2123
boolean hasJfrStackDepthApplied) {
22-
super(configProvider, ddprofUnavailableReason, hasJfrStackDepthApplied);
24+
super(configProvider, context.getDatadogProfilerUnavailableReason(), hasJfrStackDepthApplied);
2325
this.jfrImplementation =
2426
Platform.isNativeImage()
2527
? "native-image"
2628
: (JavaVirtualMachine.isOracleJDK8() ? "oracle" : "openjdk");
29+
this.isDdprofActive = context.isDatadogProfilerEnabled();
2730
}
2831

2932
public void publish() {
@@ -56,8 +59,16 @@ public void publish() {
5659
new ProfilerSettingEvent(PERF_EVENTS_PARANOID_KEY, perfEventsParanoid).commit();
5760
new ProfilerSettingEvent(NATIVE_STACKS_KEY, String.valueOf(hasNativeStacks)).commit();
5861
new ProfilerSettingEvent(JFR_IMPLEMENTATION_KEY, jfrImplementation).commit();
59-
if (hasJfrStackDepthApplied) {
60-
new ProfilerSettingEvent(STACK_DEPTH_KEY, String.valueOf(stackDepth)).commit();
62+
new ProfilerSettingEvent(
63+
"JFR " + STACK_DEPTH_KEY,
64+
String.valueOf(hasJfrStackDepthApplied ? requestedStackDepth : jfrStackDepth))
65+
.commit();
66+
if (isDdprofActive) {
67+
// emit this setting only if datadog profiler is also active
68+
new ProfilerSettingEvent(
69+
"ddprof " + STACK_DEPTH_KEY,
70+
String.valueOf(hasJfrStackDepthApplied ? requestedStackDepth : jfrStackDepth))
71+
.commit();
6172
}
6273
new ProfilerSettingEvent(SELINUX_STATUS_KEY, seLinuxStatus).commit();
6374
if (ddprofUnavailableReason != null) {

‎dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/OpenJdkOngoingRecording.java‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -60,10 +60,7 @@ public class OpenJdkOngoingRecording implements OngoingRecording {
6060
recording.start();
6161
log.debug("Recording {} started", recordingName);
6262
this.configMemento =
63-
new JfrProfilerSettings(
64-
configProvider,
65-
context.getDatadogProfilerUnavailableReason(),
66-
jfrStackDepthSettingApplied);
63+
new JfrProfilerSettings(configProvider, context, jfrStackDepthSettingApplied);
6764
}
6865

6966
OpenJdkOngoingRecording(
@@ -77,10 +74,7 @@ public class OpenJdkOngoingRecording implements OngoingRecording {
7774
recording.start();
7875
log.debug("Recording {} started", recording.getName());
7976
this.configMemento =
80-
new JfrProfilerSettings(
81-
ConfigProvider.getInstance(),
82-
context.getDatadogProfilerUnavailableReason(),
83-
jfrStackDepthSettingApplied);
77+
new JfrProfilerSettings(ConfigProvider.getInstance(), context, jfrStackDepthSettingApplied);
8478
}
8579

8680
private void disableOverriddenEvents(ControllerContext.Snapshot context) {

‎dd-java-agent/agent-profiling/profiling-controller/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ excludedClassesCoverage += [
1919
dependencies {
2020
api libs.slf4j
2121
api project(':internal-api')
22+
api project(':components:environment')
2223
api project(':dd-java-agent:agent-profiling:profiling-utils')
2324

2425
testImplementation libs.bundles.junit5

‎dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/ProfilerSettingsSupport.java‎

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import static datadog.trace.api.telemetry.LogCollector.SEND_TELEMETRY;
44

5+
import datadog.environment.JavaVirtualMachine;
56
import datadog.environment.OperatingSystem;
67
import datadog.trace.api.Config;
78
import datadog.trace.api.config.ProfilingConfig;
@@ -108,7 +109,8 @@ public String toString() {
108109

109110
protected final ProfilerActivationSetting profilerActivationSetting;
110111

111-
protected final int stackDepth;
112+
protected final int jfrStackDepth;
113+
protected final int requestedStackDepth;
112114
protected final boolean hasJfrStackDepthApplied;
113115

114116
protected ProfilerSettingsSupport(
@@ -170,11 +172,10 @@ protected ProfilerSettingsSupport(
170172
configProvider.getString(
171173
"profiling.async.cstack",
172174
ProfilingConfig.PROFILING_DATADOG_PROFILER_CSTACK_DEFAULT)));
173-
stackDepth =
175+
requestedStackDepth =
174176
configProvider.getInteger(
175-
ProfilingConfig.PROFILING_STACKDEPTH,
176-
ProfilingConfig.PROFILING_STACKDEPTH_DEFAULT,
177-
ProfilingConfig.PROFILING_DATADOG_PROFILER_STACKDEPTH);
177+
ProfilingConfig.PROFILING_STACKDEPTH, ProfilingConfig.PROFILING_STACKDEPTH_DEFAULT);
178+
jfrStackDepth = getStackDepth();
178179

179180
seLinuxStatus = getSELinuxStatus();
180181
this.ddprofUnavailableReason = ddprofUnavailableReason;
@@ -191,6 +192,30 @@ protected ProfilerSettingsSupport(
191192
"Profiler settings: " + this); // telemetry receiver does not recognize formatting
192193
}
193194

195+
private static int getStackDepth() {
196+
String value =
197+
JavaVirtualMachine.getVmOptions().stream()
198+
.filter(o -> o.startsWith("-XX:FlightRecorderOptions"))
199+
.findFirst()
200+
.orElse(null);
201+
if (value != null) {
202+
int start = value.indexOf("stackdepth=");
203+
if (start != -1) {
204+
start += "stackdepth=".length();
205+
int end = value.indexOf(',', start);
206+
if (end == -1) {
207+
end = value.length();
208+
}
209+
try {
210+
return Integer.parseInt(value.substring(start, end));
211+
} catch (NumberFormatException e) {
212+
logger.debug(SEND_TELEMETRY, "Failed to parse stack depth from JFR options: " + value, e);
213+
}
214+
}
215+
}
216+
return 64; // default stack depth if not set in JFR options
217+
}
218+
194219
private static String getServiceInjection(ConfigProvider configProvider) {
195220
// usually set via DD_INJECTION_ENABLED env var
196221
return configProvider.getString("injection.enabled");
@@ -284,7 +309,8 @@ public String toString() {
284309
+ ", serviceInjection='" + serviceInjection + '\''
285310
+ ", ddprofUnavailableReason='" + ddprofUnavailableReason + '\''
286311
+ ", profilerActivationSetting=" + profilerActivationSetting
287-
+ ", stackDepth=" + stackDepth
312+
+ ", jfrStackDepth=" + jfrStackDepth
313+
+ ", requestedStackDepth=" + requestedStackDepth
288314
+ ", hasJfrStackDepthApplied=" + hasJfrStackDepthApplied
289315
+ '}';
290316
// spotless:on

0 commit comments

Comments
 (0)