Skip to content

Commit 14a80e9

Browse files
authored
Merge pull request #7287 from DataDog/dougqh/guardrails-telemetry
Adding InitializationTelemetry - e.g. guard rails reporting
2 parents 62eb0c2 + e1e17fc commit 14a80e9

File tree

18 files changed

+2391
-44
lines changed

18 files changed

+2391
-44
lines changed

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java

Lines changed: 33 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,19 @@ public boolean isEnabledByDefault() {
151151
private static boolean debuggerEnabled = false;
152152
private static boolean agentlessLogSubmissionEnabled = false;
153153

154-
public static void start(final Instrumentation inst, final URL agentJarURL, String agentArgs) {
154+
/**
155+
* Starts the agent; returns a boolean indicating if Agent started successfully
156+
*
157+
* <p>The Agent is considered to start successfully if Instrumentation can be activated. All other
158+
* pieces are considered optional.
159+
*/
160+
public static void start(
161+
final Object bootstrapInitTelemetry,
162+
final Instrumentation inst,
163+
final URL agentJarURL,
164+
final String agentArgs) {
165+
InitializationTelemetry initTelemetry = InitializationTelemetry.proxy(bootstrapInitTelemetry);
166+
155167
StaticEventLogger.begin("Agent");
156168
StaticEventLogger.begin("Agent.start");
157169

@@ -163,8 +175,9 @@ public static void start(final Instrumentation inst, final URL agentJarURL, Stri
163175
remoteConfigEnabled = false;
164176
telemetryEnabled = false;
165177
// apply trace instrumentation, but skip starting other services
166-
startDatadogAgent(inst);
178+
startDatadogAgent(initTelemetry, inst);
167179
StaticEventLogger.end("Agent.start");
180+
168181
return;
169182
}
170183

@@ -279,7 +292,7 @@ public void run() {
279292
* when it will happen after the class transformers were added.
280293
*/
281294
AgentTaskScheduler.initialize();
282-
startDatadogAgent(inst);
295+
startDatadogAgent(initTelemetry, inst);
283296

284297
final EnumSet<Library> libraries = detectLibraries(log);
285298

@@ -326,7 +339,7 @@ public void run() {
326339
* logging facility. Likewise on IBM JDKs OkHttp may indirectly load 'IBMSASL' which in turn loads LogManager.
327340
*/
328341
InstallDatadogTracerCallback installDatadogTracerCallback =
329-
new InstallDatadogTracerCallback(inst);
342+
new InstallDatadogTracerCallback(initTelemetry, inst);
330343
if (delayOkHttp) {
331344
log.debug("Custom logger detected. Delaying Datadog Tracer initialization.");
332345
registerLogManagerCallback(installDatadogTracerCallback);
@@ -397,10 +410,10 @@ public static synchronized Class<?> installAgentCLI() throws Exception {
397410
}
398411

399412
/** Used by AgentCLI to send sample traces from the command-line. */
400-
public static void startDatadogTracer() throws Exception {
413+
public static void startDatadogTracer(InitializationTelemetry initTelemetry) throws Exception {
401414
Class<?> scoClass =
402415
AGENT_CLASSLOADER.loadClass("datadog.communication.ddagent.SharedCommunicationObjects");
403-
installDatadogTracer(scoClass, scoClass.getConstructor().newInstance());
416+
installDatadogTracer(initTelemetry, scoClass, scoClass.getConstructor().newInstance());
404417
startJmx(); // send runtime metrics along with the traces
405418
}
406419

@@ -475,9 +488,12 @@ public void execute() {
475488
}
476489

477490
protected static class InstallDatadogTracerCallback extends ClassLoadCallBack {
491+
private final InitializationTelemetry initTelemetry;
478492
private final Instrumentation instrumentation;
479493

480-
public InstallDatadogTracerCallback(Instrumentation instrumentation) {
494+
public InstallDatadogTracerCallback(
495+
InitializationTelemetry initTelemetry, Instrumentation instrumentation) {
496+
this.initTelemetry = initTelemetry;
481497
this.instrumentation = instrumentation;
482498
}
483499

@@ -502,7 +518,7 @@ public void execute() {
502518
throw new UndeclaredThrowableException(e);
503519
}
504520

505-
installDatadogTracer(scoClass, sco);
521+
installDatadogTracer(initTelemetry, scoClass, sco);
506522
maybeStartAppSec(scoClass, sco);
507523
maybeStartIast(scoClass, sco);
508524
maybeStartCiVisibility(instrumentation, scoClass, sco);
@@ -582,9 +598,9 @@ private static void maybeStartRemoteConfig(Class<?> scoClass, Object sco) {
582598
StaticEventLogger.end("Remote Config");
583599
}
584600

585-
private static synchronized void startDatadogAgent(final Instrumentation inst) {
601+
private static synchronized void startDatadogAgent(
602+
final InitializationTelemetry initTelemetry, final Instrumentation inst) {
586603
if (null != inst) {
587-
588604
StaticEventLogger.begin("BytebuddyAgent");
589605

590606
try {
@@ -595,13 +611,15 @@ private static synchronized void startDatadogAgent(final Instrumentation inst) {
595611
agentInstallerMethod.invoke(null, inst);
596612
} catch (final Throwable ex) {
597613
log.error("Throwable thrown while installing the Datadog Agent", ex);
614+
initTelemetry.onFatalError(ex);
615+
} finally {
616+
StaticEventLogger.end("BytebuddyAgent");
598617
}
599-
600-
StaticEventLogger.end("BytebuddyAgent");
601618
}
602619
}
603620

604-
private static synchronized void installDatadogTracer(Class<?> scoClass, Object sco) {
621+
private static synchronized void installDatadogTracer(
622+
InitializationTelemetry initTelemetry, Class<?> scoClass, Object sco) {
605623
if (AGENT_CLASSLOADER == null) {
606624
throw new IllegalStateException("Datadog agent should have been started already");
607625
}
@@ -622,6 +640,8 @@ private static synchronized void installDatadogTracer(Class<?> scoClass, Object
622640
throw ex;
623641
} catch (final Throwable ex) {
624642
log.error("Throwable thrown while installing the Datadog Tracer", ex);
643+
644+
initTelemetry.onFatalError(ex);
625645
}
626646

627647
StaticEventLogger.end("GlobalTracer");
Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
package datadog.trace.bootstrap;
2+
3+
import java.lang.invoke.MethodHandle;
4+
import java.lang.invoke.MethodHandles;
5+
import java.lang.invoke.MethodType;
6+
7+
/**
8+
* Thread safe wrapper around BootstrapInitializationTelemetry used inside the Datadog ClassLoader.
9+
*
10+
* <p>Right now, this is needed because of the build separation between the two portions of the
11+
* bootstrap. We should consider adjusting the build to allow Agent et al to reference
12+
* BootstrapInitializationTelemetry, then we could remove this proxy.
13+
*/
14+
public abstract class InitializationTelemetry {
15+
/** Returns a proxy around a BoostrapInitializationTelemetry object */
16+
public static final InitializationTelemetry proxy(Object bootstrapInitTelemetry) {
17+
if (bootstrapInitTelemetry == null) {
18+
return InitializationTelemetry.noOpInstance();
19+
} else {
20+
return new BootstrapProxy(bootstrapInitTelemetry);
21+
}
22+
}
23+
24+
/** Returns a singleton of the no op InitializationTelemetry */
25+
public static final InitializationTelemetry noOpInstance() {
26+
return NoOp.INSTANCE;
27+
}
28+
29+
/**
30+
* Indicates that an abort condition occurred during the bootstrapping process. Abort conditions
31+
* are assumed to leave the bootstrapping process incomplete. {@link #markIncomplete()}
32+
*/
33+
public abstract void onAbort(String reasonCode);
34+
35+
/**
36+
* Indicates that an exception occurred during the bootstrapping process By default the exception
37+
* is assumed to NOT have fully stopped the initialization of the tracer.
38+
*
39+
* <p>If this exception stops the core bootstrapping of the tracer, then {@link #markIncomplete()}
40+
* should also be called.
41+
*/
42+
public abstract void onError(Throwable t);
43+
44+
/**
45+
* Indicates an exception that occurred during the bootstrapping process that left initialization
46+
* incomplete. Equivalent to calling {@link #onError(Throwable)} and {@link #markIncomplete()}
47+
*/
48+
public abstract void onFatalError(Throwable t);
49+
50+
/**
51+
* Indicates that an exception conditional occurred during the bootstrapping process. By default
52+
* the exceptional condition is assumed to NOT have fully stopped the initialization of the
53+
* tracer.
54+
*
55+
* <p>If this exception stops the core bootstrapping of the tracer, then {@link #markIncomplete()}
56+
* should also be called.
57+
*/
58+
public abstract void onError(String reasonCode);
59+
60+
/**
61+
* Marks bootstrapping of tracer as an incomplete Should only be called when a core (e.g.
62+
* non-optional) component fails to initialize
63+
*/
64+
public abstract void markIncomplete();
65+
66+
/** No telemetry - used for delayed initialization outside bootstrap invocation */
67+
static final class NoOp extends InitializationTelemetry {
68+
static final NoOp INSTANCE = new NoOp();
69+
70+
NoOp() {}
71+
72+
@Override
73+
public void onAbort(String reasonCode) {}
74+
75+
@Override
76+
public void onError(String reasonCode) {}
77+
78+
@Override
79+
public void onError(Throwable t) {}
80+
81+
@Override
82+
public void onFatalError(Throwable t) {}
83+
84+
@Override
85+
public void markIncomplete() {}
86+
}
87+
88+
/** Reflective proxy to BootstrapInitializationTelemetry */
89+
static final class BootstrapProxy extends InitializationTelemetry {
90+
private final Object bootstrapInitTelemetry;
91+
private volatile MethodHandle bmh_onAbortString;
92+
private volatile MethodHandle bmh_onErrorString;
93+
private volatile MethodHandle bmh_onErrorThrowable;
94+
private volatile MethodHandle bmh_onFatalErrorThrowable;
95+
private volatile MethodHandle bmh_markIncomplete;
96+
97+
// DQH - Decided not to eager access MethodHandles, since exceptions are uncommon
98+
// However, MethodHandles are cached on lookup
99+
100+
/** @param bootstrapInitTelemetry - non-null BootstrapInitializationTelemetry */
101+
BootstrapProxy(final Object bootstrapInitTelemetry) {
102+
this.bootstrapInitTelemetry = bootstrapInitTelemetry;
103+
}
104+
105+
@Override
106+
public void onAbort(String reasonCode) {
107+
MethodHandle bmh_onAbortString = this.bmh_onAbortString;
108+
if (bmh_onAbortString == null) {
109+
bmh_onAbortString = findBoundHandle("onAbort", String.class);
110+
this.bmh_onAbortString = bmh_onAbortString;
111+
}
112+
if (bmh_onAbortString != null) {
113+
try {
114+
bmh_onAbortString.invokeExact(reasonCode);
115+
} catch (Throwable t) {
116+
// ignore
117+
}
118+
}
119+
}
120+
121+
@Override
122+
public void onError(String reasonCode) {
123+
MethodHandle bmh_onErrorString = this.bmh_onErrorString;
124+
if (bmh_onErrorString == null) {
125+
bmh_onErrorString = findBoundHandle("onError", String.class);
126+
this.bmh_onErrorString = bmh_onErrorString;
127+
}
128+
if (bmh_onErrorString != null) {
129+
try {
130+
bmh_onErrorString.invokeExact(reasonCode);
131+
} catch (Throwable t) {
132+
// ignore
133+
}
134+
}
135+
}
136+
137+
@Override
138+
public void onError(Throwable cause) {
139+
MethodHandle bmh_onErrorThrowable = this.bmh_onErrorThrowable;
140+
if (bmh_onErrorThrowable == null) {
141+
bmh_onErrorThrowable = findBoundHandle("onError", Throwable.class);
142+
this.bmh_onErrorThrowable = bmh_onErrorThrowable;
143+
}
144+
if (bmh_onErrorThrowable != null) {
145+
try {
146+
bmh_onErrorThrowable.invokeExact(cause);
147+
} catch (Throwable t) {
148+
// ignore
149+
}
150+
}
151+
}
152+
153+
@Override
154+
public void onFatalError(Throwable cause) {
155+
MethodHandle bmh_onFatalErrorThrowable = this.bmh_onFatalErrorThrowable;
156+
if (bmh_onFatalErrorThrowable == null) {
157+
bmh_onFatalErrorThrowable = findBoundHandle("onFatalError", Throwable.class);
158+
this.bmh_onFatalErrorThrowable = bmh_onFatalErrorThrowable;
159+
}
160+
if (bmh_onFatalErrorThrowable != null) {
161+
try {
162+
bmh_onFatalErrorThrowable.invokeExact(cause);
163+
} catch (Throwable t) {
164+
// ignore
165+
}
166+
}
167+
}
168+
169+
@Override
170+
public void markIncomplete() {
171+
MethodHandle bmh_markIncomplete = this.bmh_markIncomplete;
172+
if (bmh_markIncomplete == null) {
173+
bmh_markIncomplete = findBoundHandle("markIncomplete");
174+
this.bmh_markIncomplete = bmh_markIncomplete;
175+
}
176+
if (bmh_markIncomplete != null) {
177+
try {
178+
bmh_markIncomplete.invokeExact();
179+
} catch (Throwable t) {
180+
// ignore
181+
}
182+
}
183+
}
184+
185+
private final MethodHandle findBoundHandle(String name, Class<?> paramType) {
186+
try {
187+
MethodHandle virtualHandle =
188+
MethodHandles.publicLookup()
189+
.findVirtual(
190+
bootstrapInitTelemetry.getClass(),
191+
name,
192+
MethodType.methodType(void.class, paramType));
193+
194+
return virtualHandle.bindTo(bootstrapInitTelemetry);
195+
} catch (NoSuchMethodException | IllegalAccessException e) {
196+
return null;
197+
}
198+
}
199+
200+
private final MethodHandle findBoundHandle(String name) {
201+
try {
202+
MethodHandle virtualHandle =
203+
MethodHandles.publicLookup()
204+
.findVirtual(
205+
bootstrapInitTelemetry.getClass(), name, MethodType.methodType(void.class));
206+
207+
return virtualHandle.bindTo(bootstrapInitTelemetry);
208+
} catch (NoSuchMethodException | IllegalAccessException e) {
209+
return null;
210+
}
211+
}
212+
}
213+
}
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
package datadog.trace.bootstrap
2+
3+
import datadog.trace.test.util.DDSpecification
4+
5+
class InitializationTelemetryTest extends DDSpecification {
6+
def "telemetry wrapper - null case"() {
7+
expect:
8+
InitializationTelemetry.proxy(null) == InitializationTelemetry.noOpInstance()
9+
}
10+
11+
def "telemetry wrapper - wrap bootstrap"() {
12+
// TODO: Figure out how to test the wrapper fully
13+
expect:
14+
InitializationTelemetry.proxy(new Object()) != InitializationTelemetry.noOpInstance()
15+
}
16+
}

dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/AgentCLI.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import datadog.trace.agent.tooling.bytebuddy.SharedTypePools;
66
import datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers;
77
import datadog.trace.bootstrap.Agent;
8+
import datadog.trace.bootstrap.InitializationTelemetry;
89
import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
910
import datadog.trace.bootstrap.instrumentation.api.AgentTracer;
1011
import java.io.File;
@@ -53,7 +54,7 @@ public static void printIntegrationNames() {
5354
* @param interval the interval (in seconds) to wait for each trace
5455
*/
5556
public static void sendSampleTraces(final int count, final double interval) throws Exception {
56-
Agent.startDatadogTracer();
57+
Agent.startDatadogTracer(InitializationTelemetry.noOpInstance());
5758

5859
int numTraces = 0;
5960
while (++numTraces <= count || count < 0) {

0 commit comments

Comments
 (0)