Skip to content

Commit 5fe236b

Browse files
steveraolaurit
steverao
and
laurit
authored
Add GC cause to jvm GC related metric (#13750)
Co-authored-by: Lauri Tulmin <[email protected]>
1 parent 7364927 commit 5fe236b

File tree

9 files changed

+96
-39
lines changed

9 files changed

+96
-39
lines changed

‎instrumentation/runtime-telemetry/README.md‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
# Settings for the Runtime Telemetry instrumentation
22

3-
| System property | Type | Default | Description |
4-
|--------------------------------------------------------------------------|---------|---------|-------------------------------------------------------------------|
5-
| `otel.instrumentation.runtime-telemetry.emit-experimental-telemetry` | Boolean | `false` | Enable the capture of experimental metrics. |
6-
| `otel.instrumentation.runtime-telemetry-java17.enable-all` | Boolean | `false` | Enable the capture of all JFR based metrics. |
7-
| `otel.instrumentation.runtime-telemetry-java17.enabled` | Boolean | `false` | Enable the capture of JFR based metrics. |
8-
| `otel.instrumentation.runtime-telemetry.package-emitter.enabled` | Boolean | `false` | Enable creating events for JAR libraries used by the application. |
9-
| `otel.instrumentation.runtime-telemetry.package-emitter.jars-per-second` | Integer | 10 | The number of JAR files processed per second. |
3+
| System property | Type | Default | Description |
4+
|--------------------------------------------------------------------------|---------|---------|-----------------------------------------------------------------------------------|
5+
| `otel.instrumentation.runtime-telemetry.capture-gc-cause` | Boolean | `false` | Enable the capture of the jvm.gc.cause attribute with the jvm.gc.duration metric. |
6+
| `otel.instrumentation.runtime-telemetry.emit-experimental-telemetry` | Boolean | `false` | Enable the capture of experimental metrics. |
7+
| `otel.instrumentation.runtime-telemetry-java17.enable-all` | Boolean | `false` | Enable the capture of all JFR based metrics. |
8+
| `otel.instrumentation.runtime-telemetry-java17.enabled` | Boolean | `false` | Enable the capture of JFR based metrics. |
9+
| `otel.instrumentation.runtime-telemetry.package-emitter.enabled` | Boolean | `false` | Enable creating events for JAR libraries used by the application. |
10+
| `otel.instrumentation.runtime-telemetry.package-emitter.jars-per-second` | Integer | 10 | The number of JAR files processed per second. |

‎instrumentation/runtime-telemetry/runtime-telemetry-java17/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/java17/RuntimeMetricsBuilder.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ public final class RuntimeMetricsBuilder {
2222

2323
private boolean disableJmx = false;
2424
private boolean enableExperimentalJmxTelemetry = false;
25+
private boolean captureGcCause = false;
2526

2627
RuntimeMetricsBuilder(OpenTelemetry openTelemetry) {
2728
this.openTelemetry = openTelemetry;
@@ -81,13 +82,20 @@ public RuntimeMetricsBuilder enableExperimentalJmxTelemetry() {
8182
return this;
8283
}
8384

85+
/** Enable the capture of the jvm.gc.cause attribute with the jvm.gc.duration metric. */
86+
@CanIgnoreReturnValue
87+
public RuntimeMetricsBuilder captureGcCause() {
88+
captureGcCause = true;
89+
return this;
90+
}
91+
8492
/** Build and start an {@link RuntimeMetrics} with the config from this builder. */
8593
public RuntimeMetrics build() {
8694
List<AutoCloseable> observables =
8795
disableJmx
8896
? List.of()
8997
: JmxRuntimeMetricsFactory.buildObservables(
90-
openTelemetry, enableExperimentalJmxTelemetry);
98+
openTelemetry, enableExperimentalJmxTelemetry, captureGcCause);
9199
RuntimeMetrics.JfrRuntimeMetrics jfrRuntimeMetrics = buildJfrMetrics();
92100
return new RuntimeMetrics(openTelemetry, observables, jfrRuntimeMetrics);
93101
}

‎instrumentation/runtime-telemetry/runtime-telemetry-java17/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/java17/internal/RuntimeMetricsConfigUtil.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,10 @@ public static RuntimeMetrics configure(
4343
builder.enableExperimentalJmxTelemetry();
4444
}
4545

46+
if (config.getBoolean("otel.instrumentation.runtime-telemetry.capture-gc-cause", false)) {
47+
builder.captureGcCause();
48+
}
49+
4650
return builder.build();
4751
}
4852
}

‎instrumentation/runtime-telemetry/runtime-telemetry-java8/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/java8/GarbageCollector.java‎

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,9 @@
1010

1111
import com.sun.management.GarbageCollectionNotificationInfo;
1212
import io.opentelemetry.api.OpenTelemetry;
13+
import io.opentelemetry.api.common.AttributeKey;
1314
import io.opentelemetry.api.common.Attributes;
15+
import io.opentelemetry.api.common.AttributesBuilder;
1416
import io.opentelemetry.api.metrics.DoubleHistogram;
1517
import io.opentelemetry.api.metrics.Meter;
1618
import io.opentelemetry.instrumentation.runtimemetrics.java8.internal.JmxRuntimeMetricsUtil;
@@ -55,14 +57,17 @@ public final class GarbageCollector {
5557

5658
static final List<Double> GC_DURATION_BUCKETS = unmodifiableList(asList(0.01, 0.1, 1., 10.));
5759

60+
private static final AttributeKey<String> JVM_GC_CAUSE = AttributeKey.stringKey("jvm.gc.cause");
61+
5862
private static final NotificationFilter GC_FILTER =
5963
notification ->
6064
notification
6165
.getType()
6266
.equals(GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION);
6367

6468
/** Register observers for java runtime memory metrics. */
65-
public static List<AutoCloseable> registerObservers(OpenTelemetry openTelemetry) {
69+
public static List<AutoCloseable> registerObservers(
70+
OpenTelemetry openTelemetry, boolean captureGcCause) {
6671
if (!isNotificationClassPresent()) {
6772
logger.fine(
6873
"The com.sun.management.GarbageCollectionNotificationInfo class is not available;"
@@ -73,14 +78,16 @@ public static List<AutoCloseable> registerObservers(OpenTelemetry openTelemetry)
7378
return registerObservers(
7479
openTelemetry,
7580
ManagementFactory.getGarbageCollectorMXBeans(),
76-
GarbageCollector::extractNotificationInfo);
81+
GarbageCollector::extractNotificationInfo,
82+
captureGcCause);
7783
}
7884

7985
// Visible for testing
8086
static List<AutoCloseable> registerObservers(
8187
OpenTelemetry openTelemetry,
8288
List<GarbageCollectorMXBean> gcBeans,
83-
Function<Notification, GarbageCollectionNotificationInfo> notificationInfoExtractor) {
89+
Function<Notification, GarbageCollectionNotificationInfo> notificationInfoExtractor,
90+
boolean captureGcCause) {
8491
Meter meter = JmxRuntimeMetricsUtil.getMeter(openTelemetry);
8592

8693
DoubleHistogram gcDuration =
@@ -98,7 +105,7 @@ static List<AutoCloseable> registerObservers(
98105
}
99106
NotificationEmitter notificationEmitter = (NotificationEmitter) gcBean;
100107
GcNotificationListener listener =
101-
new GcNotificationListener(gcDuration, notificationInfoExtractor);
108+
new GcNotificationListener(gcDuration, notificationInfoExtractor, captureGcCause);
102109
notificationEmitter.addNotificationListener(listener, GC_FILTER, null);
103110
result.add(() -> notificationEmitter.removeNotificationListener(listener));
104111
}
@@ -107,13 +114,16 @@ static List<AutoCloseable> registerObservers(
107114

108115
private static final class GcNotificationListener implements NotificationListener {
109116

117+
private final boolean captureGcCause;
110118
private final DoubleHistogram gcDuration;
111119
private final Function<Notification, GarbageCollectionNotificationInfo>
112120
notificationInfoExtractor;
113121

114122
private GcNotificationListener(
115123
DoubleHistogram gcDuration,
116-
Function<Notification, GarbageCollectionNotificationInfo> notificationInfoExtractor) {
124+
Function<Notification, GarbageCollectionNotificationInfo> notificationInfoExtractor,
125+
boolean captureGcCause) {
126+
this.captureGcCause = captureGcCause;
117127
this.gcDuration = gcDuration;
118128
this.notificationInfoExtractor = notificationInfoExtractor;
119129
}
@@ -126,10 +136,14 @@ public void handleNotification(Notification notification, Object unused) {
126136
String gcName = notificationInfo.getGcName();
127137
String gcAction = notificationInfo.getGcAction();
128138
double duration = notificationInfo.getGcInfo().getDuration() / MILLIS_PER_S;
129-
130-
gcDuration.record(
131-
duration,
132-
Attributes.of(JvmAttributes.JVM_GC_NAME, gcName, JvmAttributes.JVM_GC_ACTION, gcAction));
139+
AttributesBuilder builder = Attributes.builder();
140+
builder.put(JvmAttributes.JVM_GC_NAME, gcName);
141+
builder.put(JvmAttributes.JVM_GC_ACTION, gcAction);
142+
if (captureGcCause) {
143+
String gcCause = notificationInfo.getGcCause();
144+
builder.put(JVM_GC_CAUSE, gcCause);
145+
}
146+
gcDuration.record(duration, builder.build());
133147
}
134148
}
135149

‎instrumentation/runtime-telemetry/runtime-telemetry-java8/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/java8/RuntimeMetricsBuilder.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ public final class RuntimeMetricsBuilder {
1616
private final OpenTelemetry openTelemetry;
1717

1818
private boolean enableExperimentalJmxTelemetry = false;
19+
private boolean captureGcCause = false;
1920

2021
RuntimeMetricsBuilder(OpenTelemetry openTelemetry) {
2122
this.openTelemetry = openTelemetry;
@@ -28,10 +29,18 @@ public RuntimeMetricsBuilder enableExperimentalJmxTelemetry() {
2829
return this;
2930
}
3031

32+
/** Enable the capture of the jvm.gc.cause attribute with the jvm.gc.duration metric. */
33+
@CanIgnoreReturnValue
34+
public RuntimeMetricsBuilder captureGcCause() {
35+
captureGcCause = true;
36+
return this;
37+
}
38+
3139
/** Build and start an {@link RuntimeMetrics} with the config from this builder. */
3240
public RuntimeMetrics build() {
3341
List<AutoCloseable> observables =
34-
JmxRuntimeMetricsFactory.buildObservables(openTelemetry, enableExperimentalJmxTelemetry);
42+
JmxRuntimeMetricsFactory.buildObservables(
43+
openTelemetry, enableExperimentalJmxTelemetry, captureGcCause);
3544
return new RuntimeMetrics(observables);
3645
}
3746
}

‎instrumentation/runtime-telemetry/runtime-telemetry-java8/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/java8/internal/JmxRuntimeMetricsFactory.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,12 @@
2121
public class JmxRuntimeMetricsFactory {
2222
@SuppressWarnings("CatchingUnchecked")
2323
public static List<AutoCloseable> buildObservables(
24-
OpenTelemetry openTelemetry, boolean enableExperimentalJmxTelemetry) {
24+
OpenTelemetry openTelemetry, boolean enableExperimentalJmxTelemetry, boolean captureGcCause) {
2525
// Set up metrics gathered by JMX
2626
List<AutoCloseable> observables = new ArrayList<>();
2727
observables.addAll(Classes.registerObservers(openTelemetry));
2828
observables.addAll(Cpu.registerObservers(openTelemetry));
29-
observables.addAll(GarbageCollector.registerObservers(openTelemetry));
29+
observables.addAll(GarbageCollector.registerObservers(openTelemetry, captureGcCause));
3030
observables.addAll(MemoryPools.registerObservers(openTelemetry));
3131
observables.addAll(Threads.registerObservers(openTelemetry));
3232
if (enableExperimentalJmxTelemetry) {

‎instrumentation/runtime-telemetry/runtime-telemetry-java8/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/java8/internal/RuntimeMetricsConfigUtil.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,10 @@ public static RuntimeMetrics configure(
3131
builder.enableExperimentalJmxTelemetry();
3232
}
3333

34+
if (config.getBoolean("otel.instrumentation.runtime-telemetry.capture-gc-cause", false)) {
35+
builder.captureGcCause();
36+
}
37+
3438
return builder.build();
3539
}
3640
}

‎instrumentation/runtime-telemetry/runtime-telemetry-java8/library/src/test/java/io/opentelemetry/instrumentation/runtimemetrics/java8/GarbageCollectorTest.java‎

Lines changed: 30 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,12 @@
55

66
package io.opentelemetry.instrumentation.runtimemetrics.java8;
77

8+
import static io.opentelemetry.api.common.AttributeKey.stringKey;
89
import static io.opentelemetry.instrumentation.runtimemetrics.java8.ScopeUtil.EXPECTED_SCOPE;
910
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat;
11+
import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.equalTo;
12+
import static io.opentelemetry.semconv.JvmAttributes.JVM_GC_ACTION;
13+
import static io.opentelemetry.semconv.JvmAttributes.JVM_GC_NAME;
1014
import static java.util.Collections.singletonList;
1115
import static org.mockito.ArgumentMatchers.any;
1216
import static org.mockito.Mockito.mock;
@@ -15,17 +19,17 @@
1519

1620
import com.sun.management.GarbageCollectionNotificationInfo;
1721
import com.sun.management.GcInfo;
18-
import io.opentelemetry.api.common.Attributes;
1922
import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension;
2023
import io.opentelemetry.instrumentation.testing.junit.LibraryInstrumentationExtension;
2124
import java.lang.management.GarbageCollectorMXBean;
2225
import java.util.concurrent.atomic.AtomicLong;
2326
import javax.management.Notification;
2427
import javax.management.NotificationEmitter;
2528
import javax.management.NotificationListener;
26-
import org.junit.jupiter.api.Test;
2729
import org.junit.jupiter.api.extension.ExtendWith;
2830
import org.junit.jupiter.api.extension.RegisterExtension;
31+
import org.junit.jupiter.params.ParameterizedTest;
32+
import org.junit.jupiter.params.provider.ValueSource;
2933
import org.mockito.ArgumentCaptor;
3034
import org.mockito.Captor;
3135
import org.mockito.Mock;
@@ -48,23 +52,27 @@ class GarbageCollectorTest {
4852

4953
@Captor private ArgumentCaptor<NotificationListener> listenerCaptor;
5054

51-
@Test
52-
void registerObservers() {
55+
@ParameterizedTest
56+
@ValueSource(booleans = {true, false})
57+
void registerObservers(boolean captureGcCause) {
5358
GarbageCollector.registerObservers(
5459
testing.getOpenTelemetry(),
5560
singletonList(gcBean),
56-
GarbageCollectorTest::getGcNotificationInfo);
61+
GarbageCollectorTest::getGcNotificationInfo,
62+
captureGcCause);
5763

5864
NotificationEmitter notificationEmitter = (NotificationEmitter) gcBean;
5965
verify(notificationEmitter).addNotificationListener(listenerCaptor.capture(), any(), any());
6066
NotificationListener listener = listenerCaptor.getValue();
6167

6268
listener.handleNotification(
63-
createTestNotification("G1 Young Generation", "end of minor GC", 10), null);
69+
createTestNotification("G1 Young Generation", "end of minor GC", "Allocation Failure", 10),
70+
null);
6471
listener.handleNotification(
65-
createTestNotification("G1 Young Generation", "end of minor GC", 12), null);
72+
createTestNotification("G1 Young Generation", "end of minor GC", "Allocation Failure", 12),
73+
null);
6674
listener.handleNotification(
67-
createTestNotification("G1 Old Generation", "end of major GC", 11), null);
75+
createTestNotification("G1 Old Generation", "end of major GC", "System.gc()", 11), null);
6876

6977
testing.waitAndAssertMetrics(
7078
"io.opentelemetry.runtime-telemetry-java8",
@@ -83,30 +91,33 @@ void registerObservers() {
8391
point
8492
.hasCount(2)
8593
.hasSum(0.022)
86-
.hasAttributes(
87-
Attributes.builder()
88-
.put("jvm.gc.name", "G1 Young Generation")
89-
.put("jvm.gc.action", "end of minor GC")
90-
.build())
94+
.hasAttributesSatisfyingExactly(
95+
equalTo(JVM_GC_NAME, "G1 Young Generation"),
96+
equalTo(JVM_GC_ACTION, "end of minor GC"),
97+
equalTo(
98+
stringKey("jvm.gc.cause"),
99+
captureGcCause ? "Allocation Failure" : null))
91100
.hasBucketBoundaries(GC_DURATION_BUCKETS),
92101
point ->
93102
point
94103
.hasCount(1)
95104
.hasSum(0.011)
96-
.hasAttributes(
97-
Attributes.builder()
98-
.put("jvm.gc.name", "G1 Old Generation")
99-
.put("jvm.gc.action", "end of major GC")
100-
.build())
105+
.hasAttributesSatisfyingExactly(
106+
equalTo(JVM_GC_NAME, "G1 Old Generation"),
107+
equalTo(JVM_GC_ACTION, "end of major GC"),
108+
equalTo(
109+
stringKey("jvm.gc.cause"),
110+
captureGcCause ? "System.gc()" : null))
101111
.hasBucketBoundaries(GC_DURATION_BUCKETS)))));
102112
}
103113

104114
private static Notification createTestNotification(
105-
String gcName, String gcAction, long duration) {
115+
String gcName, String gcAction, String gcCause, long duration) {
106116
GarbageCollectionNotificationInfo gcNotificationInfo =
107117
mock(GarbageCollectionNotificationInfo.class);
108118
when(gcNotificationInfo.getGcName()).thenReturn(gcName);
109119
when(gcNotificationInfo.getGcAction()).thenReturn(gcAction);
120+
when(gcNotificationInfo.getGcCause()).thenReturn(gcCause);
110121
GcInfo gcInfo = mock(GcInfo.class);
111122
when(gcInfo.getDuration()).thenReturn(duration);
112123
when(gcNotificationInfo.getGcInfo()).thenReturn(gcInfo);

‎instrumentation/spring/spring-boot-autoconfigure/src/main/resources/META-INF/additional-spring-configuration-metadata.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -482,6 +482,12 @@
482482
"description": "Enables the DB statement sanitization.",
483483
"defaultValue": true
484484
},
485+
{
486+
"name": "otel.instrumentation.runtime-telemetry.capture-gc-cause",
487+
"type": "java.lang.Boolean",
488+
"description": "Enable the capture of the jvm.gc.cause attribute with the jvm.gc.duration metric.",
489+
"defaultValue": false
490+
},
485491
{
486492
"name": "otel.instrumentation.runtime-telemetry.enabled",
487493
"type": "java.lang.Boolean",

0 commit comments

Comments
 (0)