Skip to content

Commit 4393cd1

Browse files
trasktrask
authored
Add smoke test showing OOM due to recursive log capture on Wildfly (#3796)
1 parent cf2584f commit 4393cd1

File tree

8 files changed

+161
-5
lines changed

8 files changed

+161
-5
lines changed

‎agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentLogExporter.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,9 @@
1515
import com.microsoft.applicationinsights.agent.internal.telemetry.BatchItemProcessor;
1616
import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient;
1717
import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryObservers;
18+
import io.opentelemetry.api.logs.LoggerProvider;
1819
import io.opentelemetry.api.trace.SpanContext;
20+
import io.opentelemetry.javaagent.bootstrap.CallDepth;
1921
import io.opentelemetry.sdk.common.CompletableResultCode;
2022
import io.opentelemetry.sdk.logs.data.LogRecordData;
2123
import io.opentelemetry.sdk.logs.export.LogRecordExporter;
@@ -71,6 +73,23 @@ public void setSeverityThreshold(int severityThreshold) {
7173

7274
@Override
7375
public CompletableResultCode export(Collection<LogRecordData> logs) {
76+
// incrementing CallDepth for LoggerProvider causes the OpenTelemetry Java agent logging
77+
// instrumentation to back off
78+
//
79+
// note: recursive log capture is only known to be an issue on Wildfly, because it redirects
80+
// System.out back to a logging library which is itself instrumented by the Java agent
81+
//
82+
// see OutOfMemoryWithDebugLevelTest for repro that will fail without this code
83+
CallDepth callDepth = CallDepth.forClass(LoggerProvider.class);
84+
callDepth.getAndIncrement();
85+
try {
86+
return internalExport(logs);
87+
} finally {
88+
callDepth.decrementAndGet();
89+
}
90+
}
91+
92+
private CompletableResultCode internalExport(Collection<LogRecordData> logs) {
7493
if (TelemetryClient.getActive().getConnectionString() == null) {
7594
// Azure Functions consumption plan
7695
logger.debug("Instrumentation key is null or empty. Fail to export logs.");

‎agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/BatchItemProcessor.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@
99
import com.azure.monitor.opentelemetry.exporter.implementation.logging.OperationLogger;
1010
import com.azure.monitor.opentelemetry.exporter.implementation.models.TelemetryItem;
1111
import com.azure.monitor.opentelemetry.exporter.implementation.pipeline.TelemetryItemExporter;
12+
import io.opentelemetry.api.logs.LoggerProvider;
1213
import io.opentelemetry.internal.shaded.jctools.queues.MpscArrayQueue;
14+
import io.opentelemetry.javaagent.bootstrap.CallDepth;
1315
import io.opentelemetry.sdk.common.CompletableResultCode;
1416
import io.opentelemetry.sdk.internal.DaemonThreadFactory;
1517
import java.util.ArrayList;
@@ -169,6 +171,23 @@ private void addItem(TelemetryItem item) {
169171

170172
@Override
171173
public void run() {
174+
// incrementing CallDepth for LoggerProvider causes the OpenTelemetry Java agent logging
175+
// instrumentation to back off
176+
//
177+
// note: recursive log capture is only known to be an issue on Wildfly, because it redirects
178+
// System.out back to a logging library which is itself instrumented by the Java agent
179+
//
180+
// see OutOfMemoryWithDebugLevelTest for repro that will fail without this code
181+
CallDepth callDepth = CallDepth.forClass(LoggerProvider.class);
182+
callDepth.getAndIncrement();
183+
try {
184+
internalRun();
185+
} finally {
186+
callDepth.decrementAndGet();
187+
}
188+
}
189+
190+
public void internalRun() {
172191
updateNextExportTime();
173192

174193
while (continueWork) {

‎settings.gradle.kts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@ hideFromDependabot(":smoke-tests:apps:OpenTelemetryApiLogBridge")
128128
hideFromDependabot(":smoke-tests:apps:OpenTelemetryMetric")
129129
hideFromDependabot(":smoke-tests:apps:OtelResourceCustomMetric")
130130
hideFromDependabot(":smoke-tests:apps:OtlpMetrics")
131+
hideFromDependabot(":smoke-tests:apps:OutOfMemoryWithDebugLevel")
131132
hideFromDependabot(":smoke-tests:apps:PreAggMetricsWithRoleNameOverridesAndSampling")
132133
hideFromDependabot(":smoke-tests:apps:PreferForwardedUrlScheme")
133134
hideFromDependabot(":smoke-tests:apps:RateLimitedSampling")

‎smoke-tests/apps/OutOfMemoryWithDebugLevel/build.gradle.kts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
plugins {
2+
id("ai.smoke-test-war")
3+
}

‎smoke-tests/apps/OutOfMemoryWithDebugLevel/src/main/java/com/microsoft/applicationinsights/smoketestapp/HealthCheckServlet.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
package com.microsoft.applicationinsights.smoketestapp;
5+
6+
import java.io.IOException;
7+
import javax.servlet.annotation.WebServlet;
8+
import javax.servlet.http.HttpServlet;
9+
import javax.servlet.http.HttpServletRequest;
10+
import javax.servlet.http.HttpServletResponse;
11+
12+
// this is used by the test infra in order to know when it's ok to start running the tests
13+
@WebServlet("")
14+
public class HealthCheckServlet extends HttpServlet {
15+
16+
@Override
17+
protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException {}
18+
}

‎smoke-tests/apps/OutOfMemoryWithDebugLevel/src/main/java/com/microsoft/applicationinsights/smoketestapp/OutOfMemoryWithDebugLevelServlet.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
package com.microsoft.applicationinsights.smoketestapp;
5+
6+
import java.io.IOException;
7+
import java.util.logging.Logger;
8+
import javax.servlet.ServletException;
9+
import javax.servlet.annotation.WebServlet;
10+
import javax.servlet.http.HttpServlet;
11+
import javax.servlet.http.HttpServletRequest;
12+
import javax.servlet.http.HttpServletResponse;
13+
14+
@WebServlet("/*")
15+
public class OutOfMemoryWithDebugLevelServlet extends HttpServlet {
16+
17+
private static final Logger logger = Logger.getLogger("smoketestapp");
18+
19+
@Override
20+
protected void doGet(HttpServletRequest req, HttpServletResponse resp)
21+
throws ServletException, IOException {
22+
23+
logger.info("hello");
24+
}
25+
}

‎smoke-tests/apps/OutOfMemoryWithDebugLevel/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/OutOfMemoryWithDebugLevelTest.java‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
package com.microsoft.applicationinsights.smoketest;
5+
6+
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_11;
7+
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_11_OPENJ9;
8+
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_17;
9+
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_21;
10+
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_8;
11+
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_8_OPENJ9;
12+
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.WILDFLY_13_JAVA_8;
13+
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.WILDFLY_13_JAVA_8_OPENJ9;
14+
import static java.util.concurrent.TimeUnit.NANOSECONDS;
15+
import static java.util.concurrent.TimeUnit.SECONDS;
16+
import static org.assertj.core.api.Assertions.assertThat;
17+
18+
import com.microsoft.applicationinsights.smoketest.schemav2.MessageData;
19+
import java.util.List;
20+
import org.junit.jupiter.api.Test;
21+
import org.junit.jupiter.api.extension.RegisterExtension;
22+
23+
@UseAgent
24+
abstract class OutOfMemoryWithDebugLevelTest {
25+
26+
@RegisterExtension
27+
static final SmokeTestExtension testing =
28+
SmokeTestExtension.builder()
29+
.setSelfDiagnosticsLevel(
30+
"debug") // intentionally running with selfDiagnosticLevel "debug"
31+
.build();
32+
33+
private static final int COUNT = 100;
34+
35+
@Test
36+
@TargetUri(value = "/test", callCount = COUNT)
37+
void test() throws Exception {
38+
long start = System.nanoTime();
39+
while (testing.mockedIngestion.getCountForType("RequestData") < COUNT
40+
&& NANOSECONDS.toSeconds(System.nanoTime() - start) < 10) {}
41+
// wait ten more seconds before checking that we didn't receive too many
42+
Thread.sleep(SECONDS.toMillis(10));
43+
44+
List<MessageData> messages = testing.mockedIngestion.getMessageDataInRequest(COUNT);
45+
46+
assertThat(messages).hasSize(COUNT);
47+
}
48+
49+
@Environment(TOMCAT_8_JAVA_8)
50+
static class Tomcat8Java8Test extends OutOfMemoryWithDebugLevelTest {}
51+
52+
@Environment(TOMCAT_8_JAVA_8_OPENJ9)
53+
static class Tomcat8Java8OpenJ9Test extends OutOfMemoryWithDebugLevelTest {}
54+
55+
@Environment(TOMCAT_8_JAVA_11)
56+
static class Tomcat8Java11Test extends OutOfMemoryWithDebugLevelTest {}
57+
58+
@Environment(TOMCAT_8_JAVA_11_OPENJ9)
59+
static class Tomcat8Java11OpenJ9Test extends OutOfMemoryWithDebugLevelTest {}
60+
61+
@Environment(TOMCAT_8_JAVA_17)
62+
static class Tomcat8Java17Test extends OutOfMemoryWithDebugLevelTest {}
63+
64+
@Environment(TOMCAT_8_JAVA_21)
65+
static class Tomcat8Java21Test extends OutOfMemoryWithDebugLevelTest {}
66+
67+
@Environment(WILDFLY_13_JAVA_8)
68+
static class Wildfly13Java8Test extends OutOfMemoryWithDebugLevelTest {}
69+
70+
@Environment(WILDFLY_13_JAVA_8_OPENJ9)
71+
static class Wildfly13Java8OpenJ9Test extends OutOfMemoryWithDebugLevelTest {}
72+
}

‎smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/fakeingestion/MockedAppInsightsIngestionServer.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -103,13 +103,12 @@ public <T extends Domain> List<T> getTelemetryDataByTypeInRequest(String type) {
103103

104104
public <T extends Domain> List<T> getMessageDataInRequest(int numItems)
105105
throws ExecutionException, InterruptedException, TimeoutException {
106-
List<Envelope> items = waitForItems("MessageData", numItems);
106+
List<Envelope> items =
107+
waitForItems("MessageData", e -> e.getTags().containsKey("ai.operation.id"), numItems);
107108
List<T> dataItems = new ArrayList<>();
108109
for (Envelope e : items) {
109-
if (e.getTags().containsKey("ai.operation.id")) {
110-
Data<T> dt = (Data<T>) e.getData();
111-
dataItems.add(dt.getBaseData());
112-
}
110+
Data<T> dt = (Data<T>) e.getData();
111+
dataItems.add(dt.getBaseData());
113112
}
114113
return dataItems;
115114
}

0 commit comments

Comments
 (0)