Skip to content

Commit ad8756f

Browse files
trasktrask
authored
Fix flaky live metrics smoke test (#4275)
1 parent 7ac0007 commit ad8756f

File tree

7 files changed

+207
-161
lines changed

7 files changed

+207
-161
lines changed

‎smoke-tests/apps/AzureSdk/build.gradle.kts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,5 @@ dependencies {
77
exclude("org.springframework.boot", "spring-boot-starter-tomcat")
88
}
99
// want to test with one of the earliest version supported, and not managed version used in agent
10-
implementation(enforcedPlatform("com.azure:azure-sdk-bom:1.2.13"))
11-
implementation("com.azure:azure-core")
10+
implementation("com.azure:azure-core:1.39.0")
1211
}

‎smoke-tests/apps/LiveMetrics/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/LiveMetricsTest.java‎

Lines changed: 16 additions & 136 deletions
Original file line numberDiff line numberDiff line change
@@ -17,20 +17,10 @@
1717
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.WILDFLY_13_JAVA_8;
1818
import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.WILDFLY_13_JAVA_8_OPENJ9;
1919
import static org.assertj.core.api.Assertions.assertThat;
20+
import static org.awaitility.Awaitility.await;
2021

21-
import com.azure.json.JsonProviders;
22-
import com.azure.json.JsonReader;
23-
import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.DocumentIngress;
24-
import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.DocumentType;
25-
import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.Exception;
26-
import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.MetricPoint;
27-
import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.MonitoringDataPoint;
28-
import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.Trace;
29-
import java.io.IOException;
22+
import com.microsoft.applicationinsights.smoketest.fakeingestion.LiveMetricsVerifier;
3023
import java.time.Duration;
31-
import java.util.ArrayList;
32-
import java.util.List;
33-
import org.awaitility.Awaitility;
3424
import org.junit.jupiter.api.Test;
3525
import org.junit.jupiter.api.extension.RegisterExtension;
3626

@@ -41,135 +31,25 @@ abstract class LiveMetricsTest {
4131

4232
@Test
4333
@TargetUri("/test")
44-
void testTelemetryDataFlow() throws java.lang.Exception {
45-
Awaitility.await()
34+
void testTelemetryDataFlow() {
35+
await()
4636
.atMost(Duration.ofSeconds(60))
4737
.until(() -> testing.mockedIngestion.getCountForType("RequestData") == 1);
4838

49-
PostBodyVerifier postBodyVerifier = new PostBodyVerifier();
39+
await()
40+
.untilAsserted(
41+
() -> {
42+
LiveMetricsVerifier verifier = testing.mockedIngestion.getLiveMetrics();
5043

51-
assertThat(testing.mockedIngestion.isPingReceived()).isTrue();
44+
verifier.confirmDocsAreFiltered();
45+
verifier.confirmPerfCountersNonZero();
5246

53-
List<String> postBodies = testing.mockedIngestion.getPostBodies();
54-
assertThat(postBodies).hasSizeGreaterThan(0); // should post at least once
55-
56-
for (String postBody : postBodies) {
57-
postBodyVerifier.searchPostBody(postBody);
58-
}
59-
60-
assertThat(postBodyVerifier.hasExceptionDoc()).isTrue();
61-
assertThat(postBodyVerifier.hasTraceDoc()).isTrue();
62-
assertThat(postBodyVerifier.hasDependency()).isTrue();
63-
assertThat(postBodyVerifier.hasRequest()).isTrue();
64-
}
65-
66-
class PostBodyVerifier {
67-
boolean foundExceptionDoc = false;
68-
boolean foundTraceDoc = false;
69-
boolean foundDependency = false;
70-
boolean foundRequest = false;
71-
72-
public void searchPostBody(String postBody) {
73-
// Each post body is a list with a singular MonitoringDataPoint
74-
List<MonitoringDataPoint> dataPoints = new ArrayList<>();
75-
try {
76-
JsonReader reader = JsonProviders.createReader(postBody);
77-
dataPoints = reader.readArray(MonitoringDataPoint::fromJson);
78-
} catch (IOException e) {
79-
throw new IllegalStateException("Failed to parse post request body", e);
80-
}
81-
82-
// Because the mock ping/posts should succeed, we should only have one MonitoringDataPoint per
83-
// post
84-
assertThat(dataPoints).hasSize(1);
85-
MonitoringDataPoint dataPoint = dataPoints.get(0);
86-
87-
List<DocumentIngress> docs = dataPoint.getDocuments();
88-
List<MetricPoint> metrics = dataPoint.getMetrics();
89-
90-
confirmDocsAreFiltered(docs);
91-
confirmPerfCountersNonZero(metrics);
92-
foundExceptionDoc = foundExceptionDoc || hasException(docs);
93-
foundTraceDoc = foundTraceDoc || hasTrace(docs);
94-
foundDependency = foundDependency || hasDependency(metrics);
95-
foundRequest = foundRequest || hasRequest(metrics);
96-
}
97-
98-
public boolean hasExceptionDoc() {
99-
return foundExceptionDoc;
100-
}
101-
102-
public boolean hasTraceDoc() {
103-
return foundTraceDoc;
104-
}
105-
106-
public boolean hasDependency() {
107-
return foundDependency;
108-
}
109-
110-
public boolean hasRequest() {
111-
return foundRequest;
112-
}
113-
114-
private void confirmDocsAreFiltered(List<DocumentIngress> docs) {
115-
for (DocumentIngress doc : docs) {
116-
assertThat(doc.getDocumentType()).isNotEqualTo(DocumentType.REMOTE_DEPENDENCY);
117-
assertThat(doc.getDocumentType()).isNotEqualTo(DocumentType.REQUEST);
118-
}
119-
}
120-
121-
private boolean hasException(List<DocumentIngress> docs) {
122-
for (DocumentIngress doc : docs) {
123-
if (doc.getDocumentType().equals(DocumentType.EXCEPTION)
124-
&& ((Exception) doc).getExceptionMessage().equals("Fake Exception")) {
125-
return true;
126-
}
127-
}
128-
return false;
129-
}
130-
131-
private boolean hasTrace(List<DocumentIngress> docs) {
132-
for (DocumentIngress doc : docs) {
133-
if (doc.getDocumentType().equals(DocumentType.TRACE)
134-
&& ((Trace) doc).getMessage().equals("This message should generate a trace")) {
135-
return true;
136-
}
137-
}
138-
return false;
139-
}
140-
141-
private boolean hasDependency(List<MetricPoint> metrics) {
142-
for (MetricPoint metric : metrics) {
143-
String name = metric.getName();
144-
double value = metric.getValue();
145-
if (name.equals("\\ApplicationInsights\\Dependency Calls/Sec")) {
146-
return value == 1;
147-
}
148-
}
149-
return false;
150-
}
151-
152-
private boolean hasRequest(List<MetricPoint> metrics) {
153-
for (MetricPoint metric : metrics) {
154-
String name = metric.getName();
155-
double value = metric.getValue();
156-
if (name.equals("\\ApplicationInsights\\Requests/Sec")) {
157-
return value == 1;
158-
}
159-
}
160-
return false;
161-
}
162-
163-
private void confirmPerfCountersNonZero(List<MetricPoint> metrics) {
164-
for (MetricPoint metric : metrics) {
165-
String name = metric.getName();
166-
double value = metric.getValue();
167-
if (name.equals("\\Process\\Physical Bytes")
168-
|| name.equals("\\% Process\\Processor Time Normalized")) {
169-
assertThat(value).isNotEqualTo(0);
170-
}
171-
}
172-
}
47+
assertThat(verifier.getExceptionCount("Fake Exception")).isEqualTo(1);
48+
assertThat(verifier.getTraceCount("This message should generate a trace"))
49+
.isEqualTo(1);
50+
assertThat(verifier.getDependencyCountFromMetric()).isEqualTo(1);
51+
assertThat(verifier.getRequestCountFromMetric()).isEqualTo(1);
52+
});
17353
}
17454

17555
@Environment(TOMCAT_8_JAVA_8)

‎smoke-tests/framework/build.gradle.kts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,4 +29,7 @@ dependencies {
2929
implementation("ch.qos.logback:logback-classic")
3030

3131
implementation("org.assertj:assertj-core")
32+
33+
implementation("com.azure:azure-json:1.0.0")
34+
implementation("com.azure:azure-monitor-opentelemetry-autoconfigure:1.1.0")
3235
}

‎smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/SmokeTestExtension.java‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import java.util.function.Predicate;
4040
import java.util.stream.Collectors;
4141
import javax.annotation.Nullable;
42+
import org.awaitility.core.ConditionTimeoutException;
4243
import org.junit.jupiter.api.extension.AfterAllCallback;
4344
import org.junit.jupiter.api.extension.AfterEachCallback;
4445
import org.junit.jupiter.api.extension.BeforeAllCallback;
@@ -75,8 +76,7 @@ public class SmokeTestExtension
7576
private static final File appFile = new File(System.getProperty("ai.smoke-test.test-app-file"));
7677

7778
// TODO (trask) make private and expose methods on AiSmokeTest(?)
78-
protected final MockedAppInsightsIngestionServer mockedIngestion =
79-
new MockedAppInsightsIngestionServer();
79+
protected final MockedAppInsightsIngestionServer mockedIngestion;
8080

8181
protected final MockedOtlpIngestionServer mockedOtlpIngestion = new MockedOtlpIngestionServer();
8282

@@ -167,6 +167,8 @@ public static SmokeTestExtensionBuilder builder() {
167167
this.jvmArgs = jvmArgs;
168168
this.useDefaultHttpPort = useDefaultHttpPort;
169169
this.useOtlpEndpoint = useOtlpEndpoint;
170+
171+
mockedIngestion = new MockedAppInsightsIngestionServer(useOld3xAgent);
170172
}
171173

172174
private static String getProfilerEndpoint(ProfilerState profilerState) {
@@ -281,9 +283,24 @@ protected String getAppContext() {
281283

282284
private void clearOutAnyInitLogs() throws Exception {
283285
if (!skipHealthCheck) {
286+
if (!useOld3xAgent) {
287+
await().until(mockedIngestion::isReceivingLiveMetrics);
288+
}
284289
String contextRootUrl = getBaseUrl() + "/";
285290
HttpHelper.getResponseCodeEnsuringSampled(contextRootUrl);
286291
waitForHealthCheckTelemetry(contextRootUrl);
292+
if (!useOld3xAgent) {
293+
try {
294+
await()
295+
.untilAsserted(
296+
() ->
297+
assertThat(mockedIngestion.getLiveMetrics().getRequestCount(contextRootUrl))
298+
.isEqualTo(1));
299+
} catch (ConditionTimeoutException e) {
300+
// TODO (trask) need to fix race condition in live metrics
301+
// where sometimes it loses telemetry
302+
}
303+
}
287304
System.out.println("Clearing any RequestData from health check.");
288305
mockedIngestion.resetData();
289306
}

‎smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/fakeingestion/LiveMetricsVerifier.java‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
package com.microsoft.applicationinsights.smoketest.fakeingestion;
5+
6+
import static org.assertj.core.api.Assertions.assertThat;
7+
8+
import com.azure.json.JsonProviders;
9+
import com.azure.json.JsonReader;
10+
import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.DocumentIngress;
11+
import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.DocumentType;
12+
import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.Exception;
13+
import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.MetricPoint;
14+
import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.MonitoringDataPoint;
15+
import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.Request;
16+
import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.Trace;
17+
import java.io.IOException;
18+
import java.util.ArrayList;
19+
import java.util.List;
20+
21+
public class LiveMetricsVerifier {
22+
23+
private final List<MonitoringDataPoint> points = new ArrayList<>();
24+
25+
public void apply(String postBody) throws IOException {
26+
List<MonitoringDataPoint> dataPoints;
27+
try (JsonReader reader = JsonProviders.createReader(postBody)) {
28+
dataPoints = reader.readArray(MonitoringDataPoint::fromJson);
29+
}
30+
31+
// Because the mock ping/posts should succeed, there should be one MonitoringDataPoint per post
32+
assertThat(dataPoints).hasSize(1);
33+
points.add(dataPoints.get(0));
34+
}
35+
36+
public int getRequestCountFromMetric() {
37+
return getMetricCount("\\ApplicationInsights\\Requests/Sec");
38+
}
39+
40+
public int getDependencyCountFromMetric() {
41+
return getMetricCount("\\ApplicationInsights\\Dependency Calls/Sec");
42+
}
43+
44+
public int getRequestCount(String url) {
45+
int count = 0;
46+
for (MonitoringDataPoint point : points) {
47+
List<DocumentIngress> docs = point.getDocuments();
48+
for (DocumentIngress doc : docs) {
49+
if (doc.getDocumentType().equals(DocumentType.REQUEST)) {
50+
Request request = (Request) doc;
51+
if (url.equals(request.getUrl())) {
52+
count++;
53+
}
54+
}
55+
}
56+
}
57+
return count;
58+
}
59+
60+
public int getExceptionCount(String exceptionMessage) {
61+
int count = 0;
62+
for (MonitoringDataPoint point : points) {
63+
List<DocumentIngress> docs = point.getDocuments();
64+
for (DocumentIngress doc : docs) {
65+
if (doc.getDocumentType().equals(DocumentType.EXCEPTION)) {
66+
Exception ex = (Exception) doc;
67+
if (ex.getExceptionMessage().equals(exceptionMessage)) {
68+
count++;
69+
}
70+
}
71+
}
72+
}
73+
return count;
74+
}
75+
76+
public int getTraceCount(String traceMessage) {
77+
int count = 0;
78+
for (MonitoringDataPoint point : points) {
79+
List<DocumentIngress> docs = point.getDocuments();
80+
for (DocumentIngress doc : docs) {
81+
if (doc.getDocumentType().equals(DocumentType.TRACE)) {
82+
Trace trace = (Trace) doc;
83+
if (trace.getMessage().equals(traceMessage)) {
84+
count++;
85+
}
86+
}
87+
}
88+
}
89+
return count;
90+
}
91+
92+
public void confirmDocsAreFiltered() {
93+
for (MonitoringDataPoint point : points) {
94+
List<DocumentIngress> docs = point.getDocuments();
95+
for (DocumentIngress doc : docs) {
96+
assertThat(doc.getDocumentType()).isNotEqualTo(DocumentType.REMOTE_DEPENDENCY);
97+
}
98+
}
99+
}
100+
101+
public void confirmPerfCountersNonZero() {
102+
for (MonitoringDataPoint point : points) {
103+
List<MetricPoint> metrics = point.getMetrics();
104+
for (MetricPoint metric : metrics) {
105+
String name = metric.getName();
106+
double value = metric.getValue();
107+
if (name.equals("\\Process\\Physical Bytes")
108+
|| name.equals("\\% Process\\Processor Time Normalized")) {
109+
assertThat(value).isNotEqualTo(0);
110+
}
111+
}
112+
}
113+
}
114+
115+
private int getMetricCount(String metricName) {
116+
int count = 0;
117+
for (MonitoringDataPoint point : points) {
118+
List<MetricPoint> metrics = point.getMetrics();
119+
for (MetricPoint metric : metrics) {
120+
String name = metric.getName();
121+
double value = metric.getValue();
122+
if (name.equals(metricName)) {
123+
if (Math.floor(value) != value) {
124+
throw new IllegalStateException("Not an integer: " + value);
125+
}
126+
count += (int) value;
127+
}
128+
}
129+
}
130+
return count;
131+
}
132+
}

0 commit comments

Comments
 (0)