Fix flaky live metrics smoke test #4275
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -39,6 +39,7 @@ | |
| import java.util.function.Predicate; | ||
| import java.util.stream.Collectors; | ||
| import javax.annotation.Nullable; | ||
| import org.awaitility.core.ConditionTimeoutException; | ||
| import org.junit.jupiter.api.extension.AfterAllCallback; | ||
| import org.junit.jupiter.api.extension.AfterEachCallback; | ||
| import org.junit.jupiter.api.extension.BeforeAllCallback; | ||
|
|
@@ -75,8 +76,7 @@ public class SmokeTestExtension | |
| private static final File appFile = new File(System.getProperty("ai.smoke-test.test-app-file")); | ||
|
|
||
| // TODO (trask) make private and expose methods on AiSmokeTest(?) | ||
| protected final MockedAppInsightsIngestionServer mockedIngestion; | ||
|
|
||
| protected final MockedOtlpIngestionServer mockedOtlpIngestion = new MockedOtlpIngestionServer(); | ||
|
|
||
|
|
@@ -167,6 +167,8 @@ public static SmokeTestExtensionBuilder builder() { | |
| this.jvmArgs = jvmArgs; | ||
| this.useDefaultHttpPort = useDefaultHttpPort; | ||
| this.useOtlpEndpoint = useOtlpEndpoint; | ||
|
|
||
| mockedIngestion = new MockedAppInsightsIngestionServer(useOld3xAgent); | ||
| } | ||
|
|
||
| private static String getProfilerEndpoint(ProfilerState profilerState) { | ||
|
|
@@ -281,9 +283,24 @@ protected String getAppContext() { | |
|
|
||
| private void clearOutAnyInitLogs() throws Exception { | ||
| if (!skipHealthCheck) { | ||
| if (!useOld3xAgent) { | ||
| await().until(mockedIngestion::isReceivingLiveMetrics); | ||
| } | ||
| String contextRootUrl = getBaseUrl() + "/"; | ||
| HttpHelper.getResponseCodeEnsuringSampled(contextRootUrl); | ||
| waitForHealthCheckTelemetry(contextRootUrl); | ||
| if (!useOld3xAgent) { | ||
| try { | ||
| await() | ||
| .untilAsserted( | ||
| () -> | ||
| assertThat(mockedIngestion.getLiveMetrics().getRequestCount(contextRootUrl)) | ||
| .isEqualTo(1)); | ||
| } catch (ConditionTimeoutException e) { | ||
| // TODO (trask) need to fix race condition in live metrics | ||
| // where sometimes it loses telemetry | ||
|
Comment on lines
+300
to
+301
There was a problem hiding this comment. I'm working on a separate fix for this in azure-sdk-for-java repo There was a problem hiding this comment. Azure/azure-sdk-for-java#45944 I will test it in a separate PR by removing this catch (after this PR is merged) |
||
| } | ||
| } | ||
| System.out.println("Clearing any RequestData from health check."); | ||
| mockedIngestion.resetData(); | ||
| } | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,132 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License. | ||
|
|
||
| package com.microsoft.applicationinsights.smoketest.fakeingestion; | ||
|
|
||
| import static org.assertj.core.api.Assertions.assertThat; | ||
|
|
||
| import com.azure.json.JsonProviders; | ||
| import com.azure.json.JsonReader; | ||
| import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.DocumentIngress; | ||
| import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.DocumentType; | ||
| import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.Exception; | ||
| import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.MetricPoint; | ||
| import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.MonitoringDataPoint; | ||
| import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.Request; | ||
| import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.Trace; | ||
| import java.io.IOException; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
|
|
||
| public class LiveMetricsVerifier { | ||
|
|
||
| private final List<MonitoringDataPoint> points = new ArrayList<>(); | ||
|
|
||
| public void apply(String postBody) throws IOException { | ||
| List<MonitoringDataPoint> dataPoints; | ||
| try (JsonReader reader = JsonProviders.createReader(postBody)) { | ||
| dataPoints = reader.readArray(MonitoringDataPoint::fromJson); | ||
| } | ||
|
|
||
| // Because the mock ping/posts should succeed, there should be one MonitoringDataPoint per post | ||
| assertThat(dataPoints).hasSize(1); | ||
| points.add(dataPoints.get(0)); | ||
| } | ||
|
|
||
| public int getRequestCountFromMetric() { | ||
| return getMetricCount("\\ApplicationInsights\\Requests/Sec"); | ||
| } | ||
|
|
||
| public int getDependencyCountFromMetric() { | ||
| return getMetricCount("\\ApplicationInsights\\Dependency Calls/Sec"); | ||
| } | ||
|
|
||
| public int getRequestCount(String url) { | ||
| int count = 0; | ||
| for (MonitoringDataPoint point : points) { | ||
| List<DocumentIngress> docs = point.getDocuments(); | ||
| for (DocumentIngress doc : docs) { | ||
| if (doc.getDocumentType().equals(DocumentType.REQUEST)) { | ||
| Request request = (Request) doc; | ||
| if (url.equals(request.getUrl())) { | ||
| count++; | ||
| } | ||
| } | ||
| } | ||
| } | ||
| return count; | ||
| } | ||
|
|
||
| public int getExceptionCount(String exceptionMessage) { | ||
| int count = 0; | ||
| for (MonitoringDataPoint point : points) { | ||
| List<DocumentIngress> docs = point.getDocuments(); | ||
| for (DocumentIngress doc : docs) { | ||
| if (doc.getDocumentType().equals(DocumentType.EXCEPTION)) { | ||
| Exception ex = (Exception) doc; | ||
| if (ex.getExceptionMessage().equals(exceptionMessage)) { | ||
| count++; | ||
| } | ||
| } | ||
| } | ||
| } | ||
| return count; | ||
| } | ||
|
|
||
| public int getTraceCount(String traceMessage) { | ||
| int count = 0; | ||
| for (MonitoringDataPoint point : points) { | ||
| List<DocumentIngress> docs = point.getDocuments(); | ||
| for (DocumentIngress doc : docs) { | ||
| if (doc.getDocumentType().equals(DocumentType.TRACE)) { | ||
| Trace trace = (Trace) doc; | ||
| if (trace.getMessage().equals(traceMessage)) { | ||
| count++; | ||
| } | ||
| } | ||
| } | ||
| } | ||
| return count; | ||
| } | ||
|
|
||
| public void confirmDocsAreFiltered() { | ||
| for (MonitoringDataPoint point : points) { | ||
| List<DocumentIngress> docs = point.getDocuments(); | ||
| for (DocumentIngress doc : docs) { | ||
| assertThat(doc.getDocumentType()).isNotEqualTo(DocumentType.REMOTE_DEPENDENCY); | ||
harsimar marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| } | ||
| } | ||
| } | ||
|
|
||
| public void confirmPerfCountersNonZero() { | ||
| for (MonitoringDataPoint point : points) { | ||
| List<MetricPoint> metrics = point.getMetrics(); | ||
| for (MetricPoint metric : metrics) { | ||
| String name = metric.getName(); | ||
| double value = metric.getValue(); | ||
| if (name.equals("\\Process\\Physical Bytes") | ||
| || name.equals("\\% Process\\Processor Time Normalized")) { | ||
| assertThat(value).isNotEqualTo(0); | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private int getMetricCount(String metricName) { | ||
| int count = 0; | ||
| for (MonitoringDataPoint point : points) { | ||
| List<MetricPoint> metrics = point.getMetrics(); | ||
| for (MetricPoint metric : metrics) { | ||
| String name = metric.getName(); | ||
| double value = metric.getValue(); | ||
| if (name.equals(metricName)) { | ||
| if (Math.floor(value) != value) { | ||
| throw new IllegalStateException("Not an integer: " + value); | ||
| } | ||
| count += (int) value; | ||
| } | ||
| } | ||
| } | ||
| return count; | ||
| } | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.