diff --git a/smoke-tests/apps/AzureSdk/build.gradle.kts b/smoke-tests/apps/AzureSdk/build.gradle.kts index 299fa5fb5cb..c1a2263a5fb 100644 --- a/smoke-tests/apps/AzureSdk/build.gradle.kts +++ b/smoke-tests/apps/AzureSdk/build.gradle.kts @@ -7,6 +7,5 @@ dependencies { exclude("org.springframework.boot", "spring-boot-starter-tomcat") } // want to test with one of the earliest version supported, and not managed version used in agent - implementation(enforcedPlatform("com.azure:azure-sdk-bom:1.2.13")) - implementation("com.azure:azure-core") + implementation("com.azure:azure-core:1.39.0") } diff --git a/smoke-tests/apps/LiveMetrics/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/LiveMetricsTest.java b/smoke-tests/apps/LiveMetrics/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/LiveMetricsTest.java index e4ed179055a..5445307ea16 100644 --- a/smoke-tests/apps/LiveMetrics/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/LiveMetricsTest.java +++ b/smoke-tests/apps/LiveMetrics/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/LiveMetricsTest.java @@ -17,20 +17,10 @@ import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.WILDFLY_13_JAVA_8; import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.WILDFLY_13_JAVA_8_OPENJ9; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; -import com.azure.json.JsonProviders; -import com.azure.json.JsonReader; -import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.DocumentIngress; -import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.DocumentType; -import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.Exception; -import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.MetricPoint; -import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.MonitoringDataPoint; -import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.Trace; -import java.io.IOException; +import com.microsoft.applicationinsights.smoketest.fakeingestion.LiveMetricsVerifier; import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import org.awaitility.Awaitility; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -41,135 +31,25 @@ abstract class LiveMetricsTest { @Test @TargetUri("/test") - void testTelemetryDataFlow() throws java.lang.Exception { - Awaitility.await() + void testTelemetryDataFlow() { + await() .atMost(Duration.ofSeconds(60)) .until(() -> testing.mockedIngestion.getCountForType("RequestData") == 1); - PostBodyVerifier postBodyVerifier = new PostBodyVerifier(); + await() + .untilAsserted( + () -> { + LiveMetricsVerifier verifier = testing.mockedIngestion.getLiveMetrics(); - assertThat(testing.mockedIngestion.isPingReceived()).isTrue(); + verifier.confirmDocsAreFiltered(); + verifier.confirmPerfCountersNonZero(); - List postBodies = testing.mockedIngestion.getPostBodies(); - assertThat(postBodies).hasSizeGreaterThan(0); // should post at least once - - for (String postBody : postBodies) { - postBodyVerifier.searchPostBody(postBody); - } - - assertThat(postBodyVerifier.hasExceptionDoc()).isTrue(); - assertThat(postBodyVerifier.hasTraceDoc()).isTrue(); - assertThat(postBodyVerifier.hasDependency()).isTrue(); - assertThat(postBodyVerifier.hasRequest()).isTrue(); - } - - class PostBodyVerifier { - boolean foundExceptionDoc = false; - boolean foundTraceDoc = false; - boolean foundDependency = false; - boolean foundRequest = false; - - public void searchPostBody(String postBody) { - // Each post body is a list with a singular MonitoringDataPoint - List dataPoints = new ArrayList<>(); - try { - JsonReader reader = JsonProviders.createReader(postBody); - dataPoints = reader.readArray(MonitoringDataPoint::fromJson); - } catch (IOException e) { - throw new IllegalStateException("Failed to parse post request body", e); - } - - // Because the mock ping/posts should succeed, we should only have one MonitoringDataPoint per - // post - assertThat(dataPoints).hasSize(1); - MonitoringDataPoint dataPoint = dataPoints.get(0); - - List docs = dataPoint.getDocuments(); - List metrics = dataPoint.getMetrics(); - - confirmDocsAreFiltered(docs); - confirmPerfCountersNonZero(metrics); - foundExceptionDoc = foundExceptionDoc || hasException(docs); - foundTraceDoc = foundTraceDoc || hasTrace(docs); - foundDependency = foundDependency || hasDependency(metrics); - foundRequest = foundRequest || hasRequest(metrics); - } - - public boolean hasExceptionDoc() { - return foundExceptionDoc; - } - - public boolean hasTraceDoc() { - return foundTraceDoc; - } - - public boolean hasDependency() { - return foundDependency; - } - - public boolean hasRequest() { - return foundRequest; - } - - private void confirmDocsAreFiltered(List docs) { - for (DocumentIngress doc : docs) { - assertThat(doc.getDocumentType()).isNotEqualTo(DocumentType.REMOTE_DEPENDENCY); - assertThat(doc.getDocumentType()).isNotEqualTo(DocumentType.REQUEST); - } - } - - private boolean hasException(List docs) { - for (DocumentIngress doc : docs) { - if (doc.getDocumentType().equals(DocumentType.EXCEPTION) - && ((Exception) doc).getExceptionMessage().equals("Fake Exception")) { - return true; - } - } - return false; - } - - private boolean hasTrace(List docs) { - for (DocumentIngress doc : docs) { - if (doc.getDocumentType().equals(DocumentType.TRACE) - && ((Trace) doc).getMessage().equals("This message should generate a trace")) { - return true; - } - } - return false; - } - - private boolean hasDependency(List metrics) { - for (MetricPoint metric : metrics) { - String name = metric.getName(); - double value = metric.getValue(); - if (name.equals("\\ApplicationInsights\\Dependency Calls/Sec")) { - return value == 1; - } - } - return false; - } - - private boolean hasRequest(List metrics) { - for (MetricPoint metric : metrics) { - String name = metric.getName(); - double value = metric.getValue(); - if (name.equals("\\ApplicationInsights\\Requests/Sec")) { - return value == 1; - } - } - return false; - } - - private void confirmPerfCountersNonZero(List metrics) { - for (MetricPoint metric : metrics) { - String name = metric.getName(); - double value = metric.getValue(); - if (name.equals("\\Process\\Physical Bytes") - || name.equals("\\% Process\\Processor Time Normalized")) { - assertThat(value).isNotEqualTo(0); - } - } - } + assertThat(verifier.getExceptionCount("Fake Exception")).isEqualTo(1); + assertThat(verifier.getTraceCount("This message should generate a trace")) + .isEqualTo(1); + assertThat(verifier.getDependencyCountFromMetric()).isEqualTo(1); + assertThat(verifier.getRequestCountFromMetric()).isEqualTo(1); + }); } @Environment(TOMCAT_8_JAVA_8) diff --git a/smoke-tests/framework/build.gradle.kts b/smoke-tests/framework/build.gradle.kts index 4b6ea5779ff..3858db0c6f8 100644 --- a/smoke-tests/framework/build.gradle.kts +++ b/smoke-tests/framework/build.gradle.kts @@ -29,4 +29,7 @@ dependencies { implementation("ch.qos.logback:logback-classic") implementation("org.assertj:assertj-core") + + implementation("com.azure:azure-json:1.0.0") + implementation("com.azure:azure-monitor-opentelemetry-autoconfigure:1.1.0") } diff --git a/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/SmokeTestExtension.java b/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/SmokeTestExtension.java index b5b876d0c0b..1a8891204f6 100644 --- a/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/SmokeTestExtension.java +++ b/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/SmokeTestExtension.java @@ -39,6 +39,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import javax.annotation.Nullable; +import org.awaitility.core.ConditionTimeoutException; import org.junit.jupiter.api.extension.AfterAllCallback; import org.junit.jupiter.api.extension.AfterEachCallback; import org.junit.jupiter.api.extension.BeforeAllCallback; @@ -75,8 +76,7 @@ public class SmokeTestExtension private static final File appFile = new File(System.getProperty("ai.smoke-test.test-app-file")); // TODO (trask) make private and expose methods on AiSmokeTest(?) - protected final MockedAppInsightsIngestionServer mockedIngestion = - new MockedAppInsightsIngestionServer(); + protected final MockedAppInsightsIngestionServer mockedIngestion; protected final MockedOtlpIngestionServer mockedOtlpIngestion = new MockedOtlpIngestionServer(); @@ -167,6 +167,8 @@ public static SmokeTestExtensionBuilder builder() { this.jvmArgs = jvmArgs; this.useDefaultHttpPort = useDefaultHttpPort; this.useOtlpEndpoint = useOtlpEndpoint; + + mockedIngestion = new MockedAppInsightsIngestionServer(useOld3xAgent); } private static String getProfilerEndpoint(ProfilerState profilerState) { @@ -281,9 +283,24 @@ protected String getAppContext() { private void clearOutAnyInitLogs() throws Exception { if (!skipHealthCheck) { + if (!useOld3xAgent) { + await().until(mockedIngestion::isReceivingLiveMetrics); + } String contextRootUrl = getBaseUrl() + "/"; HttpHelper.getResponseCodeEnsuringSampled(contextRootUrl); waitForHealthCheckTelemetry(contextRootUrl); + if (!useOld3xAgent) { + try { + await() + .untilAsserted( + () -> + assertThat(mockedIngestion.getLiveMetrics().getRequestCount(contextRootUrl)) + .isEqualTo(1)); + } catch (ConditionTimeoutException e) { + // TODO (trask) need to fix race condition in live metrics + // where sometimes it loses telemetry + } + } System.out.println("Clearing any RequestData from health check."); mockedIngestion.resetData(); } diff --git a/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/fakeingestion/LiveMetricsVerifier.java b/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/fakeingestion/LiveMetricsVerifier.java new file mode 100644 index 00000000000..4cda17a476b --- /dev/null +++ b/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/fakeingestion/LiveMetricsVerifier.java @@ -0,0 +1,132 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.smoketest.fakeingestion; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.azure.json.JsonProviders; +import com.azure.json.JsonReader; +import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.DocumentIngress; +import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.DocumentType; +import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.Exception; +import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.MetricPoint; +import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.MonitoringDataPoint; +import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.Request; +import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.swagger.models.Trace; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class LiveMetricsVerifier { + + private final List points = new ArrayList<>(); + + public void apply(String postBody) throws IOException { + List dataPoints; + try (JsonReader reader = JsonProviders.createReader(postBody)) { + dataPoints = reader.readArray(MonitoringDataPoint::fromJson); + } + + // Because the mock ping/posts should succeed, there should be one MonitoringDataPoint per post + assertThat(dataPoints).hasSize(1); + points.add(dataPoints.get(0)); + } + + public int getRequestCountFromMetric() { + return getMetricCount("\\ApplicationInsights\\Requests/Sec"); + } + + public int getDependencyCountFromMetric() { + return getMetricCount("\\ApplicationInsights\\Dependency Calls/Sec"); + } + + public int getRequestCount(String url) { + int count = 0; + for (MonitoringDataPoint point : points) { + List docs = point.getDocuments(); + for (DocumentIngress doc : docs) { + if (doc.getDocumentType().equals(DocumentType.REQUEST)) { + Request request = (Request) doc; + if (url.equals(request.getUrl())) { + count++; + } + } + } + } + return count; + } + + public int getExceptionCount(String exceptionMessage) { + int count = 0; + for (MonitoringDataPoint point : points) { + List docs = point.getDocuments(); + for (DocumentIngress doc : docs) { + if (doc.getDocumentType().equals(DocumentType.EXCEPTION)) { + Exception ex = (Exception) doc; + if (ex.getExceptionMessage().equals(exceptionMessage)) { + count++; + } + } + } + } + return count; + } + + public int getTraceCount(String traceMessage) { + int count = 0; + for (MonitoringDataPoint point : points) { + List docs = point.getDocuments(); + for (DocumentIngress doc : docs) { + if (doc.getDocumentType().equals(DocumentType.TRACE)) { + Trace trace = (Trace) doc; + if (trace.getMessage().equals(traceMessage)) { + count++; + } + } + } + } + return count; + } + + public void confirmDocsAreFiltered() { + for (MonitoringDataPoint point : points) { + List docs = point.getDocuments(); + for (DocumentIngress doc : docs) { + assertThat(doc.getDocumentType()).isNotEqualTo(DocumentType.REMOTE_DEPENDENCY); + } + } + } + + public void confirmPerfCountersNonZero() { + for (MonitoringDataPoint point : points) { + List metrics = point.getMetrics(); + for (MetricPoint metric : metrics) { + String name = metric.getName(); + double value = metric.getValue(); + if (name.equals("\\Process\\Physical Bytes") + || name.equals("\\% Process\\Processor Time Normalized")) { + assertThat(value).isNotEqualTo(0); + } + } + } + } + + private int getMetricCount(String metricName) { + int count = 0; + for (MonitoringDataPoint point : points) { + List metrics = point.getMetrics(); + for (MetricPoint metric : metrics) { + String name = metric.getName(); + double value = metric.getValue(); + if (name.equals(metricName)) { + if (Math.floor(value) != value) { + throw new IllegalStateException("Not an integer: " + value); + } + count += (int) value; + } + } + } + return count; + } +} diff --git a/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/fakeingestion/MockedAppInsightsIngestionServer.java b/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/fakeingestion/MockedAppInsightsIngestionServer.java index 7556f6c4451..bd62c9295d3 100644 --- a/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/fakeingestion/MockedAppInsightsIngestionServer.java +++ b/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/fakeingestion/MockedAppInsightsIngestionServer.java @@ -27,14 +27,14 @@ public class MockedAppInsightsIngestionServer { private final MockedQuickPulseServlet quickPulseServlet; private final Server server; - public MockedAppInsightsIngestionServer() { + public MockedAppInsightsIngestionServer(boolean usingOld3xAgent) { server = new Server(DEFAULT_PORT); ServletHandler handler = new ServletHandler(); server.setHandler(handler); servlet = new MockedAppInsightsIngestionServlet(); profilerSettingsServlet = new MockedProfilerSettingsServlet(); - quickPulseServlet = new MockedQuickPulseServlet(); + quickPulseServlet = new MockedQuickPulseServlet(usingOld3xAgent); handler.addServletWithMapping(new ServletHolder(profilerSettingsServlet), "/profiler/*"); handler.addServletWithMapping(new ServletHolder(quickPulseServlet), "/QuickPulseService.svc/*"); @@ -52,10 +52,12 @@ public void stopServer() throws Exception { System.out.println("Stopping fake Breeze ingestion..."); server.stop(); server.join(); + quickPulseServlet.resetData(); } public void resetData() { - this.servlet.resetData(); + servlet.resetData(); + quickPulseServlet.resetData(); } public boolean hasData() { @@ -283,17 +285,17 @@ public boolean test(Envelope input) { return items; } - public boolean isPingReceived() { - return quickPulseServlet.isPingReceived(); + public boolean isReceivingLiveMetrics() { + return quickPulseServlet.isReceivingLiveMetrics(); } - public List getPostBodies() { - return quickPulseServlet.getPostBodies(); + public LiveMetricsVerifier getLiveMetrics() { + return quickPulseServlet.getVerifier(); } @SuppressWarnings("SystemOut") public static void main(String[] args) throws Exception { - MockedAppInsightsIngestionServer i = new MockedAppInsightsIngestionServer(); + MockedAppInsightsIngestionServer i = new MockedAppInsightsIngestionServer(false); System.out.println("Starting mocked ingestion on port " + DEFAULT_PORT); Runtime.getRuntime() .addShutdownHook( diff --git a/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/fakeingestion/MockedQuickPulseServlet.java b/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/fakeingestion/MockedQuickPulseServlet.java index 654dfef505e..9cf2b631edd 100644 --- a/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/fakeingestion/MockedQuickPulseServlet.java +++ b/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/fakeingestion/MockedQuickPulseServlet.java @@ -6,8 +6,6 @@ import com.google.common.io.CharStreams; import java.io.IOException; import java.io.StringWriter; -import java.util.ArrayList; -import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; @@ -15,16 +13,20 @@ public class MockedQuickPulseServlet extends HttpServlet { - private final AtomicBoolean pingReceived = new AtomicBoolean(false); - private final List postBodies = new ArrayList<>(); + private final boolean usingOld3xAgent; + private final AtomicBoolean pingReceived = new AtomicBoolean(); + private final AtomicBoolean postReceived = new AtomicBoolean(); + private volatile LiveMetricsVerifier verifier = new LiveMetricsVerifier(); private final Object lock = new Object(); private static final String BODY = - "{\"ETag\":\"fake::etag\",\"Metrics\":[],\"QuotaInfo\":null,\"DocumentStreams\":[{\"Id\":\"all-types-default\",\"DocumentFilterGroups\":[{\"TelemetryType\":\"Request\",\"Filters\":{\"Filters\":[{\"FieldName\":\"Success\",\"Predicate\":\"Equal\",\"Comparand\":\"false\"}]}},{\"TelemetryType\":\"Dependency\",\"Filters\":{\"Filters\":[{\"FieldName\":\"Success\",\"Predicate\":\"Equal\",\"Comparand\":\"false\"}]}},{\"TelemetryType\":\"Exception\",\"Filters\":{\"Filters\":[]}},{\"TelemetryType\":\"Event\",\"Filters\":{\"Filters\":[]}},{\"TelemetryType\":\"Trace\",\"Filters\":{\"Filters\":[]}}]}]}"; + "{\"ETag\":\"fake::etag\",\"Metrics\":[],\"QuotaInfo\":null,\"DocumentStreams\":[{\"Id\":\"all-types-default\",\"DocumentFilterGroups\":[{\"TelemetryType\":\"Request\",\"Filters\":{\"Filters\":[{\"FieldName\":\"Success\",\"Predicate\":\"Equal\",\"Comparand\":\"true\"}]}},{\"TelemetryType\":\"Dependency\",\"Filters\":{\"Filters\":[{\"FieldName\":\"Success\",\"Predicate\":\"Equal\",\"Comparand\":\"false\"}]}},{\"TelemetryType\":\"Exception\",\"Filters\":{\"Filters\":[]}},{\"TelemetryType\":\"Event\",\"Filters\":{\"Filters\":[]}},{\"TelemetryType\":\"Trace\",\"Filters\":{\"Filters\":[]}}]}]}"; private volatile boolean loggingEnabled; - public MockedQuickPulseServlet() {} + public MockedQuickPulseServlet(boolean usingOld3xAgent) { + this.usingOld3xAgent = usingOld3xAgent; + } @SuppressWarnings("SystemOut") private void logit(String message) { @@ -35,6 +37,10 @@ private void logit(String message) { @Override protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException { + if (usingOld3xAgent) { + // the old 3.x agent doesn't conform to the expectations of this mock server + return; + } Readable reader = req.getReader(); StringWriter sw = new StringWriter(); CharStreams.copy(reader, sw); @@ -52,8 +58,9 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws I resp.getWriter().write(BODY); } else if (path.equals("/post")) { + postReceived.set(true); synchronized (lock) { - postBodies.add(body); + verifier.apply(body); } logit("post body: " + body); // continue to post @@ -65,17 +72,23 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws I } } - public boolean isPingReceived() { - return pingReceived.get(); + public boolean isReceivingLiveMetrics() { + return pingReceived.get() && postReceived.get(); } - public List getPostBodies() { - synchronized (lock) { - return new ArrayList<>(postBodies); - } + public LiveMetricsVerifier getVerifier() { + return verifier; } public void setRequestLoggingEnabled(boolean enabled) { loggingEnabled = enabled; } + + public void resetData() { + synchronized (lock) { + pingReceived.set(false); + postReceived.set(false); + verifier = new LiveMetricsVerifier(); + } + } }