Merge pull request #1056 from microsoft/trask/more-test-improvements

trask · web-flow · commit 0fd120a8bf24 · 2019-09-05T13:51:24.000-07:00
More test improvements
diff --git a/test/smoke/framework/testCore/src/main/java/com/microsoft/applicationinsights/smoketest/AiSmokeTest.java b/test/smoke/framework/testCore/src/main/java/com/microsoft/applicationinsights/smoketest/AiSmokeTest.java
@@ -155,6 +155,7 @@ protected static void stopContainer(ContainerInfo info) throws Exception {
     protected String targetUri;
     protected String httpMethod;
     protected long targetUriDelayMs;
+    protected long targetUriCallCount;
     protected long targetUriTimeoutMs;
     //endregion
 
@@ -187,6 +188,7 @@ protected void starting(Description description) {
                 thiz.targetUri = null;
                 thiz.httpMethod = null;
                 thiz.targetUriDelayMs = 0L;
+                thiz.targetUriCallCount = 1;
                 thiz.targetUriTimeoutMs = TELEMETRY_RECEIVE_TIMEOUT_SECONDS * 1000;
             } else {
                 thiz.targetUri = targetUri.value();
@@ -195,6 +197,7 @@ protected void starting(Description description) {
                 }
                 thiz.httpMethod = targetUri.method().toUpperCase();
                 thiz.targetUriDelayMs = targetUri.delay();
+                thiz.targetUriCallCount = targetUri.callCount();
                 thiz.targetUriTimeoutMs = targetUri.timeout() > 0 ? targetUri.timeout() : TELEMETRY_RECEIVE_TIMEOUT_SECONDS * 1000;
             }
 
@@ -395,20 +398,26 @@ protected void callTargetUriAndWaitForTelemetry() throws Exception {
         }
         System.out.println("Calling "+targetUri+" ...");
         String url = getBaseUrl()+targetUri;
-        System.out.println("calling " + url);
-        final String content;
-        switch(httpMethod) {
-            case "GET":
-                content = HttpHelper.get(url);
-                break;
-            default:
-                throw new NotSupportedException(String.format("http method '%s' is not currently supported", httpMethod));
+        if (targetUriCallCount == 1) {
+            System.out.println("calling " + url);
+        } else {
+            System.out.println("calling " + url + " " + targetUriCallCount + " times");
+        }
+        for (int i = 0; i < targetUriCallCount; i++) {
+            final String content;
+            switch (httpMethod) {
+                case "GET":
+                    content = HttpHelper.get(url);
+                    break;
+                default:
+                    throw new NotSupportedException(
+                            String.format("http method '%s' is not currently supported", httpMethod));
+            }
+            String expectationMessage = "The base context in testApps should return a nonempty response.";
+            assertNotNull(String.format("Null response from targetUri: '%s'. %s", targetUri, expectationMessage),                    content);
+            assertTrue(String.format("Empty response from targetUri: '%s'. %s", targetUri, expectationMessage),
+                    content.length() > 0);
         }
-
-        String expectationMessage = "The base context in testApps should return a nonempty response.";
-        assertNotNull(String.format("Null response from targetUri: '%s'. %s", targetUri, expectationMessage), content);
-        assertTrue(String.format("Empty response from targetUri: '%s'. %s", targetUri, expectationMessage), content.length() > 0);
-
         if (this.targetUriTimeoutMs > 0) {
             Stopwatch sw = Stopwatch.createStarted();
             mockedIngestion.awaitAnyItems(this.targetUriTimeoutMs, TimeUnit.MILLISECONDS);
diff --git a/test/smoke/framework/testCore/src/main/java/com/microsoft/applicationinsights/smoketest/TargetUri.java b/test/smoke/framework/testCore/src/main/java/com/microsoft/applicationinsights/smoketest/TargetUri.java
@@ -16,6 +16,11 @@
      */
     long delay() default 0L;
 
+    /**
+     * The number of times to call the target uri.
+     */
+    int callCount() default 1;
+
     /**
      * The number of milliseconds to wait for a response.
      */
diff --git a/test/smoke/testApps/AutoPerfCounters/src/main/resources/ApplicationInsights.xml b/test/smoke/testApps/AutoPerfCounters/src/main/resources/ApplicationInsights.xml
@@ -32,7 +32,6 @@
     </TelemetryInitializers>
 
     <Channel>
-        <!-- <EndpointAddress>http://localhost:60606/v2/track</EndpointAddress> -->
         <EndpointAddress>http://fakeingestion:60606/v2/track</EndpointAddress>
         <DeveloperMode>true</DeveloperMode>
         <FlushIntervalInSeconds>1</FlushIntervalInSeconds>
diff --git a/test/smoke/testApps/CachingCalculator/src/main/resources/ApplicationInsights.xml b/test/smoke/testApps/CachingCalculator/src/main/resources/ApplicationInsights.xml
@@ -33,7 +33,6 @@
     </TelemetryInitializers>
 
     <Channel>
-        <!-- <EndpointAddress>http://localhost:60606/v2/track</EndpointAddress> -->
         <EndpointAddress>http://fakeingestion:60606/v2/track</EndpointAddress>
         <DeveloperMode>true</DeveloperMode>
         <FlushIntervalInSeconds>1</FlushIntervalInSeconds>
diff --git a/test/smoke/testApps/CachingCalculator/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SampleTestWithDependencyContainer.java b/test/smoke/testApps/CachingCalculator/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SampleTestWithDependencyContainer.java
@@ -5,7 +5,6 @@
 import com.microsoft.applicationinsights.internal.schemav2.Data;
 import com.microsoft.applicationinsights.internal.schemav2.Envelope;
 import com.microsoft.applicationinsights.internal.schemav2.RemoteDependencyData;
-import com.microsoft.applicationinsights.internal.schemav2.RequestData;
 import org.junit.*;
 
 import static org.hamcrest.Matchers.hasSize;
diff --git a/test/smoke/testApps/CoreAndFilter/src/main/java/com/microsoft/ajl/simplecalc/SimpleTrackPageViewServlet.java b/test/smoke/testApps/CoreAndFilter/src/main/java/com/microsoft/ajl/simplecalc/SimpleTrackPageViewServlet.java
@@ -1,8 +1,6 @@
 package com.microsoft.ajl.simplecalc;
 
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
 
 import javax.servlet.ServletException;
 import javax.servlet.annotation.WebServlet;
diff --git a/test/smoke/testApps/CoreAndFilter/src/main/java/com/microsoft/ajl/simplecalc/SlowRequestCpuBoundServlet.java b/test/smoke/testApps/CoreAndFilter/src/main/java/com/microsoft/ajl/simplecalc/SlowRequestCpuBoundServlet.java
@@ -1,9 +1,6 @@
 package com.microsoft.ajl.simplecalc;
 
-import com.google.common.base.Predicate;
-import com.google.common.base.Predicates;
 import org.apache.commons.lang3.StringUtils;
-import org.checkerframework.checker.nullness.compatqual.NullableDecl;
 
 import javax.servlet.ServletException;
 import javax.servlet.annotation.WebServlet;
diff --git a/test/smoke/testApps/CoreAndFilter/src/main/resources/ApplicationInsights.xml b/test/smoke/testApps/CoreAndFilter/src/main/resources/ApplicationInsights.xml
@@ -33,7 +33,6 @@
     </TelemetryInitializers>
 
     <Channel>
-        <!-- <EndpointAddress>http://localhost:60606/v2/track</EndpointAddress> -->
         <EndpointAddress>http://fakeingestion:60606/v2/track</EndpointAddress>
         <DeveloperMode>true</DeveloperMode>
         <FlushIntervalInSeconds>1</FlushIntervalInSeconds>
diff --git a/test/smoke/testApps/CoreAndFilter/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/CoreAndFilterTests.java b/test/smoke/testApps/CoreAndFilter/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/CoreAndFilterTests.java
@@ -21,7 +21,6 @@
 import com.microsoft.applicationinsights.smoketest.matchers.PageViewDataMatchers;
 import com.microsoft.applicationinsights.smoketest.matchers.TraceDataMatchers;
 import com.microsoft.applicationinsights.telemetry.Duration;
-import org.junit.Ignore;
 import org.junit.Test;
 
 import static com.microsoft.applicationinsights.smoketest.matchers.ExceptionDataMatchers.ExceptionDetailsMatchers.withMessage;
diff --git a/test/smoke/testApps/CustomInstrumentation/src/smokeTest/java/com/microsoft/applicationinsights/smoketestapp/CustomInstrumentationTest.java b/test/smoke/testApps/CustomInstrumentation/src/smokeTest/java/com/microsoft/applicationinsights/smoketestapp/CustomInstrumentationTest.java
@@ -13,10 +13,8 @@
 import com.microsoft.applicationinsights.smoketest.UseAgent;
 import org.junit.Test;
 
-import static org.hamcrest.Matchers.hasSize;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 
 @UseAgent("CustomInstrumentation")
diff --git a/test/smoke/testApps/FixedRateSampling/src/main/java/com/microsoft/ajl/simplecalc/SimpleFixedRateSamplingServlet.java b/test/smoke/testApps/FixedRateSampling/src/main/java/com/microsoft/ajl/simplecalc/SimpleFixedRateSamplingServlet.java
@@ -1,6 +1,7 @@
 package com.microsoft.ajl.simplecalc;
 
 import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import javax.servlet.ServletException;
 import javax.servlet.annotation.WebServlet;
@@ -19,6 +20,8 @@ public class SimpleFixedRateSamplingServlet extends HttpServlet {
     private static final long serialVersionUID = -5889330779672565409L;
     private TelemetryClient client = new TelemetryClient();
 
+    private final AtomicInteger count = new AtomicInteger();
+
     /**
      * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse
      *      response)
@@ -27,11 +30,6 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response)
             throws ServletException, IOException {
         ServletFuncs.getRenderedHtml(request, response);
         client.trackTrace("Trace Test.");
-        for (int i = 0; i < 100; i++) {
-            String str = String.format("Event Test %s", i);
-            EventTelemetry et = new EventTelemetry(str);
-            et.getContext().getOperation().setId(String.valueOf(i));
-            client.trackEvent(et);
-        }
+        client.trackEvent(new EventTelemetry("Event Test " + count.getAndIncrement()));
     }
 }
diff --git a/test/smoke/testApps/FixedRateSampling/src/main/resources/ApplicationInsights.xml b/test/smoke/testApps/FixedRateSampling/src/main/resources/ApplicationInsights.xml
@@ -32,7 +32,6 @@
     </TelemetryInitializers>
 
     <Channel>
-        <!-- <EndpointAddress>http://localhost:60606/v2/track</EndpointAddress> -->
         <EndpointAddress>http://fakeingestion:60606/v2/track</EndpointAddress>
         <DeveloperMode>true</DeveloperMode>
         <FlushIntervalInSeconds>1</FlushIntervalInSeconds>
diff --git a/test/smoke/testApps/FixedRateSampling/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/FixedRateSamplingTest.java b/test/smoke/testApps/FixedRateSampling/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/FixedRateSamplingTest.java
@@ -1,5 +1,7 @@
 package com.microsoft.applicationinsights.smoketest;
 
+import java.util.List;
+
 import com.microsoft.applicationinsights.internal.schemav2.Envelope;
 import org.junit.*;
 
@@ -9,29 +11,29 @@
 public class FixedRateSamplingTest extends AiSmokeTest {
     @Test
     @TargetUri("/fixedRateSampling")
-    public void testFixedRateSamplingInExcludedTypes() {
-        assertEquals(1, mockedIngestion.getCountForType("RequestData"));
-        assertEquals(100.0, getSampleRate("RequestData", 0), Math.ulp(50.0));
+    public void testFixedRateSamplingInExcludedTypes() throws Exception {
+        List<Envelope> rdList = mockedIngestion.waitForItems("RequestData", 1);
+        Envelope rd = rdList.get(0);
+        assertEquals(100.0, rd.getSampleRate(), Math.ulp(50.0));
     }
 
     @Test
-    @TargetUri(value = "/fixedRateSampling", delay = 10000)
-    public void testFixedRateSamplingInIncludedTypes() {
-        int count = mockedIngestion.getCountForType("EventData");
-        assertThat(count, both(greaterThanOrEqualTo(40)).and(lessThanOrEqualTo(60)));
-        assertEquals(50.0, getSampleRate("EventData", 0), Math.ulp(50.0));
+    @TargetUri(value = "/fixedRateSampling", callCount = 100)
+    public void testFixedRateSamplingInIncludedTypes() throws Exception {
+        mockedIngestion.waitForItems("RequestData", 100);
+        List<Envelope> edList = mockedIngestion.getItemsEnvelopeDataType("EventData");
+        // super super low chance that number of events sampled is less than 10 or greater than 90
+        assertThat(edList.size(), both(greaterThanOrEqualTo(10)).and(lessThanOrEqualTo(90)));
+        Envelope ed = edList.get(0);
+        assertEquals(50.0, ed.getSampleRate(), Math.ulp(50.0));
     }
 
     @Test
     @TargetUri("/fixedRateSampling")
-    public void testFixedRateSamplingNotInExcludedTypes() {
-        assertEquals(1, mockedIngestion.getCountForType("MessageData"));
-        assertEquals(100.0, getSampleRate("MessageData", 0), Math.ulp(50.0));
-    }
-
-    protected double getSampleRate(String type, int index) {
-        Envelope envelope = mockedIngestion.getItemsEnvelopeDataType(type).get(index);
-        return envelope.getSampleRate();
+    public void testFixedRateSamplingNotInExcludedTypes() throws Exception {
+        mockedIngestion.waitForItems("RequestData", 1);
+        List<Envelope> mdList = mockedIngestion.waitForItems("MessageData", 1);
+        Envelope md = mdList.get(0);
+        assertEquals(100.0, md.getSampleRate(), Math.ulp(50.0));
     }
-
 }
diff --git a/test/smoke/testApps/HeartBeat/src/main/resources/ApplicationInsights.xml b/test/smoke/testApps/HeartBeat/src/main/resources/ApplicationInsights.xml
@@ -31,7 +31,6 @@
     </TelemetryInitializers>
 
     <Channel>
-        <!-- <EndpointAddress>http://localhost:60606/v2/track</EndpointAddress> -->
         <EndpointAddress>http://fakeingestion:60606/v2/track</EndpointAddress>
         <DeveloperMode>true</DeveloperMode>
         <FlushIntervalInSeconds>1</FlushIntervalInSeconds>
diff --git a/test/smoke/testApps/TraceLog4j1_2/src/main/resources/ApplicationInsights.xml b/test/smoke/testApps/TraceLog4j1_2/src/main/resources/ApplicationInsights.xml
@@ -28,7 +28,6 @@
     </TelemetryInitializers>
 
     <Channel>
-        <!-- <EndpointAddress>http://localhost:60606/v2/track</EndpointAddress> -->
         <EndpointAddress>http://fakeingestion:60606/v2/track</EndpointAddress>
         <DeveloperMode>true</DeveloperMode>
         <FlushIntervalInSeconds>1</FlushIntervalInSeconds>
diff --git a/test/smoke/testApps/TraceLog4j1_2UsingAgent/src/main/resources/ApplicationInsights.xml b/test/smoke/testApps/TraceLog4j1_2UsingAgent/src/main/resources/ApplicationInsights.xml
@@ -28,7 +28,6 @@
     </TelemetryInitializers>
 
     <Channel>
-        <!-- <EndpointAddress>http://localhost:60606/v2/track</EndpointAddress> -->
         <EndpointAddress>http://fakeingestion:60606/v2/track</EndpointAddress>
     </Channel>
     <PerformanceCounters>
diff --git a/test/smoke/testApps/TraceLog4j1_2UsingAgent/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/TraceLog4j1_2Test.java b/test/smoke/testApps/TraceLog4j1_2UsingAgent/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/TraceLog4j1_2Test.java
@@ -1,9 +1,13 @@
 package com.microsoft.applicationinsights.smoketest;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 
+import java.util.Comparator;
 import java.util.List;
 
+import com.microsoft.applicationinsights.internal.schemav2.Data;
+import com.microsoft.applicationinsights.internal.schemav2.Envelope;
 import com.microsoft.applicationinsights.internal.schemav2.ExceptionData;
 import com.microsoft.applicationinsights.internal.schemav2.ExceptionDetails;
 import com.microsoft.applicationinsights.internal.schemav2.MessageData;
@@ -15,43 +19,76 @@ public class TraceLog4j1_2Test extends AiSmokeTest {
 
     @Test
     @TargetUri("/traceLog4j1_2")
-    public void testTraceLog4j1_2() {
+    public void testTraceLog4j1_2() throws Exception {
+        List<Envelope> rdList = mockedIngestion.waitForItems("RequestData", 1);
+        List<Envelope> mdList = mockedIngestion.waitForItems("MessageData", 3);
 
-        assertEquals(3, mockedIngestion.getCountForType("MessageData"));
+        Envelope rdEnvelope = rdList.get(0);
+        Envelope mdEnvelope1 = mdList.get(0);
+        Envelope mdEnvelope2 = mdList.get(1);
+        Envelope mdEnvelope3 = mdList.get(2);
+
+        List<MessageData> logs = mockedIngestion.getTelemetryDataByType("MessageData");
+        logs.sort(new Comparator<MessageData>() {
+            @Override
+            public int compare(MessageData o1, MessageData o2) {
+                return o1.getSeverityLevel().compareTo(o2.getSeverityLevel());
+            }
+        });
+
+        MessageData md1 = logs.get(0);
+        MessageData md2 = logs.get(1);
+        MessageData md3 = logs.get(2);
 
-        MessageData md1 = getTelemetryDataForType(0, "MessageData");
         assertEquals("This is log4j1.2 warn.", md1.getMessage());
         assertEquals(SeverityLevel.Warning, md1.getSeverityLevel());
         assertEquals("Logger", md1.getProperties().get("SourceType"));
         assertEquals("WARN", md1.getProperties().get("LoggingLevel"));
+        assertSameOperationId(mdEnvelope1, rdEnvelope);
 
-
-        MessageData md2 = getTelemetryDataForType(1, "MessageData");
         assertEquals("This is log4j1.2 error.", md2.getMessage());
         assertEquals(SeverityLevel.Error, md2.getSeverityLevel());
         assertEquals("Logger", md2.getProperties().get("SourceType"));
         assertEquals("ERROR", md2.getProperties().get("LoggingLevel"));
+        assertSameOperationId(mdEnvelope2, rdEnvelope);
 
-        MessageData md3 = getTelemetryDataForType(2, "MessageData");
         assertEquals("This is log4j1.2 fatal.", md3.getMessage());
         assertEquals(SeverityLevel.Critical, md3.getSeverityLevel());
         assertEquals("Logger", md3.getProperties().get("SourceType"));
         assertEquals("FATAL", md3.getProperties().get("LoggingLevel"));
+        assertSameOperationId(mdEnvelope3, rdEnvelope);
     }
 
     @Test
     @TargetUri("traceLog4j1_2WithException")
-    public void testTraceLog4j1_2WithExeption() {
-        assertEquals(1, mockedIngestion.getCountForType("ExceptionData"));
+    public void testTraceLog4j1_2WithExeption() throws Exception {
+        List<Envelope> rdList = mockedIngestion.waitForItems("RequestData", 1);
+        List<Envelope> edList = mockedIngestion.waitForItems("ExceptionData", 1);
+
+        Envelope rdEnvelope = rdList.get(0);
+        Envelope edEnvelope = edList.get(0);
+
+        ExceptionData ed = (ExceptionData) ((Data) edEnvelope.getData()).getBaseData();
 
-        ExceptionData ed1 = getTelemetryDataForType(0, "ExceptionData");
-        List<ExceptionDetails> details = ed1.getExceptions();
+        List<ExceptionDetails> details = ed.getExceptions();
         ExceptionDetails ex = details.get(0);
 
         assertEquals("Fake Exception", ex.getMessage());
-        assertEquals(SeverityLevel.Error, ed1.getSeverityLevel());
-        assertEquals("This is an exception!", ed1.getProperties().get("Logger Message"));
-        assertEquals("Logger", ed1.getProperties().get("SourceType"));
-        assertEquals("ERROR", ed1.getProperties().get("LoggingLevel"));
+        assertEquals(SeverityLevel.Error, ed.getSeverityLevel());
+        assertEquals("This is an exception!", ed.getProperties().get("Logger Message"));
+        assertEquals("Logger", ed.getProperties().get("SourceType"));
+        assertEquals("ERROR", ed.getProperties().get("LoggingLevel"));
+        assertSameOperationId(edEnvelope, rdEnvelope);
+    }
+
+    private static void assertSameOperationId(Envelope rdEnvelope, Envelope rddEnvelope) {
+        String operationId = rdEnvelope.getTags().get("ai.operation.id");
+        String operationParentId = rdEnvelope.getTags().get("ai.operation.parentId");
+
+        assertNotNull(operationId);
+        assertNotNull(operationParentId);
+
+        assertEquals(operationId, rddEnvelope.getTags().get("ai.operation.id"));
+        assertEquals(operationParentId, rddEnvelope.getTags().get("ai.operation.parentId"));
     }
 }
diff --git a/test/smoke/testApps/TraceLog4j2UsingAgent/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/TraceLog4j2Test.java b/test/smoke/testApps/TraceLog4j2UsingAgent/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/TraceLog4j2Test.java
diff --git a/test/smoke/testApps/TraceLogBack/src/main/resources/ApplicationInsights.xml b/test/smoke/testApps/TraceLogBack/src/main/resources/ApplicationInsights.xml
diff --git a/test/smoke/testApps/TraceLogBack/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/TraceLogBackTest.java b/test/smoke/testApps/TraceLogBack/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/TraceLogBackTest.java
diff --git a/test/smoke/testApps/TraceLogBackUsingAgent/src/main/resources/ApplicationInsights.xml b/test/smoke/testApps/TraceLogBackUsingAgent/src/main/resources/ApplicationInsights.xml
diff --git a/test/smoke/testApps/TraceLogBackUsingAgent/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/TraceLogBackTest.java b/test/smoke/testApps/TraceLogBackUsingAgent/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/TraceLogBackTest.java
