Retry webflux tests 3 times when it's failing (#1712)

* Retry webflux 3 times when it's failing

* print stack trace when the final retry fails

* Reword

Co-authored-by: Trask Stalnaker <[email protected]>

Co-authored-by: Trask Stalnaker <[email protected]>

Author: heyams

test/smoke/framework/testCore/src/main/java/com/microsoft/applicationinsights/smoketest/Retry.java

@@ -0,0 +1,40 @@
+package com.microsoft.applicationinsights.smoketest;
+
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+public class Retry implements TestRule {
+
+    private final int retryCount;
+
+    public Retry(int retryCount) {
+        this.retryCount = retryCount;
+    }
+
+    @Override
+    public Statement apply(Statement base, Description description) {
+        return statement(base, description);
+    }
+
+    private Statement statement(Statement base, Description description) {
+        return new Statement() {
+            @Override
+            public void evaluate() throws Throwable {
+                Throwable caughtThrowable = null;
+                for (int i = 0; i < retryCount; i++) {
+                    try {
+                        base.evaluate();  // this will invoke test method, when the test fails, it will throw an AssertionError.
+                        return;
+                    } catch (Throwable t) {
+                        caughtThrowable = t;
+                        System.err.println(description.getDisplayName() + ": run " + (i+1) + " failed");
+                    }
+                }
+                System.err.println(description.getDisplayName() + ": giving up after " + retryCount + " failures");
+                caughtThrowable.printStackTrace();
+                throw caughtThrowable;
+            }
+        };
+    }
+}

test/smoke/testApps/WebFlux/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/WebFluxTest.java

@@ -3,6 +3,7 @@
 import com.microsoft.applicationinsights.internal.schemav2.Data;
 import com.microsoft.applicationinsights.internal.schemav2.Envelope;
 import com.microsoft.applicationinsights.internal.schemav2.RequestData;
+import org.junit.Rule;
 import org.junit.Test;
 
 import java.util.List;
@@ -12,6 +13,10 @@
 @UseAgent
 public class WebFluxTest extends AiSmokeTest {
 
+    // TODO revisit and try to get to the bottom of this sporadic failure
+    @Rule
+    public Retry retry = new Retry(5);
+
     @Test
     @TargetUri("/test")
     public void doMostBasicTest() throws Exception {