[Internal] Refactoring live-tests to reduce flakiness (Azure#45216)

* Refactoring FaultInjection tests

* Update IncrementalChangeFeedProcessorTest.java

* Update IncrementalChangeFeedProcessorTest.java

* Update IncrementalChangeFeedProcessorTest.java

* Fixing flaky test

* Fixing flaky tests

* Update ClientMetricsTest.java

* Refactoring CF split tests

* Fixing split awaiting

* Update ClientMetricsTest.java

* Fixing test flakiness

* Update pom.xml

* Update ClientMetricsTest.java

* Update ClientMetricsTest.java

* Reducing log noise level for metrics

* Refactoring ClientMetricsTest to have more robust state management when a test times out

* Adding logs to flaky test ClientMetricsTest.readAllItemsWithDetailMetricsWithExplicitPageSize

* Test robustness improvements

* Marking more tests as flaky

* Changing test config

* Test fixes

* Fixing occassional hangs in CosmosPagedIterable when page size is specified

* Test fixes

* Update FaultInjectionWithAvailabilityStrategyTestsBase.java

* Fixing flaky tests

Author: FabianMeiswinkel

sdk/cosmos/azure-cosmos-benchmark/pom.xml

@@ -473,6 +473,27 @@ Licensed under the MIT License.
         </plugins>
       </build>
     </profile>
+    <profile>
+      <!-- integration tests, requires Cosmos DB endpoint with multi master support -->
+      <id>fi-multi-master</id>
+      <properties>
+        <test.groups>fi-multi-master</test.groups>
+      </properties>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-failsafe-plugin</artifactId>
+            <version>3.5.2</version> <!-- {x-version-update;org.apache.maven.plugins:maven-failsafe-plugin;external_dependency} -->
+            <configuration>
+              <suiteXmlFiles>
+                <suiteXmlFile>src/test/resources/fi-multi-master-testng.xml</suiteXmlFile>
+              </suiteXmlFiles>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
     <profile>
       <!-- integration tests, requires Cosmos DB endpoint -->
       <id>examples</id>

sdk/cosmos/azure-cosmos-benchmark/src/test/resources/fi-multi-master-testng.xml

@@ -0,0 +1,35 @@
+<!--
+  ~ The MIT License (MIT)
+  ~ Copyright (c) 2018 Microsoft Corporation
+  ~
+  ~ Permission is hereby granted, free of charge, to any person obtaining a copy
+  ~ of this software and associated documentation files (the "Software"), to deal
+  ~ in the Software without restriction, including without limitation the rights
+  ~ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  ~ copies of the Software, and to permit persons to whom the Software is
+  ~ furnished to do so, subject to the following conditions:
+  ~
+  ~ The above copyright notice and this permission notice shall be included in all
+  ~ copies or substantial portions of the Software.
+  ~
+  ~ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  ~ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  ~ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  ~ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  ~ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  ~ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  ~ SOFTWARE.
+  -->
+<!DOCTYPE suite SYSTEM "https://testng.org/testng-1.0.dtd">
+<suite name="fi-multi-master">
+  <test name="fi-multi-master" group-by-instances="true">
+    <groups>
+      <run>
+        <include name="fi-multi-master"/>
+      </run>
+    </groups>
+    <packages>
+      <package name="com.azure.cosmos.*"/>
+    </packages>
+  </test>
+</suite>

sdk/cosmos/azure-cosmos-spark_3_2-12/docs/AAD-Auth.md

@@ -173,7 +173,7 @@ from cryptography.hazmat.primitives import serialization
 secret64 = mssparkutils.credentials.getSecret("xxx", "yyy", "zzz")
 
 # Decode to bytes.
-secret64Bytes = bytes(secret, "utf-8")
+secret64Bytes = bytes(secret64, "utf-8")
 secretBytes = base64.b64decode(secret64)
 
 # Load as a certificate: https://cryptography.io/en/latest/hazmat/primitives/asymmetric/serialization/#pkcs12.

sdk/cosmos/azure-cosmos-tests/pom.xml

@@ -549,6 +549,27 @@ Licensed under the MIT License.
         </plugins>
       </build>
     </profile>
+    <profile>
+      <!-- integration tests, requires Cosmos DB endpoint with multi master support -->
+      <id>fi-multi-master</id>
+      <properties>
+        <test.groups>fi-multi-master</test.groups>
+      </properties>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-failsafe-plugin</artifactId>
+            <version>3.5.2</version> <!-- {x-version-update;org.apache.maven.plugins:maven-failsafe-plugin;external_dependency} -->
+            <configuration>
+              <suiteXmlFiles>
+                <suiteXmlFile>src/test/resources/fi-multi-master-testng.xml</suiteXmlFile>
+              </suiteXmlFiles>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
     <profile>
       <!-- integration tests, requires Cosmos DB endpoint with multi region support -->
       <id>multi-region</id>

sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/AsyncCacheNonBlockingIntegrationTest.java

@@ -63,7 +63,7 @@ public void afterClass() {
         safeClose(this.bulkClient);
     }
 
-    @Test(groups = {"split"}, timeOut = TIMEOUT * 20)
+    @Test(groups = {"split"}, timeOut = TIMEOUT * 20, retryAnalyzer = SplitTestsRetryAnalyzer.class)
     public void createItem_withCacheRefresh() throws InterruptedException {
         String containerId = "bulksplittestcontainer_" + UUID.randomUUID();
         int totalRequest = getTotalRequest();
@@ -126,14 +126,26 @@ public void createItem_withCacheRefresh() throws InterruptedException {
 
 
         // Wait for the throughput update to complete so that we get the partition split
-        while (true) {
-            assert throughputResponse != null;
-            if (!throughputResponse.isReplacePending()) {
-                break;
-            }
+        int i = 0;
+        // Only wait for 10 minutes for the split to complete
+        // If backend does not finish split within 10 minutes
+        // something is off in the backend
+        // it could be due to limits on how many splits can be executed concurrently etc.
+        // nothing that can really be done in the SDK
+        while (i < 60 && throughputResponse.isReplacePending()) {
             logger.info("Waiting for split to complete");
             Thread.sleep(10 * 1000);
             throughputResponse = container.readThroughput().block();
+            i++;
+        }
+
+        if (throughputResponse.isReplacePending()) {
+            throw new SplitTimeoutException(
+                "Backend did not finish split for container '"
+                    + getEndpoint() + "/"
+                    + container.getDatabase().getId() + "/"
+                    + container.getId()
+                    + "' - skipping this test case");
         }
 
         // Read number of partitions. Should be greater than one

sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/BatchTestBase.java

@@ -10,6 +10,7 @@
 import com.azure.cosmos.implementation.apachecommons.lang.StringUtils;
 import com.azure.cosmos.models.CosmosBatchOperationResult;
 import com.azure.cosmos.models.CosmosBatchResponse;
+import com.azure.cosmos.models.CosmosItemRequestOptions;
 import com.azure.cosmos.models.CosmosItemResponse;
 import com.azure.cosmos.models.PartitionKey;
 import com.azure.cosmos.rx.TestSuiteBase;
@@ -99,7 +100,7 @@ void verifyByRead(CosmosContainer container, TestDoc doc) {
     void verifyByRead(CosmosContainer container, TestDoc doc, String eTag) {
         PartitionKey partitionKey = this.getPartitionKey(doc.getStatus());
 
-        CosmosItemResponse<TestDoc> response = container.readItem(doc.getId(), partitionKey, TestDoc.class);
+        CosmosItemResponse<TestDoc> response = verifyExists(container, doc.getId(), partitionKey, TestDoc.class);
 
         assertThat(response.getStatusCode()).isEqualTo(HttpResponseStatus.OK.code());
         assertThat(response.getItem()).isEqualTo(doc);