use rerunfailures

Author: nammn

docker/mongodb-kubernetes-tests/tests/multicluster/multi_cluster_backup_restore.py

@@ -25,6 +25,7 @@
 from kubetester.operator import Operator
 from kubetester.opsmanager import MongoDBOpsManager
 from kubetester.phase import Phase
+import pytest
 from pytest import fixture, mark
 from tests.conftest import (
     assert_data_got_restored,
@@ -474,17 +475,9 @@ def test_mongodb_multi_one_running_state(self, mongodb_multi_one: MongoDBMulti):
 
     @skip_if_local
     @mark.e2e_multi_cluster_backup_restore
+    @pytest.mark.flaky(reruns=16, reruns_delay=6)
     def test_add_test_data(self, mongodb_multi_one_collection):
-        max_attempts = 100
-        while max_attempts > 0:
-            try:
-                mongodb_multi_one_collection.insert_one(TEST_DATA)
-                return
-            except Exception as e:
-                print(e)
-                max_attempts -= 1
-                time.sleep(6)
-        raise Exception("❌ Failed to insert test data after multiple attempts")
+        mongodb_multi_one_collection.insert_one(TEST_DATA)
 
     @mark.e2e_multi_cluster_backup_restore
     def test_mdb_backed_up(self, project_one: OMTester):

docker/mongodb-kubernetes-tests/tests/multicluster/multi_cluster_dr_connect.py

@@ -54,18 +54,9 @@ def test_replica_set_is_reachable(mongodb_multi: MongoDBMulti):
 
 
 @pytest.mark.e2e_multi_cluster_dr
+@pytest.mark.flaky(reruns=16, reruns_delay=6)
 def test_add_test_data(mongodb_multi_collection):
-    # TODO: remove this retry mechanism, for some reason the resource exits the running state and then
-    # enters it later. The subsequent test fails because the resource is not actually
-    max_attempts = 100
-    while max_attempts > 0:
-        try:
-            mongodb_multi_collection.insert_one(TEST_DATA)
-            return
-        except Exception as e:
-            print(e)
-            max_attempts -= 1
-            time.sleep(6)
+    mongodb_multi_collection.insert_one(TEST_DATA)
 
 
 @pytest.mark.e2e_multi_cluster_dr

docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_s3_based_backup_restore.py

@@ -3,6 +3,7 @@
 
 import kubernetes.client
 import pymongo
+import pytest
 from kubetester import create_or_update_configmap, try_load
 from kubetester.kubetester import ensure_ent_version
 from kubetester.kubetester import fixture as yaml_fixture
@@ -203,16 +204,9 @@ def test_mongodb_multi_one_running_state(self, mongodb_multi_one: MongoDBMulti):
         # we might fail connection in the beginning since we set a custom dns in coredns
         mongodb_multi_one.assert_reaches_phase(Phase.Running, ignore_errors=True, timeout=600)
 
+    @pytest.mark.flaky(reruns=16, reruns_delay=6)
     def test_add_test_data(self, mongodb_multi_one_collection):
-        max_attempts = 100
-        while max_attempts > 0:
-            try:
-                mongodb_multi_one_collection.insert_one(TEST_DATA)
-                return
-            except Exception as e:
-                print(e)
-                max_attempts -= 1
-                time.sleep(6)
+        mongodb_multi_one_collection.insert_one(TEST_DATA)
 
     def test_mdb_backed_up(self, project_one: OMTester):
         project_one.wait_until_backup_snapshots_are_ready(expected_count=1)

docker/mongodb-kubernetes-tests/tests/multicluster_om/multicluster_om_appdb_no_mesh.py

@@ -557,16 +557,9 @@ def test_create_mongodb_multi(server_certs: str, mongodb_multi: MongoDBMulti):
 
 @skip_if_local
 @mark.e2e_multi_cluster_om_appdb_no_mesh
+@pytest.mark.flaky(reruns=16, reruns_delay=6)
 def test_add_test_data(mongodb_multi_collection):
-    max_attempts = 100
-    while max_attempts > 0:
-        try:
-            mongodb_multi_collection.insert_one(TEST_DATA)
-            return
-        except Exception as e:
-            print(e)
-            max_attempts -= 1
-            time.sleep(6)
+    mongodb_multi_collection.insert_one(TEST_DATA)
 
 
 @mark.e2e_multi_cluster_om_appdb_no_mesh

docker/mongodb-kubernetes-tests/tests/replicaset/replica_set_liveness_probe.py

@@ -25,26 +25,17 @@ def _get_pods(podname_template: str, qty: int = 3):
 
 @skip_if_static_containers
 @pytest.mark.e2e_replica_set_liveness_probe
+@pytest.mark.flaky(reruns=10, reruns_delay=30)
 def test_pods_are_running(replica_set: MongoDB, namespace: str):
     corev1_client = client.CoreV1Api()
     running_pods: Set[str] = set()
-    tries = 10
     # Wait for all the pods to be running
     # We can't wait for the replica set to be running
     # as it will never get to it (mongod is not starting)
-    while tries:
-        if len(running_pods) == 3:
-            break
-        for podname in _get_pods("my-replica-set-{}", 3):
-            try:
-                pod = corev1_client.read_namespaced_pod(podname, namespace)
-                if pod.status.phase == "Running":
-                    running_pods.add(podname)
-            except:
-                # Pod not found, will retry
-                pass
-        tries -= 1
-        time.sleep(30)
+    for podname in _get_pods("my-replica-set-{}", 3):
+        pod = corev1_client.read_namespaced_pod(podname, namespace)
+        if pod.status.phase == "Running":
+            running_pods.add(podname)
     assert len(running_pods) == 3
 
 

docker/mongodb-kubernetes-tests/tests/replicaset/replica_set_pv.py

@@ -22,7 +22,7 @@ def test_replica_set_sts_exists(self):
         sts = self.appsv1.read_namespaced_stateful_set("rs001-pv", self.namespace)
         assert sts
 
-    @pytest.mark.flaky(reruns=3, reruns_delay=2)
+    @pytest.mark.flaky(reruns=15, reruns_delay=5)
     def test_sts_creation(self):
         sts = self.appsv1.read_namespaced_stateful_set("rs001-pv", self.namespace)