From 52746f85274855c2efa56e9f46e43b1c64be3bc1 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Mon, 1 Sep 2025 10:55:43 +0200 Subject: [PATCH 1/5] use rerunfailures --- .../mongodb-kubernetes-tests/tests/replicaset/replica_set_pv.py | 1 + requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/docker/mongodb-kubernetes-tests/tests/replicaset/replica_set_pv.py b/docker/mongodb-kubernetes-tests/tests/replicaset/replica_set_pv.py index cb4a48d10..bd81699f0 100644 --- a/docker/mongodb-kubernetes-tests/tests/replicaset/replica_set_pv.py +++ b/docker/mongodb-kubernetes-tests/tests/replicaset/replica_set_pv.py @@ -22,6 +22,7 @@ def test_replica_set_sts_exists(self): sts = self.appsv1.read_namespaced_stateful_set("rs001-pv", self.namespace) assert sts + @pytest.mark.flaky(reruns=3, reruns_delay=2) def test_sts_creation(self): sts = self.appsv1.read_namespaced_stateful_set("rs001-pv", self.namespace) diff --git a/requirements.txt b/requirements.txt index 89b2868d2..d660d860a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -49,3 +49,4 @@ types-python-dateutil==2.9.0.20250809 pipupgrade==1.12.0 pytest-cov==6.2.1 pytest-socket==0.7.0 +pytest-rerunfailures==16.0 From e76fdb62e96c40f5addc5885e1ab0acc560f7972 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Mon, 1 Sep 2025 11:01:37 +0200 Subject: [PATCH 2/5] use rerunfailures --- .../multi_cluster_backup_restore.py | 13 +++---------- .../multicluster/multi_cluster_dr_connect.py | 13 ++----------- ...ticluster_appdb_s3_based_backup_restore.py | 12 +++--------- .../multicluster_om_appdb_no_mesh.py | 11 ++--------- .../replicaset/replica_set_liveness_probe.py | 19 +++++-------------- .../tests/replicaset/replica_set_pv.py | 2 +- 6 files changed, 16 insertions(+), 54 deletions(-) diff --git a/docker/mongodb-kubernetes-tests/tests/multicluster/multi_cluster_backup_restore.py b/docker/mongodb-kubernetes-tests/tests/multicluster/multi_cluster_backup_restore.py index eea9bb3dc..57282fd4b 100644 --- a/docker/mongodb-kubernetes-tests/tests/multicluster/multi_cluster_backup_restore.py +++ b/docker/mongodb-kubernetes-tests/tests/multicluster/multi_cluster_backup_restore.py @@ -25,6 +25,7 @@ from kubetester.operator import Operator from kubetester.opsmanager import MongoDBOpsManager from kubetester.phase import Phase +import pytest from pytest import fixture, mark from tests.conftest import ( assert_data_got_restored, @@ -474,17 +475,9 @@ def test_mongodb_multi_one_running_state(self, mongodb_multi_one: MongoDBMulti): @skip_if_local @mark.e2e_multi_cluster_backup_restore + @pytest.mark.flaky(reruns=16, reruns_delay=6) def test_add_test_data(self, mongodb_multi_one_collection): - max_attempts = 100 - while max_attempts > 0: - try: - mongodb_multi_one_collection.insert_one(TEST_DATA) - return - except Exception as e: - print(e) - max_attempts -= 1 - time.sleep(6) - raise Exception("❌ Failed to insert test data after multiple attempts") + mongodb_multi_one_collection.insert_one(TEST_DATA) @mark.e2e_multi_cluster_backup_restore def test_mdb_backed_up(self, project_one: OMTester): diff --git a/docker/mongodb-kubernetes-tests/tests/multicluster/multi_cluster_dr_connect.py b/docker/mongodb-kubernetes-tests/tests/multicluster/multi_cluster_dr_connect.py index a2186995b..c74a0d018 100644 --- a/docker/mongodb-kubernetes-tests/tests/multicluster/multi_cluster_dr_connect.py +++ b/docker/mongodb-kubernetes-tests/tests/multicluster/multi_cluster_dr_connect.py @@ -54,18 +54,9 @@ def test_replica_set_is_reachable(mongodb_multi: MongoDBMulti): @pytest.mark.e2e_multi_cluster_dr +@pytest.mark.flaky(reruns=16, reruns_delay=6) def test_add_test_data(mongodb_multi_collection): - # TODO: remove this retry mechanism, for some reason the resource exits the running state and then - # enters it later. The subsequent test fails because the resource is not actually - max_attempts = 100 - while max_attempts > 0: - try: - mongodb_multi_collection.insert_one(TEST_DATA) - return - except Exception as e: - print(e) - max_attempts -= 1 - time.sleep(6) + mongodb_multi_collection.insert_one(TEST_DATA) @pytest.mark.e2e_multi_cluster_dr diff --git a/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_s3_based_backup_restore.py b/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_s3_based_backup_restore.py index e612d4127..2e187edb2 100644 --- a/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_s3_based_backup_restore.py +++ b/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_s3_based_backup_restore.py @@ -3,6 +3,7 @@ import kubernetes.client import pymongo +import pytest from kubetester import create_or_update_configmap, try_load from kubetester.kubetester import ensure_ent_version from kubetester.kubetester import fixture as yaml_fixture @@ -203,16 +204,9 @@ def test_mongodb_multi_one_running_state(self, mongodb_multi_one: MongoDBMulti): # we might fail connection in the beginning since we set a custom dns in coredns mongodb_multi_one.assert_reaches_phase(Phase.Running, ignore_errors=True, timeout=600) + @pytest.mark.flaky(reruns=16, reruns_delay=6) def test_add_test_data(self, mongodb_multi_one_collection): - max_attempts = 100 - while max_attempts > 0: - try: - mongodb_multi_one_collection.insert_one(TEST_DATA) - return - except Exception as e: - print(e) - max_attempts -= 1 - time.sleep(6) + mongodb_multi_one_collection.insert_one(TEST_DATA) def test_mdb_backed_up(self, project_one: OMTester): project_one.wait_until_backup_snapshots_are_ready(expected_count=1) diff --git a/docker/mongodb-kubernetes-tests/tests/multicluster_om/multicluster_om_appdb_no_mesh.py b/docker/mongodb-kubernetes-tests/tests/multicluster_om/multicluster_om_appdb_no_mesh.py index 7d6929f37..34014913c 100644 --- a/docker/mongodb-kubernetes-tests/tests/multicluster_om/multicluster_om_appdb_no_mesh.py +++ b/docker/mongodb-kubernetes-tests/tests/multicluster_om/multicluster_om_appdb_no_mesh.py @@ -557,16 +557,9 @@ def test_create_mongodb_multi(server_certs: str, mongodb_multi: MongoDBMulti): @skip_if_local @mark.e2e_multi_cluster_om_appdb_no_mesh +@pytest.mark.flaky(reruns=16, reruns_delay=6) def test_add_test_data(mongodb_multi_collection): - max_attempts = 100 - while max_attempts > 0: - try: - mongodb_multi_collection.insert_one(TEST_DATA) - return - except Exception as e: - print(e) - max_attempts -= 1 - time.sleep(6) + mongodb_multi_collection.insert_one(TEST_DATA) @mark.e2e_multi_cluster_om_appdb_no_mesh diff --git a/docker/mongodb-kubernetes-tests/tests/replicaset/replica_set_liveness_probe.py b/docker/mongodb-kubernetes-tests/tests/replicaset/replica_set_liveness_probe.py index 591d1240a..47e364587 100644 --- a/docker/mongodb-kubernetes-tests/tests/replicaset/replica_set_liveness_probe.py +++ b/docker/mongodb-kubernetes-tests/tests/replicaset/replica_set_liveness_probe.py @@ -25,26 +25,17 @@ def _get_pods(podname_template: str, qty: int = 3): @skip_if_static_containers @pytest.mark.e2e_replica_set_liveness_probe +@pytest.mark.flaky(reruns=10, reruns_delay=30) def test_pods_are_running(replica_set: MongoDB, namespace: str): corev1_client = client.CoreV1Api() running_pods: Set[str] = set() - tries = 10 # Wait for all the pods to be running # We can't wait for the replica set to be running # as it will never get to it (mongod is not starting) - while tries: - if len(running_pods) == 3: - break - for podname in _get_pods("my-replica-set-{}", 3): - try: - pod = corev1_client.read_namespaced_pod(podname, namespace) - if pod.status.phase == "Running": - running_pods.add(podname) - except: - # Pod not found, will retry - pass - tries -= 1 - time.sleep(30) + for podname in _get_pods("my-replica-set-{}", 3): + pod = corev1_client.read_namespaced_pod(podname, namespace) + if pod.status.phase == "Running": + running_pods.add(podname) assert len(running_pods) == 3 diff --git a/docker/mongodb-kubernetes-tests/tests/replicaset/replica_set_pv.py b/docker/mongodb-kubernetes-tests/tests/replicaset/replica_set_pv.py index bd81699f0..8556c330e 100644 --- a/docker/mongodb-kubernetes-tests/tests/replicaset/replica_set_pv.py +++ b/docker/mongodb-kubernetes-tests/tests/replicaset/replica_set_pv.py @@ -22,7 +22,7 @@ def test_replica_set_sts_exists(self): sts = self.appsv1.read_namespaced_stateful_set("rs001-pv", self.namespace) assert sts - @pytest.mark.flaky(reruns=3, reruns_delay=2) + @pytest.mark.flaky(reruns=15, reruns_delay=5) def test_sts_creation(self): sts = self.appsv1.read_namespaced_stateful_set("rs001-pv", self.namespace) From 5a7cc559205f6b3327abbacd79fae4e3818aa9a9 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Mon, 1 Sep 2025 11:05:12 +0200 Subject: [PATCH 3/5] use rerunfailures --- docker/mongodb-kubernetes-tests/tests/replicaset/replica_set.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/mongodb-kubernetes-tests/tests/replicaset/replica_set.py b/docker/mongodb-kubernetes-tests/tests/replicaset/replica_set.py index 5ba57314b..1fa76f93b 100644 --- a/docker/mongodb-kubernetes-tests/tests/replicaset/replica_set.py +++ b/docker/mongodb-kubernetes-tests/tests/replicaset/replica_set.py @@ -117,6 +117,7 @@ def test_replica_set_sts_exists(self): sts = self.appsv1.read_namespaced_stateful_set(RESOURCE_NAME, self.namespace) assert sts + @pytest.mark.flaky(reruns=15, reruns_delay=5) def test_sts_creation(self): sts = self.appsv1.read_namespaced_stateful_set(RESOURCE_NAME, self.namespace) From 81e5ca9322d4e839e8205439f617639513053ca5 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Mon, 1 Sep 2025 11:09:11 +0200 Subject: [PATCH 4/5] linter --- .../tests/multicluster/multi_cluster_backup_restore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/mongodb-kubernetes-tests/tests/multicluster/multi_cluster_backup_restore.py b/docker/mongodb-kubernetes-tests/tests/multicluster/multi_cluster_backup_restore.py index 57282fd4b..b07906959 100644 --- a/docker/mongodb-kubernetes-tests/tests/multicluster/multi_cluster_backup_restore.py +++ b/docker/mongodb-kubernetes-tests/tests/multicluster/multi_cluster_backup_restore.py @@ -5,6 +5,7 @@ import kubernetes import kubernetes.client import pymongo +import pytest from kubernetes import client from kubetester import ( create_or_update_configmap, @@ -25,7 +26,6 @@ from kubetester.operator import Operator from kubetester.opsmanager import MongoDBOpsManager from kubetester.phase import Phase -import pytest from pytest import fixture, mark from tests.conftest import ( assert_data_got_restored, From 1976f2d8b29f132c9d5197c358d30ccbf8a9efe1 Mon Sep 17 00:00:00 2001 From: Nam Nguyen Date: Mon, 1 Sep 2025 11:12:08 +0200 Subject: [PATCH 5/5] linter --- .../tests/multicluster/multi_cluster_backup_restore.py | 2 +- .../tests/multicluster/multi_cluster_dr_connect.py | 2 +- .../multicluster_appdb_s3_based_backup_restore.py | 2 +- .../tests/multicluster_om/multicluster_om_appdb_no_mesh.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/mongodb-kubernetes-tests/tests/multicluster/multi_cluster_backup_restore.py b/docker/mongodb-kubernetes-tests/tests/multicluster/multi_cluster_backup_restore.py index b07906959..981df49d4 100644 --- a/docker/mongodb-kubernetes-tests/tests/multicluster/multi_cluster_backup_restore.py +++ b/docker/mongodb-kubernetes-tests/tests/multicluster/multi_cluster_backup_restore.py @@ -475,7 +475,7 @@ def test_mongodb_multi_one_running_state(self, mongodb_multi_one: MongoDBMulti): @skip_if_local @mark.e2e_multi_cluster_backup_restore - @pytest.mark.flaky(reruns=16, reruns_delay=6) + @pytest.mark.flaky(reruns=100, reruns_delay=6) def test_add_test_data(self, mongodb_multi_one_collection): mongodb_multi_one_collection.insert_one(TEST_DATA) diff --git a/docker/mongodb-kubernetes-tests/tests/multicluster/multi_cluster_dr_connect.py b/docker/mongodb-kubernetes-tests/tests/multicluster/multi_cluster_dr_connect.py index c74a0d018..c2cc0d988 100644 --- a/docker/mongodb-kubernetes-tests/tests/multicluster/multi_cluster_dr_connect.py +++ b/docker/mongodb-kubernetes-tests/tests/multicluster/multi_cluster_dr_connect.py @@ -54,7 +54,7 @@ def test_replica_set_is_reachable(mongodb_multi: MongoDBMulti): @pytest.mark.e2e_multi_cluster_dr -@pytest.mark.flaky(reruns=16, reruns_delay=6) +@pytest.mark.flaky(reruns=100, reruns_delay=6) def test_add_test_data(mongodb_multi_collection): mongodb_multi_collection.insert_one(TEST_DATA) diff --git a/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_s3_based_backup_restore.py b/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_s3_based_backup_restore.py index 2e187edb2..b46744278 100644 --- a/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_s3_based_backup_restore.py +++ b/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_s3_based_backup_restore.py @@ -204,7 +204,7 @@ def test_mongodb_multi_one_running_state(self, mongodb_multi_one: MongoDBMulti): # we might fail connection in the beginning since we set a custom dns in coredns mongodb_multi_one.assert_reaches_phase(Phase.Running, ignore_errors=True, timeout=600) - @pytest.mark.flaky(reruns=16, reruns_delay=6) + @pytest.mark.flaky(reruns=100, reruns_delay=6) def test_add_test_data(self, mongodb_multi_one_collection): mongodb_multi_one_collection.insert_one(TEST_DATA) diff --git a/docker/mongodb-kubernetes-tests/tests/multicluster_om/multicluster_om_appdb_no_mesh.py b/docker/mongodb-kubernetes-tests/tests/multicluster_om/multicluster_om_appdb_no_mesh.py index 34014913c..820a5defb 100644 --- a/docker/mongodb-kubernetes-tests/tests/multicluster_om/multicluster_om_appdb_no_mesh.py +++ b/docker/mongodb-kubernetes-tests/tests/multicluster_om/multicluster_om_appdb_no_mesh.py @@ -557,7 +557,7 @@ def test_create_mongodb_multi(server_certs: str, mongodb_multi: MongoDBMulti): @skip_if_local @mark.e2e_multi_cluster_om_appdb_no_mesh -@pytest.mark.flaky(reruns=16, reruns_delay=6) +@pytest.mark.flaky(reruns=100, reruns_delay=6) def test_add_test_data(mongodb_multi_collection): mongodb_multi_collection.insert_one(TEST_DATA)