add user wait and ac uplload

Author: nammn

docker/mongodb-kubernetes-tests/kubetester/mongotester.py

@@ -9,14 +9,19 @@
 from typing import Callable, Dict, List, Optional
 
 import pymongo
-from kubetester import kubetester
-from kubetester.kubetester import KubernetesTester
 from opentelemetry import trace
 from pycognito import Cognito
 from pymongo.auth_oidc import OIDCCallback, OIDCCallbackContext, OIDCCallbackResult
 from pymongo.errors import OperationFailure, PyMongoError, ServerSelectionTimeoutError
 from pytest import fail
 
+from tests.conftest import get_central_cluster_client
+import kubernetes.client as client
+from kubetester import kubetester
+from kubetester.kubetester import KubernetesTester
+from kubetester.mongodb_user import MongoDBUser
+from kubetester.phase import Phase
+
 TEST_DB = "test-db"
 TEST_COLLECTION = "test-collection"
 
@@ -76,6 +81,57 @@ def fetch(self, context: OIDCCallbackContext) -> OIDCCallbackResult:
         return OIDCCallbackResult(access_token=u.id_token)
 
 
+def _wait_for_mongodbuser_reconciliation() -> None:
+    """
+    Wait for ALL MongoDBUser resources in the namespace to be reconciled before attempting authentication.
+    This prevents race conditions when passwords or user configurations have been recently changed.
+
+    Lists all MongoDBUser resources in the namespace and waits for ALL of them to reach Updated phase.
+    """
+    try:
+        namespace = KubernetesTester.get_namespace()
+        api_client = client.CustomObjectsApi(api_client=get_central_cluster_client())
+
+        try:
+            mongodb_users = api_client.list_namespaced_custom_object(
+                group="mongodb.com",
+                version="v1",
+                namespace=namespace,
+                plural="mongodbusers"
+            )
+
+            all_users = []
+
+            for user_item in mongodb_users.get("items", []):
+                user_name = user_item.get("metadata", {}).get("name", "unknown")
+                username = user_item.get("spec", {}).get("username", "unknown")
+                all_users.append((user_name, username))
+
+            if not all_users:
+                return
+
+            logging.info(f"Found {len(all_users)} MongoDBUser resource(s) in namespace '{namespace}', waiting for all to reach Updated phase...")
+
+            for user_name, username in all_users:
+                try:
+                    logging.info(f"Waiting for MongoDBUser '{user_name}' (username: {username}) to reach Updated phase...")
+
+                    user = MongoDBUser(name=user_name, namespace=namespace)
+                    user.assert_reaches_phase(Phase.Updated, timeout=300)
+                    logging.info(f"MongoDBUser '{user_name}' reached Updated phase - reconciliation complete")
+
+                except Exception as e:
+                    logging.warning(f"Failed to wait for MongoDBUser '{user_name}' reconciliation: {e}")
+                    # Continue with other users - don't fail the entire test
+
+            logging.info("All MongoDBUser resources reconciliation check complete")
+
+        except Exception as e:
+            logging.warning(f"Failed to list MongoDBUser resources: {e} - proceeding without reconciliation wait")
+    except Exception as e:
+        logging.warning(f"Error while waiting for MongoDBUser reconciliation: {e} - proceeding with authentication")
+
+
 class MongoTester:
     """MongoTester is a general abstraction to work with mongo database. It encapsulates the client created in
     the constructor. All general methods non-specific to types of mongodb topologies should reside here."""
@@ -115,7 +171,7 @@ def _init_client(self, **kwargs):
 
     def assert_connectivity(
         self,
-        attempts: int = 30,
+        attempts: int = 50,
         db: str = "admin",
         col: str = "myCol",
         opts: Optional[List[Dict[str, any]]] = None,
@@ -175,13 +231,17 @@ def assert_scram_sha_authentication(
         username: str,
         password: str,
         auth_mechanism: str,
-        attempts: int = 30,
+        attempts: int = 50,
         ssl: bool = False,
         **kwargs,
     ) -> None:
         assert attempts > 0
         assert auth_mechanism in {"SCRAM-SHA-256", "SCRAM-SHA-1"}
 
+        # Wait for ALL MongoDBUser resources to be reconciled before attempting authentication
+        # This prevents race conditions when passwords have been recently changed
+        _wait_for_mongodbuser_reconciliation()
+
         for i in reversed(range(attempts)):
             try:
                 self._authenticate_with_scram(
@@ -209,7 +269,7 @@ def assert_scram_sha_authentication_fails(
         self,
         username: str,
         password: str,
-        retries: int = 30,
+        attempts: int = 50,
         ssl: bool = False,
         **kwargs,
     ):
@@ -219,13 +279,16 @@ def assert_scram_sha_authentication_fails(
         which still exists. When we change a password, we should eventually no longer be able to auth with
         that user's credentials.
         """
-        for i in range(retries):
+
+        _wait_for_mongodbuser_reconciliation()
+
+        for i in range(attempts):
             try:
                 self._authenticate_with_scram(username, password, ssl=ssl, **kwargs)
             except OperationFailure:
                 return
             time.sleep(5)
-        fail(f"was still able to authenticate with username={username} password={password} after {retries} attempts")
+        fail(f"was still able to authenticate with username={username} password={password} after {attempts} attempts")
 
     def _authenticate_with_scram(
         self,
@@ -247,9 +310,11 @@ def _authenticate_with_scram(
         # authentication doesn't actually happen until we interact with a database
         self.client["admin"]["myCol"].insert_one({})
 
-    def assert_x509_authentication(self, cert_file_name: str, attempts: int = 30, **kwargs):
+    def assert_x509_authentication(self, cert_file_name: str, attempts: int = 50, **kwargs):
         assert attempts > 0
 
+        _wait_for_mongodbuser_reconciliation()
+
         options = self._merge_options(
             [
                 with_x509(cert_file_name, kwargs.get("tlsCAFile", kubetester.SSL_CA_CERT)),
@@ -267,14 +332,7 @@ def assert_x509_authentication(self, cert_file_name: str, attempts: int = 30, **
                 if attempts == 0:
                     fail(f"unable to authenticate after {total_attempts} attempts")
 
-                # Use adaptive retry delays for better credential propagation handling
-                remaining_attempts = attempts
-                if remaining_attempts >= total_attempts * 0.7:  # First ~30% of attempts
-                    delay = 10  # Longer delay for initial propagation
-                else:
-                    delay = 5   # Standard delay for normal retries
-
-                time.sleep(delay)
+                time.sleep(5)
 
     def assert_ldap_authentication(
         self,
@@ -284,8 +342,9 @@ def assert_ldap_authentication(
         collection: str = "myCol",
         tls_ca_file: Optional[str] = None,
         ssl_certfile: str = None,
-        attempts: int = 30,
+        attempts: int = 50,
     ):
+        _wait_for_mongodbuser_reconciliation()
 
         options = with_ldap(ssl_certfile, tls_ca_file)
         total_attempts = attempts
@@ -307,23 +366,18 @@ def assert_ldap_authentication(
                 if attempts <= 0:
                     fail(f"unable to authenticate after {total_attempts} attempts")
 
-                # Use adaptive retry delays for better credential propagation handling
-                remaining_attempts = attempts
-                if remaining_attempts >= total_attempts * 0.7:  # First ~30% of attempts
-                    delay = 10  # Longer delay for initial propagation
-                else:
-                    delay = 5   # Standard delay for normal retries
-
-                time.sleep(delay)
+                time.sleep(5)
 
     def assert_oidc_authentication(
         self,
         db: str = "admin",
         collection: str = "myCol",
-        attempts: int = 10,
+        attempts: int = 50,
     ):
         assert attempts > 0
 
+        _wait_for_mongodbuser_reconciliation()
+
         props = {"OIDC_CALLBACK": MyOIDCCallback()}
 
         total_attempts = attempts
@@ -342,14 +396,7 @@ def assert_oidc_authentication(
                 if attempts == 0:
                     raise RuntimeError(f"Unable to authenticate after {total_attempts} attempts: {e}")
 
-                # Use adaptive retry delays for better credential propagation handling
-                remaining_attempts = attempts
-                if remaining_attempts >= total_attempts * 0.7:  # First ~30% of attempts
-                    delay = 10  # Longer delay for initial propagation
-                else:
-                    delay = 5   # Standard delay for normal retries
-
-                time.sleep(delay)
+                time.sleep(5)
 
     def assert_oidc_authentication_fails(self, db: str = "admin", collection: str = "myCol", attempts: int = 10):
         assert attempts > 0

docker/mongodb-kubernetes-tests/tests/conftest.py

@@ -1684,6 +1684,20 @@ def pytest_sessionfinish(session, exitstatus):
                     ev = tester.get_project_events().json()["results"]
                     with open(f"/tmp/diagnostics/{project_id}-events.json", "w", encoding="utf-8") as f:
                         json.dump(ev, f, ensure_ascii=False, indent=4)
+
+                    if exitstatus != 0:
+                        try:
+                            automation_config_tester = tester.get_automation_config_tester()
+                            automation_config = automation_config_tester.automation_config
+                            if not automation_config:
+                                continue
+
+                            with open(f"/tmp/diagnostics/{project_id}-automation-config.json", "w", encoding="utf-8") as f:
+                                json.dump(automation_config, f, ensure_ascii=False, indent=4)
+
+                            logging.info(f"Saved automation config for project {project_id}")
+                        except Exception as e:
+                            logging.warning(f"Failed to collect automation config for project {project_id}: {e}")
                 else:
                     logging.info("om is not healthy - not collecting events information")