Fix multi-arch smoke setup and some flaky test (#387)

# Summary

This pull request introduces several improvements to CI/CD
configuration, test reliability, and scripting. The main changes are the
expansion of allowed requesters for certain build variants, enhancements
to the authentication test fixtures for better isolation and
reliability, and a simplification of the AWS CLI installation script.

**Test Improvements:**

* Changed test fixtures in `replica_set_scram_sha_256_connectivity.py`
from module scope to function scope, ensuring a fresh environment for
each test and reducing side effects between tests.
* Refactored secret creation and resource loading in authentication
tests to use `try_load` and explicit secret management, improving test
reliability and clarity.
* Removed unnecessary `attempts` parameter from the authentication check
in sharded cluster test, simplifying the test logic.
* Added "commit" as an allowed requester in several `buildvariants`
sections of `.evergreen.yml`, enabling these builds to be triggered on
direct commits in addition to patches and GitHub tags. -> i hope that
fixes smoke tests to run on master merges
* Updated `install_aws_cli_pip()` in `scripts/evergreen/setup_aws.sh` to
check if AWS CLI is already installed and working before attempting
installation, preventing unnecessary reinstalls.
* Simplified the installation process to always use `pip3` and improved
error handling and messaging for installation failures.

# Proof

- green ci
- [openshift test passed as
well](https://spruce.mongodb.com/version/68b1a0d262980b0007cba112)
- ibm worked manually triggered the build:
[PATCH](https://spruce.mongodb.com/version/68b195311add1b00074c5d51/tasks?sorts=STATUS%3AASC%3BBASE_STATUS%3ADESC)
- in this patch no smoke and ibm stuff is running
- after merge ibm stuff is triggered

## Checklist

- [x] Have you linked a jira ticket and/or is the ticket in the title?
- [x] Have you checked whether your jira ticket required DOCSP changes?
- [x] Have you added changelog file?
    - use `skip-changelog` label if not needed
- refer to [Changelog files and Release
Notes](https://github.com/mongodb/mongodb-kubernetes/blob/master/CONTRIBUTING.md#changelog-files-and-release-notes)
section in CONTRIBUTING.md for more details

Author: nammn

.evergreen.yml

@@ -1475,7 +1475,7 @@ buildvariants:
     run_on:
       - rhel9-power-small
       - rhel9-power-large
-    allowed_requesters: [ "patch", "github_tag" ]
+    allowed_requesters: [ "patch", "github_tag" , "commit"]
     depends_on:
       - name: build_operator_ubi
         variant: init_test_run
@@ -1498,7 +1498,7 @@ buildvariants:
     run_on:
       - rhel9-zseries-small
       - rhel9-zseries-large
-    allowed_requesters: [ "patch", "github_tag" ]
+    allowed_requesters: [ "patch", "github_tag", "commit"]
     depends_on:
       - name: build_operator_ubi
         variant: init_test_run
@@ -1520,7 +1520,7 @@ buildvariants:
     tags: [ "e2e_test_suite", "e2e_smoke_release_test_suite" ]
     run_on:
       - ubuntu2204-arm64-large
-    allowed_requesters: [ "patch", "github_tag" ]
+    allowed_requesters: [ "patch", "github_tag", "commit"]
     <<: *base_no_om_image_dependency
     tasks:
       - name: e2e_smoke_arm_task_group
@@ -1530,7 +1530,7 @@ buildvariants:
     tags: [ "e2e_test_suite", "e2e_smoke_release_test_suite", "static" ]
     run_on:
       - ubuntu2204-arm64-large
-    allowed_requesters: [ "patch", "github_tag" ]
+    allowed_requesters: [ "patch", "github_tag", "commit"]
     <<: *base_no_om_image_dependency
     tasks:
       - name: e2e_smoke_arm_task_group
@@ -1541,7 +1541,7 @@ buildvariants:
     run_on:
       - rhel9-zseries-small
       - rhel9-zseries-large
-    allowed_requesters: [ "patch", "github_tag" ]
+    allowed_requesters: [ "patch", "github_tag", "commit"]
     depends_on:
       - name: build_operator_ubi
         variant: init_test_run
@@ -1564,7 +1564,7 @@ buildvariants:
     run_on:
       - rhel9-power-small
       - rhel9-power-large
-    allowed_requesters: [ "patch", "github_tag" ]
+    allowed_requesters: [ "patch", "github_tag", "commit"]
     depends_on:
       - name: build_operator_ubi
         variant: init_test_run

docker/mongodb-kubernetes-tests/tests/authentication/replica_set_scram_sha_256_connectivity.py

@@ -5,6 +5,7 @@
     create_secret,
     find_fixture,
     read_secret,
+    try_load,
     update_secret,
     wait_until,
 )
@@ -22,7 +23,16 @@
 USER_DATABASE = "admin"
 
 
-@fixture(scope="module")
+def create_password_secret(namespace: str) -> str:
+    create_or_update_secret(
+        namespace,
+        PASSWORD_SECRET_NAME,
+        {"password": USER_PASSWORD},
+    )
+    return PASSWORD_SECRET_NAME
+
+
+@fixture(scope="function")
 def replica_set(namespace: str, custom_mdb_version) -> MongoDB:
     resource = MongoDB.from_yaml(
         find_fixture("replica-set-scram-sha-256.yaml"),
@@ -36,37 +46,33 @@ def replica_set(namespace: str, custom_mdb_version) -> MongoDB:
         "enabled": True,
         "modes": ["SCRAM"],
     }
-
-    return resource.update()
+    try_load(resource)
+    return resource
 
 
-@fixture(scope="module")
+@fixture(scope="function")
 def scram_user(namespace: str) -> MongoDBUser:
     resource = MongoDBUser.from_yaml(find_fixture("scram-sha-user.yaml"), namespace=namespace)
 
-    create_or_update_secret(
-        KubernetesTester.get_namespace(),
-        resource.get_secret_name(),
-        {"password": USER_PASSWORD},
-    )
-
-    return resource.update()
+    try_load(resource)
+    return resource
 
 
-@fixture(scope="module")
+@fixture(scope="function")
 def standard_secret(replica_set: MongoDB):
     secret_name = "{}-{}-{}".format(replica_set.name, USER_NAME, USER_DATABASE)
     return read_secret(replica_set.namespace, secret_name)
 
 
-@fixture(scope="module")
+@fixture(scope="function")
 def connection_string_secret(replica_set: MongoDB):
     return read_secret(replica_set.namespace, CONNECTION_STRING_SECRET_NAME)
 
 
 @mark.e2e_replica_set_scram_sha_256_user_connectivity
 class TestReplicaSetCreation(KubernetesTester):
     def test_replica_set_created(self, replica_set: MongoDB):
+        replica_set.update()
         replica_set.assert_reaches_phase(Phase.Running, timeout=400)
 
     def test_replica_set_connectivity(self, replica_set: MongoDB):
@@ -82,7 +88,9 @@ def test_ops_manager_state_correctly_updated(self, replica_set: MongoDB):
 
 
 @mark.e2e_replica_set_scram_sha_256_user_connectivity
-def test_create_user(scram_user: MongoDBUser):
+def test_create_user(scram_user: MongoDBUser, namespace: str):
+    create_password_secret(namespace)
+    scram_user.update()
     scram_user.assert_reaches_phase(Phase.Updated)
 
 
@@ -125,9 +133,15 @@ def test_user_cannot_authenticate_with_incorrect_password(self, replica_set: Mon
 @mark.e2e_replica_set_scram_sha_256_user_connectivity
 class TestCanChangePassword(KubernetesTester):
     def test_user_can_authenticate_with_new_password(self, namespace: str, replica_set: MongoDB):
-        update_secret(namespace, PASSWORD_SECRET_NAME, {"password": "my-new-password7"})
+        ac_version = replica_set.get_automation_config_tester().automation_config["version"]
+
+        new_password = "my-new-password7"
+        update_secret(namespace, PASSWORD_SECRET_NAME, {"password": new_password})
+
+        wait_until(lambda: replica_set.get_automation_config_tester().reached_version(ac_version + 1), timeout=800)
+
         replica_set.tester().assert_scram_sha_authentication(
-            password="my-new-password7",
+            password=new_password,
             username="mms-user-1",
             auth_mechanism="SCRAM-SHA-256",
         )

docker/mongodb-kubernetes-tests/tests/authentication/sharded_cluster_scram_sha_256_connectivity.py

@@ -117,7 +117,6 @@ def test_user_can_authenticate_with_new_password(self):
             password="my-new-password",
             username="mms-user-1",
             auth_mechanism="SCRAM-SHA-256",
-            attempts=20,
         )
 
     def test_user_cannot_authenticate_with_old_password(self):

scripts/evergreen/setup_aws.sh

@@ -48,14 +48,14 @@ install_aws_cli_pip() {
         return 1
     fi
 
-    # Use pip3 if available, otherwise pip
-    local pip_cmd="pip3"
-    if ! command -v pip3 &> /dev/null; then
-        pip_cmd="pip"
+    # Check if AWS CLI exists and works before installing
+    if command -v aws &> /dev/null && aws --version &> /dev/null 2>&1; then
+        echo "AWS CLI is already installed and working"
+        return 0
     fi
 
-    echo "Installing AWS CLI using ${pip_cmd}..."
-    ${pip_cmd} install --user awscli
+    echo "Installing AWS CLI using pip3..."
+    pip3 install --user awscli
 
     # Add ~/.local/bin to PATH if not already there (where pip --user installs)
     if [[ ":${PATH}:" != *":${HOME}/.local/bin:"* ]]; then
@@ -65,10 +65,9 @@ install_aws_cli_pip() {
 
     # Verify installation
     if command -v aws &> /dev/null; then
-        echo "AWS CLI v1 installed successfully:"
-        aws --version
+        echo "AWS CLI v1 installed successfully"
     else
-        echo "Error: AWS CLI v1 installation failed or not found in PATH" >&2
+        echo "Error: AWS CLI v1 installation failed" >&2
         return 1
     fi
 }