Merge branch 'master' into maciejk/ar-image-staging

# Conflicts:
#	scripts/release/tests/build_info_test.py

Author: MaciejKaras

.evergreen-functions.yml

@@ -336,6 +336,7 @@ functions:
     - command: shell.exec
       type: setup
       params:
+        continue_on_err: true
         shell: bash
         working_dir: src/github.com/mongodb/mongodb-kubernetes
         script: |
@@ -420,6 +421,7 @@ functions:
   upload_e2e_logs:
     - command: s3.put
       params:
+        continue_on_err: true
         aws_key: ${enterprise_aws_access_key_id}
         aws_secret: ${enterprise_aws_secret_access_key}
         local_files_include_filter:

.evergreen-periodic-builds.yaml

@@ -19,6 +19,15 @@ variables:
       - func: switch_context
 
 tasks:
+  - name: periodic_teardown_aws
+    commands:
+      - func: cleanup_aws
+
+  - name: periodic_teardown_cloudqa
+    commands:
+      - func: teardown_cloud_qa_all
+
+task_groups:
   - name: periodic_teardown_task_group
     <<: *setup_group
     tasks:

.evergreen.yml

@@ -117,9 +117,9 @@ variables:
       - func: setup_cloud_qa
     teardown_task_can_fail_task: true
     teardown_task:
+      - func: teardown_cloud_qa
       - func: upload_e2e_logs
       - func: teardown_kubernetes_environment
-      - func: teardown_cloud_qa
 
   - &setup_and_teardown_task
     setup_task_can_fail_task: true
@@ -156,6 +156,25 @@ variables:
       - name: build_agent_images_ubi
         variant: init_test_run
 
+  - &base_om7_dependency_with_race
+    depends_on:
+      - name: build_om_images
+        variant: build_om70_images
+      - name: build_operator_race_ubi
+        variant: init_test_run
+      - name: build_init_database_image_ubi
+        variant: init_test_run
+      - name: build_database_image_ubi
+        variant: init_test_run
+      - name: build_test_image
+        variant: init_test_run
+      - name: build_init_appdb_images_ubi
+        variant: init_test_run
+      - name: build_init_om_images_ubi
+        variant: init_test_run
+      - name: build_agent_images_ubi
+        variant: init_test_run
+
   - &base_om8_dependency
     depends_on:
       - name: build_om_images
@@ -439,18 +458,23 @@ tasks:
       - func: setup_building_host
       - func: pipeline
         vars:
-          skip_tags: ubuntu,release
-          distro: ubi
           image_name: operator
 
+  - name: build_operator_race_ubi
+    commands:
+      - func: clone
+      - func: setup_building_host
+      - func: pipeline
+        vars:
+          image_name: operator-race
+
   - name: build_init_om_images_ubi
     commands:
       - func: clone
       - func: setup_building_host
       - func: pipeline
         vars:
           image_name: init-ops-manager
-          skip_tags: ubuntu,release
 
   - name: build_init_appdb_images_ubi
     commands:
@@ -459,7 +483,6 @@ tasks:
       - func: pipeline
         vars:
           image_name: init-appdb
-          skip_tags: ubuntu,release
 
   - name: build_agent_images_ubi
     commands:
@@ -468,7 +491,6 @@ tasks:
       - func: pipeline
         vars:
           image_name: agent
-          skip_tags: ubuntu,release
 
   - name: build_init_database_image_ubi
     commands:
@@ -477,7 +499,6 @@ tasks:
       - func: pipeline
         vars:
           image_name: init-database
-          skip_tags: ubuntu,release
 
   - name: build_database_image_ubi
     commands:
@@ -486,7 +507,6 @@ tasks:
       - func: pipeline
         vars:
           image_name: database
-          skip_tags: ubuntu,release
 
   - name: build_readiness_probe_image
     commands:
@@ -495,7 +515,6 @@ tasks:
       - func: pipeline
         vars:
           image_name: readiness-probe
-          skip_tags: ubuntu,release
 
   - name: build_upgrade_hook_image
     commands:
@@ -504,7 +523,6 @@ tasks:
       - func: pipeline
         vars:
           image_name: upgrade-hook
-          skip_tags: ubuntu,release
 
   - name: prepare_aws
     priority: 59
@@ -1434,7 +1452,7 @@ buildvariants:
     tags: [ "e2e_test_suite" ]
     run_on:
       - ubuntu1804-xlarge
-    <<: *base_om7_dependency
+    <<: *base_om7_dependency_with_race
     tasks:
       - name: e2e_operator_race_with_telemetry_task_group
 
@@ -1653,6 +1671,7 @@ buildvariants:
       - ubuntu2204-small
     tasks:
       - name: build_operator_ubi
+      - name: build_operator_race_ubi
       - name: build_test_image
       - name: build_mco_test_image
       - name: build_init_appdb_images_ubi

build_info.json

@@ -23,6 +23,22 @@
         ]
       }
     },
+    "operator-race": {
+      "dockerfile-path": "docker/mongodb-kubernetes-operator/Dockerfile.atomic",
+      "patch": {
+        "repository": "268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes",
+        "platforms": [
+          "linux/amd64"
+        ]
+      },
+      "staging": {
+        "sign": true,
+        "repository": "268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes",
+        "platforms": [
+          "linux/amd64"
+        ]
+      }
+    },
     "init-database": {
       "dockerfile-path": "docker/mongodb-kubernetes-init-database/Dockerfile.atomic",
       "patch": {

changelog/20250808_fix_fixing_auth_transition_edge_cases.md

@@ -0,0 +1,7 @@
+---
+title: Fixing auth transition edge-cases
+kind: fix
+date: 2025-08-08
+---
+
+* Fixed an issue where the readiness probe reported the node as ready even when its authentication mechanism was not in sync with the other nodes, potentially causing premature restarts.

controllers/om/automation_status.go

@@ -15,7 +15,7 @@ import (
 	"github.com/mongodb/mongodb-kubernetes/pkg/util/stringutil"
 )
 
-const automationAgentKubeUpgradePlan = "ChangeVersionKube"
+const automationAgentKubeUpgradeMove = "ChangeVersionKube"
 
 // AutomationStatus represents the status of automation agents registered with Ops Manager
 type AutomationStatus struct {
@@ -85,12 +85,25 @@ func checkAutomationStatusIsGoal(as *AutomationStatus, relevantProcesses []strin
 
 	goalsNotAchievedMap := map[string]int{}
 	goalsAchievedMap := map[string]int{}
+	authTransitionsInProgress := map[string]string{}
+
 	for _, p := range as.Processes {
 		if !stringutil.Contains(relevantProcesses, p.Name) {
 			continue
 		}
 		if p.LastGoalVersionAchieved == as.GoalVersion {
 			goalsAchievedMap[p.Name] = p.LastGoalVersionAchieved
+
+			// Check if authentication transitions are in the current plan.
+			// If a process has reached goal version but still has auth-related moves in plan,
+			// it means authentication transition is likely in progress.
+			// The plan contains non-completed move names from the API.
+			for _, move := range p.Plan {
+				if isAuthenticationTransitionMove(move) {
+					authTransitionsInProgress[p.Name] = move
+					break
+				}
+			}
 		} else {
 			goalsNotAchievedMap[p.Name] = p.LastGoalVersionAchieved
 		}
@@ -103,6 +116,18 @@ func checkAutomationStatusIsGoal(as *AutomationStatus, relevantProcesses []strin
 	goalsAchievedMsgList := slices.Collect(maps.Keys(goalsAchievedMap))
 	sort.Strings(goalsAchievedMsgList)
 
+	// Check if any authentication transitions are in progress
+	if len(authTransitionsInProgress) > 0 {
+		var authTransitionMsgList []string
+		for processName, step := range authTransitionsInProgress {
+			authTransitionMsgList = append(authTransitionMsgList, fmt.Sprintf("%s:%s", processName, step))
+		}
+		log.Infow("Authentication transitions still in progress, waiting for completion",
+			"processes", authTransitionMsgList)
+		return false, fmt.Sprintf("authentication transitions in progress for %d processes: %s",
+			len(authTransitionsInProgress), authTransitionMsgList)
+	}
+
 	if len(goalsNotAchievedMap) > 0 {
 		return false, fmt.Sprintf("%d processes waiting to reach automation config goal state (version=%d): %s, %d processes reached goal state: %s",
 			len(goalsNotAchievedMap), as.GoalVersion, goalsNotAchievedMsgList, len(goalsAchievedMsgList), goalsAchievedMsgList)
@@ -113,17 +138,29 @@ func checkAutomationStatusIsGoal(as *AutomationStatus, relevantProcesses []strin
 	}
 }
 
+// isAuthenticationTransitionMove returns true if the given move is related to authentication transitions
+func isAuthenticationTransitionMove(move string) bool {
+	authMoves := map[string]struct{}{
+		"UpdateAuth":     {},
+		"WaitAuthUpdate": {},
+	}
+
+	_, ok := authMoves[move]
+
+	return ok
+}
+
 func areAnyAgentsInKubeUpgradeMode(as *AutomationStatus, relevantProcesses []string, log *zap.SugaredLogger) bool {
 	for _, p := range as.Processes {
 		if !stringutil.Contains(relevantProcesses, p.Name) {
 			continue
 		}
-		for _, plan := range p.Plan {
+		for _, move := range p.Plan {
 			// This means the following:
 			// - the cluster is in static architecture
 			// - the agents are in a dedicated upgrade process, waiting for their binaries to be replaced by kubernetes
 			// - this can only happen if the statefulset is ready, therefore we are returning ready here
-			if plan == automationAgentKubeUpgradePlan {
+			if move == automationAgentKubeUpgradeMove {
 				log.Debug("cluster is in changeVersionKube mode, returning the agent is ready.")
 				return true
 			}