Merge branch 'maciejk/ar-image-staging' into maciejk/ar-image-release

# Conflicts:
#	scripts/release/build/build_info.py

Author: MaciejKaras

.evergreen-functions.yml

@@ -336,6 +336,7 @@ functions:
     - command: shell.exec
       type: setup
       params:
+        continue_on_err: true
         shell: bash
         working_dir: src/github.com/mongodb/mongodb-kubernetes
         script: |
@@ -420,6 +421,7 @@ functions:
   upload_e2e_logs:
     - command: s3.put
       params:
+        continue_on_err: true
         aws_key: ${enterprise_aws_access_key_id}
         aws_secret: ${enterprise_aws_secret_access_key}
         local_files_include_filter:

.evergreen-periodic-builds.yaml

@@ -19,6 +19,15 @@ variables:
       - func: switch_context
 
 tasks:
+  - name: periodic_teardown_aws
+    commands:
+      - func: cleanup_aws
+
+  - name: periodic_teardown_cloudqa
+    commands:
+      - func: teardown_cloud_qa_all
+
+task_groups:
   - name: periodic_teardown_task_group
     <<: *setup_group
     tasks:

.evergreen.yml

@@ -118,9 +118,9 @@ variables:
       - func: setup_cloud_qa
     teardown_task_can_fail_task: true
     teardown_task:
+      - func: teardown_cloud_qa
       - func: upload_e2e_logs
       - func: teardown_kubernetes_environment
-      - func: teardown_cloud_qa
 
   - &setup_and_teardown_task
     setup_task_can_fail_task: true
@@ -157,6 +157,25 @@ variables:
       - name: build_agent_images_ubi
         variant: init_test_run
 
+  - &base_om7_dependency_with_race
+    depends_on:
+      - name: build_om_images
+        variant: build_om70_images
+      - name: build_operator_race_ubi
+        variant: init_test_run
+      - name: build_init_database_image_ubi
+        variant: init_test_run
+      - name: build_database_image_ubi
+        variant: init_test_run
+      - name: build_test_image
+        variant: init_test_run
+      - name: build_init_appdb_images_ubi
+        variant: init_test_run
+      - name: build_init_om_images_ubi
+        variant: init_test_run
+      - name: build_agent_images_ubi
+        variant: init_test_run
+
   - &base_om8_dependency
     depends_on:
       - name: build_om_images
@@ -376,6 +395,14 @@ tasks:
         vars:
           image_name: operator
 
+  - name: build_operator_race_ubi
+    commands:
+      - func: clone
+      - func: setup_building_host
+      - func: pipeline
+        vars:
+          image_name: operator-race
+
   - name: build_init_om_images_ubi
     commands:
       - func: clone
@@ -1323,7 +1350,7 @@ buildvariants:
     tags: [ "e2e_test_suite" ]
     run_on:
       - ubuntu1804-xlarge
-    <<: *base_om7_dependency
+    <<: *base_om7_dependency_with_race
     tasks:
       - name: e2e_operator_race_with_telemetry_task_group
 
@@ -1518,6 +1545,7 @@ buildvariants:
       - ubuntu2204-small
     tasks:
       - name: build_operator_ubi
+      - name: build_operator_race_ubi
       - name: build_test_image
       - name: build_mco_test_image
       - name: build_init_appdb_images_ubi

build_info.json

@@ -24,6 +24,22 @@
         ]
       }
     },
+    "operator-race": {
+      "dockerfile-path": "docker/mongodb-kubernetes-operator/Dockerfile.atomic",
+      "patch": {
+        "repository": "268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes",
+        "platforms": [
+          "linux/amd64"
+        ]
+      },
+      "staging": {
+        "sign": true,
+        "repository": "268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes",
+        "platforms": [
+          "linux/amd64"
+        ]
+      }
+    },
     "init-database": {
       "dockerfile-path": "docker/mongodb-kubernetes-init-database/Dockerfile.atomic",
       "patch": {

changelog/20250808_fix_fixing_auth_transition_edge_cases.md

@@ -0,0 +1,7 @@
+---
+title: Fixing auth transition edge-cases
+kind: fix
+date: 2025-08-08
+---
+
+* Fixed an issue where the readiness probe reported the node as ready even when its authentication mechanism was not in sync with the other nodes, potentially causing premature restarts.

controllers/om/automation_status.go

@@ -15,7 +15,7 @@ import (
 	"github.com/mongodb/mongodb-kubernetes/pkg/util/stringutil"
 )
 
-const automationAgentKubeUpgradePlan = "ChangeVersionKube"
+const automationAgentKubeUpgradeMove = "ChangeVersionKube"
 
 // AutomationStatus represents the status of automation agents registered with Ops Manager
 type AutomationStatus struct {
@@ -85,12 +85,25 @@ func checkAutomationStatusIsGoal(as *AutomationStatus, relevantProcesses []strin
 
 	goalsNotAchievedMap := map[string]int{}
 	goalsAchievedMap := map[string]int{}
+	authTransitionsInProgress := map[string]string{}
+
 	for _, p := range as.Processes {
 		if !stringutil.Contains(relevantProcesses, p.Name) {
 			continue
 		}
 		if p.LastGoalVersionAchieved == as.GoalVersion {
 			goalsAchievedMap[p.Name] = p.LastGoalVersionAchieved
+
+			// Check if authentication transitions are in the current plan.
+			// If a process has reached goal version but still has auth-related moves in plan,
+			// it means authentication transition is likely in progress.
+			// The plan contains non-completed move names from the API.
+			for _, move := range p.Plan {
+				if isAuthenticationTransitionMove(move) {
+					authTransitionsInProgress[p.Name] = move
+					break
+				}
+			}
 		} else {
 			goalsNotAchievedMap[p.Name] = p.LastGoalVersionAchieved
 		}
@@ -103,6 +116,18 @@ func checkAutomationStatusIsGoal(as *AutomationStatus, relevantProcesses []strin
 	goalsAchievedMsgList := slices.Collect(maps.Keys(goalsAchievedMap))
 	sort.Strings(goalsAchievedMsgList)
 
+	// Check if any authentication transitions are in progress
+	if len(authTransitionsInProgress) > 0 {
+		var authTransitionMsgList []string
+		for processName, step := range authTransitionsInProgress {
+			authTransitionMsgList = append(authTransitionMsgList, fmt.Sprintf("%s:%s", processName, step))
+		}
+		log.Infow("Authentication transitions still in progress, waiting for completion",
+			"processes", authTransitionMsgList)
+		return false, fmt.Sprintf("authentication transitions in progress for %d processes: %s",
+			len(authTransitionsInProgress), authTransitionMsgList)
+	}
+
 	if len(goalsNotAchievedMap) > 0 {
 		return false, fmt.Sprintf("%d processes waiting to reach automation config goal state (version=%d): %s, %d processes reached goal state: %s",
 			len(goalsNotAchievedMap), as.GoalVersion, goalsNotAchievedMsgList, len(goalsAchievedMsgList), goalsAchievedMsgList)
@@ -113,17 +138,29 @@ func checkAutomationStatusIsGoal(as *AutomationStatus, relevantProcesses []strin
 	}
 }
 
+// isAuthenticationTransitionMove returns true if the given move is related to authentication transitions
+func isAuthenticationTransitionMove(move string) bool {
+	authMoves := map[string]struct{}{
+		"UpdateAuth":     {},
+		"WaitAuthUpdate": {},
+	}
+
+	_, ok := authMoves[move]
+
+	return ok
+}
+
 func areAnyAgentsInKubeUpgradeMode(as *AutomationStatus, relevantProcesses []string, log *zap.SugaredLogger) bool {
 	for _, p := range as.Processes {
 		if !stringutil.Contains(relevantProcesses, p.Name) {
 			continue
 		}
-		for _, plan := range p.Plan {
+		for _, move := range p.Plan {
 			// This means the following:
 			// - the cluster is in static architecture
 			// - the agents are in a dedicated upgrade process, waiting for their binaries to be replaced by kubernetes
 			// - this can only happen if the statefulset is ready, therefore we are returning ready here
-			if plan == automationAgentKubeUpgradePlan {
+			if move == automationAgentKubeUpgradeMove {
 				log.Debug("cluster is in changeVersionKube mode, returning the agent is ready.")
 				return true
 			}

controllers/om/automation_status_test.go

@@ -75,7 +75,7 @@ func TestCheckAutomationStatusIsGoal(t *testing.T) {
 						},
 						{
 							Name:                    "b",
-							Plan:                    []string{"FCV", automationAgentKubeUpgradePlan},
+							Plan:                    []string{"FCV", automationAgentKubeUpgradeMove},
 							LastGoalVersionAchieved: 1,
 						},
 					},
@@ -119,3 +119,132 @@ func TestCheckAutomationStatusIsGoal(t *testing.T) {
 		})
 	}
 }
+
+func TestCheckAutomationStatusIsGoal_AuthenticationTransitions(t *testing.T) {
+	logger := zap.NewNop().Sugar()
+
+	tests := []struct {
+		name              string
+		automationStatus  *AutomationStatus
+		relevantProcesses []string
+		expectedReady     bool
+		expectedMessage   string
+	}{
+		{
+			name: "should wait for UpdateAuth move to complete",
+			automationStatus: &AutomationStatus{
+				GoalVersion: 5,
+				Processes: []ProcessStatus{
+					{
+						Name:                    "rs0_0",
+						LastGoalVersionAchieved: 5,
+						Plan:                    []string{"UpdateAuth"},
+					},
+				},
+			},
+			relevantProcesses: []string{"rs0_0"},
+			expectedReady:     false,
+			expectedMessage:   "authentication transitions in progress for 1 processes",
+		},
+		{
+			name: "should be ready when authentication transitions are complete",
+			automationStatus: &AutomationStatus{
+				GoalVersion: 5,
+				Processes: []ProcessStatus{
+					{
+						Name:                    "rs0_0",
+						LastGoalVersionAchieved: 5,
+						Plan:                    []string{}, // Empty plan means all moves completed
+					},
+				},
+			},
+			relevantProcesses: []string{"rs0_0"},
+			expectedReady:     true,
+			expectedMessage:   "processes that reached goal state: [rs0_0]",
+		},
+		{
+			name: "should wait for multiple processes with auth transitions",
+			automationStatus: &AutomationStatus{
+				GoalVersion: 7,
+				Processes: []ProcessStatus{
+					{
+						Name:                    "rs0_0",
+						LastGoalVersionAchieved: 7,
+						Plan:                    []string{}, // This process completed
+					},
+					{
+						Name:                    "rs0_1",
+						LastGoalVersionAchieved: 7,
+						Plan:                    []string{"WaitAuthUpdate"}, // Auth-related move in progress
+					},
+				},
+			},
+			relevantProcesses: []string{"rs0_0", "rs0_1"},
+			expectedReady:     false,
+			expectedMessage:   "authentication transitions in progress for 1 processes",
+		},
+		{
+			name: "should ignore non-authentication moves in progress",
+			automationStatus: &AutomationStatus{
+				GoalVersion: 4,
+				Processes: []ProcessStatus{
+					{
+						Name:                    "rs0_0",
+						LastGoalVersionAchieved: 4,
+						Plan:                    []string{"SomeOtherMove"}, // Non-auth move
+					},
+				},
+			},
+			relevantProcesses: []string{"rs0_0"},
+			expectedReady:     true,
+			expectedMessage:   "processes that reached goal state: [rs0_0]",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ready, message := checkAutomationStatusIsGoal(
+				tt.automationStatus,
+				tt.relevantProcesses,
+				logger,
+			)
+
+			assert.Equal(t, tt.expectedReady, ready, "Ready state should match expected")
+			assert.Contains(t, message, tt.expectedMessage, "Message should contain expected text")
+
+			if tt.expectedReady {
+				t.Logf("✅ Process correctly marked as ready: %s", message)
+			} else {
+				t.Logf("⏳ Process correctly waiting for auth transition: %s", message)
+			}
+		})
+	}
+}
+
+func TestIsAuthenticationTransitionMove(t *testing.T) {
+	authMoves := []string{
+		"UpdateAuth",
+		"WaitAuthUpdate",
+	}
+
+	nonAuthMoves := []string{
+		"SomeOtherMove",
+		"CreateIndex",
+		"DropCollection",
+		"BackupDatabase",
+	}
+
+	for _, move := range authMoves {
+		t.Run("auth_move_"+move, func(t *testing.T) {
+			assert.True(t, isAuthenticationTransitionMove(move),
+				"Move %s should be recognized as authentication transition", move)
+		})
+	}
+
+	for _, move := range nonAuthMoves {
+		t.Run("non_auth_move_"+move, func(t *testing.T) {
+			assert.False(t, isAuthenticationTransitionMove(move),
+				"Move %s should not be recognized as authentication transition", move)
+		})
+	}
+}

scripts/dev/contexts/evg-private-context

@@ -38,9 +38,8 @@ STAGING_REPO_URL="268558157000.dkr.ecr.us-east-1.amazonaws.com/staging"
 COMMIT_SHA_SHORT=$(git rev-parse --short=8 HEAD)
 
 if [ "${is_patch:-false}" = "true" ]; then
-  echo "is_patch is set, setting BASE_REPO_URL=${STAGING_REPO_URL}"
-  export BASE_REPO_URL="${STAGING_REPO_URL}"
-  export OVERRIDE_VERSION_ID="${COMMIT_SHA_SHORT}"
+  echo "is_patch is set, setting BASE_REPO_URL=${DEV_REPO_URL}"
+  export BASE_REPO_URL="${DEV_REPO_URL}"
 else
   echo "is_patch is not set, setting BASE_REPO_URL=${STAGING_REPO_URL}, OVERRIDE_VERSION_ID=${COMMIT_SHA_SHORT}"
   export BASE_REPO_URL="${STAGING_REPO_URL}"

scripts/release/atomic_pipeline.py

@@ -119,17 +119,20 @@ def build_mco_tests_image(build_configuration: ImageBuildConfiguration):
     )
 
 
-def build_operator_image(build_configuration: ImageBuildConfiguration):
+def build_operator_image(build_configuration: ImageBuildConfiguration, with_race_detection: bool = False):
     """Calculates arguments required to build the operator image, and starts the build process."""
     # In evergreen, we can pass test_suffix env to publish the operator to a quay
     # repository with a given suffix.
     test_suffix = os.getenv("test_suffix", "")
     log_automation_config_diff = os.getenv("LOG_AUTOMATION_CONFIG_DIFF", "false")
 
+    build_configuration.version = f"{build_configuration.version}{'-race' if with_race_detection else ''}"
+
     args = {
         "version": build_configuration.version,
         "log_automation_config_diff": log_automation_config_diff,
         "test_suffix": test_suffix,
+        "use_race": "true" if with_race_detection else "false",
     }
 
     logger.info(f"Building Operator args: {args}")