CLOUDP-300239: Skip unit-tests variant with retry script in EVG (#4138)

# Summary

This patch skips retrying failed tasks in the `unit_tests` variant

## Proof of Work

Failing unit tests patch without retries:
https://spruce.mongodb.com/version/67c0a9e89493f200075f06aa

## Checklist
- [x] Have you linked a jira ticket and/or is the ticket in the title?
- [x] Have you checked whether your jira ticket required DOCSP changes?
- [x] Have you checked for release_note changes?

## Reminder (Please remove this when merging)
- Please try to Approve or Reject Changes the PR, keep PRs in review as
short as possible
- Our Short Guide for PRs:
[Link](REDACTED)
- Remember the following Communication Standards - use comment prefixes
for clarity:
  * **blocking**: Must be addressed before approval.
  * **follow-up**: Can be addressed in a later PR or ticket.
  * **q**: Clarifying question.
  * **nit**: Non-blocking suggestions.
  * **note**: Side-note, non-actionable. Example: Praise
  * --> no prefix is considered a question

Author: mircea-cosbuc

controllers/om/automation_status.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"maps"
 	"slices"
+	"sort"
 
 	"go.uber.org/zap"
 	"golang.org/x/xerrors"
@@ -100,6 +101,7 @@ func checkAutomationStatusIsGoal(as *AutomationStatus, relevantProcesses []strin
 		goalsNotAchievedMsgList = append(goalsNotAchievedMsgList, fmt.Sprintf("%s@%d", processName, goalAchieved))
 	}
 	goalsAchievedMsgList := slices.Collect(maps.Keys(goalsAchievedMap))
+	sort.Strings(goalsAchievedMsgList)
 
 	if len(goalsNotAchievedMap) > 0 {
 		return false, fmt.Sprintf("%d processes waiting to reach automation config goal state (version=%d): %s, %d processes reached goal state: %s",

controllers/om/automation_status_test.go

@@ -40,7 +40,7 @@ func TestCheckAutomationStatusIsGoal(t *testing.T) {
 			},
 			expectedResult: true,
 			// We can not check for the full message as the ordering of the processes won't be deterministic (stored in a map)
-			expectedMsg: "processes that reached goal state:",
+			expectedMsg: "processes that reached goal state: [a b]",
 		}, {
 			name: "one not in goal state",
 			args: args{

scripts/evergreen/retry-evergreen.sh

@@ -19,29 +19,26 @@ if [[ "${EVERGREEN_RETRY:-"true"}" != "true" ]]; then
   exit 0
 fi
 
-if [ $# -eq 0 ]
-then
-    echo "Details URL not passed in, exiting..."
-    exit 1
+if [ $# -eq 0 ]; then
+  echo "Details URL not passed in, exiting..."
+  exit 1
 else
   VERSION=$1
 fi
-if [ -z "${EVERGREEN_USER}" ]
-then
-    echo "$$EVERGREEN_USER not set"
-    exit 1
+if [ -z "${EVERGREEN_USER}" ]; then
+  echo "$$EVERGREEN_USER not set"
+  exit 1
 fi
-if [ -z "${EVERGREEN_API_KEY}" ]
-then
-    echo "$$EVERGREEN_API_KEY not set"
-    exit 1
+if [ -z "${EVERGREEN_API_KEY}" ]; then
+  echo "$$EVERGREEN_API_KEY not set"
+  exit 1
 fi
 
 EVERGREEN_API="https://evergreen.mongodb.com/api"
 MAX_RETRIES="${EVERGREEN_MAX_RETRIES:-3}"
 
 # shellcheck disable=SC2207
-BUILD_IDS=($(curl -s -H "Api-User: ${EVERGREEN_USER}" -H "Api-Key: ${EVERGREEN_API_KEY}" ${EVERGREEN_API}/rest/v2/versions/"${VERSION}" | jq -r '.build_variants_status[] | .build_id'))
+BUILD_IDS=($(curl -s -H "Api-User: ${EVERGREEN_USER}" -H "Api-Key: ${EVERGREEN_API_KEY}" ${EVERGREEN_API}/rest/v2/versions/"${VERSION}" | jq -r '.build_variants_status[] | select(.build_variant!="unit_tests") | .build_id'))
 
 for BUILD_ID in "${BUILD_IDS[@]}"; do
   echo "Finding failed tasks in BUILD ID: ${BUILD_ID}"