Determinized the shard assignments (cadence-workflow#7184)

What changed?
We now sort the executors before we process the shard assignement.

Why?
We don't want to rely on map ordering since it's random in Go. This made tests flaky, and also would make debugging harder.

How did you test it?
Unit tests

Potential risks

Release notes

Documentation Changes

Author: jakobht

service/sharddistributor/leader/process/processor.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"math/rand"
+	"slices"
 	"sort"
 	"strconv"
 	"sync"
@@ -401,7 +402,14 @@ func assignShardsToEmptyExecutors(currentAssignments map[string][]string) bool {
 	executorsWithShards := make([]string, 0)
 	minShardsCurrentlyAssigned := 0
 
+	// Ensure the iteration is deterministic.
+	executors := make([]string, 0, len(currentAssignments))
 	for executorID := range currentAssignments {
+		executors = append(executors, executorID)
+	}
+	slices.Sort(executors)
+
+	for _, executorID := range executors {
 		if len(currentAssignments[executorID]) == 0 {
 			emptyExecutors = append(emptyExecutors, executorID)
 		} else {

service/sharddistributor/leader/process/processor_test.go

@@ -3,7 +3,6 @@ package process
 import (
 	"context"
 	"errors"
-	"slices"
 	"sync"
 	"testing"
 	"time"
@@ -399,7 +398,6 @@ func TestGetShards_Utility(t *testing.T) {
 }
 
 func TestAssignShardsToEmptyExecutors(t *testing.T) {
-	t.Skip("Skipping this test for now because it's flaky")
 	cases := []struct {
 		name                       string
 		inputAssignments           map[string][]string
@@ -468,18 +466,8 @@ func TestAssignShardsToEmptyExecutors(t *testing.T) {
 		t.Run(c.name, func(t *testing.T) {
 			actualDistributionChanged := assignShardsToEmptyExecutors(c.inputAssignments)
 
-			// Sort the assignments, so the test is stable
-			sortAssignments(c.expectedAssignments)
-			sortAssignments(c.inputAssignments)
-
 			assert.Equal(t, c.expectedAssignments, c.inputAssignments)
 			assert.Equal(t, c.expectedDistributonChanged, actualDistributionChanged)
 		})
 	}
 }
-
-func sortAssignments(assignments map[string][]string) {
-	for _, assignment := range assignments {
-		slices.Sort(assignment)
-	}
-}