Merge #145388

145388: roachtest: add IBM Cloud r=rickystewart,srosenberg a=golgeek

This patch bootstraps the ability to run roachtests on IBM Cloud.

This builds on top of #145387.
    
Epic: none
Release note: none

Co-authored-by: Ludovic Leroux <[email protected]>

Author: craig[bot]

build/teamcity/cockroach/nightlies/roachtest_compile_bits.sh

@@ -17,7 +17,7 @@ if [ "$#" -eq 0 ]; then
   echo "Builds all bits needed for roachtests and stages them in bin/ and lib/."
   echo ""
   echo "Usage: $0 [--with-code-coverage] arch [arch...]"
-  echo "  where arch is one of: amd64, arm64, amd64-fips"
+  echo "  where arch is one of: amd64, arm64, amd64-fips, s390x"
   exit 1
 fi
 

build/teamcity/cockroach/nightlies/roachtest_compile_component.sh

@@ -15,7 +15,7 @@ if [ "$#" -eq 0 ]; then
   echo ""
   echo "Usage: $(basename $0) [--with-coverage] <os/arch/component>"
   echo "  where os is one of: linux"
-  echo "        arch is one of: amd64, arm64, amd64-fips"
+  echo "        arch is one of: amd64, arm64, amd64-fips, s390x"
   echo "        component is one of: cockroach, cockroach-ea, workload, libgeos, roachtest"
   echo "  --with-coverage enables go code coverage instrumentation (only applies to cockroach binaries)"
   exit 1

build/teamcity/cockroach/nightlies/roachtest_nightly_ibm.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+# Copyright 2025 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+
+
+set -exuo pipefail
+
+dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
+
+source "$dir/teamcity-support.sh"  # For $root
+source "$dir/teamcity-bazel-support.sh"  # For run_bazel
+
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e IBM_APIKEY -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e CLEAR_CLUSTER_CACHE -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS -e MVT_UPGRADE_PATH -e MVT_DEPLOYMENT_MODE -e EXTRA_ROACHTEST_ARGS" \
+			       run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh

build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh

@@ -20,11 +20,15 @@ if [[ ! -f ~/.ssh/id_rsa.pub ]]; then
 fi
 
 arch=amd64
-if [[ ${FIPS_ENABLED:-0} == 1 ]]; then
+if [[ ${CLOUD} == "ibm" ]]; then
+  arch=s390x
+elif [[ ${FIPS_ENABLED:-0} == 1 ]]; then
   arch=amd64-fips
 fi
 $root/build/teamcity/cockroach/nightlies/roachtest_compile_bits.sh $arch
-$root/build/teamcity/cockroach/nightlies/roachtest_compile_bits.sh arm64
+if [[ $arch != "s390x" ]]; then
+  $root/build/teamcity/cockroach/nightlies/roachtest_compile_bits.sh arm64
+fi
 
 artifacts=/artifacts
 source $root/build/teamcity/util/roachtest_util.sh

build/teamcity/util/roachtest_arch_util.sh

@@ -15,6 +15,9 @@ function arch_to_config() {
     amd64-fips)
       echo "crosslinuxfips"
       ;;
+    s390x)
+      echo "crosslinuxs390x"
+      ;;
     *)
       echo "Error: invalid arch '$1'" >&2
       exit 1

pkg/cmd/roachprod/cli/util.go

@@ -217,7 +217,7 @@ func ValidateAndConfigure(cmd *cobra.Command, args []string) {
 	if archOpt := cmd.Flags().Lookup("arch"); archOpt != nil && archOpt.Changed {
 		arch := vm.CPUArch(strings.ToLower(archOpt.Value.String()))
 
-		if arch != vm.ArchAMD64 && arch != vm.ArchARM64 && arch != vm.ArchFIPS {
+		if arch != vm.ArchAMD64 && arch != vm.ArchARM64 && arch != vm.ArchFIPS && arch != vm.ArchS390x {
 			printErrAndExit(fmt.Errorf("unsupported architecture %q", arch))
 		}
 		if string(arch) != archOpt.Value.String() {

pkg/cmd/roachtest/cluster.go

@@ -101,6 +101,8 @@ func validateBinaryFormat(path string, arch vm.CPUArch, checkEA bool) (string, e
 	} else if arch == vm.ArchAMD64 && !strings.Contains(fileFormat, "x86-64") && !strings.Contains(fileFormat, "x86_64") {
 		// Otherwise, we expect a binary that was built for amd64.
 		return "", errors.Newf("%s has incompatible architecture; want: %q, got: %q", abspath, arch, fileFormat)
+	} else if arch == vm.ArchS390x && !strings.Contains(fileFormat, "ibm s/390") {
+		return "", errors.Newf("%s has incompabile architecture; want: %q, got: %q", abspath, arch, fileFormat)
 	}
 	if arch == vm.ArchFIPS && strings.HasSuffix(abspath, "cockroach") {
 		// Check that the binary is patched to use OpenSSL FIPS.
@@ -278,6 +280,8 @@ func initBinariesAndLibraries() {
 		if string(defaultArch) != runtime.GOARCH {
 			fmt.Printf("WARN: local cluster's architecture (%q) differs from default (%q)\n", runtime.GOARCH, defaultArch)
 		}
+	} else if roachtestflags.Cloud == spec.IBM {
+		defaultArch = vm.ArchS390x
 	}
 	fmt.Printf("Locating and verifying binaries for os=%q, arch=%q\n", defaultOSName, defaultArch)
 
@@ -356,7 +360,7 @@ func initBinariesAndLibraries() {
 
 	// In v20.2 or higher, optionally expect certain library files to exist.
 	// Since they may not be found in older versions, do not hard error if they are not found.
-	for _, arch := range []vm.CPUArch{vm.ArchAMD64, vm.ArchARM64, vm.ArchFIPS} {
+	for _, arch := range []vm.CPUArch{vm.ArchAMD64, vm.ArchARM64, vm.ArchFIPS, vm.ArchS390x} {
 		if roachtestflags.ARM64Probability == 0 && defaultArch != vm.ArchARM64 && arch == vm.ArchARM64 {
 			// arm64 isn't used, skip finding libs for it.
 			continue
@@ -541,7 +545,7 @@ func makeClusterName(name string) string {
 	return vm.DNSSafeName(name)
 }
 
-// MachineTypeToCPUs returns a CPU count for GCE, AWS, and Azure machine types.
+// MachineTypeToCPUs returns a CPU count for GCE, AWS, Azure and IBM machine types.
 // -1 is returned for unknown machine types.
 func MachineTypeToCPUs(s string) int {
 	{
@@ -564,6 +568,23 @@ func MachineTypeToCPUs(s string) int {
 		}
 	}
 
+	{
+		// IBM machine types.
+		var v int
+		// bz2 is the balanced IBM Z machine type family.
+		if _, err := fmt.Sscanf(s, "bz2-%dx", &v); err == nil {
+			return v
+		}
+		// bz2 is the compute optimized IBM Z machine type family.
+		if _, err := fmt.Sscanf(s, "cz2-%dx", &v); err == nil {
+			return v
+		}
+		// bz2 is the memory optimized IBM Z machine type family.
+		if _, err := fmt.Sscanf(s, "mx2-%dx", &v); err == nil {
+			return v
+		}
+	}
+
 	typeAndSize := strings.Split(s, ".")
 
 	if len(typeAndSize) == 2 {
@@ -3229,6 +3250,12 @@ func archForTest(ctx context.Context, l *logger.Logger, testSpec registry.TestSp
 		return testSpec.Cluster.Arch
 	}
 
+	if roachtestflags.Cloud == spec.IBM {
+		// N.B. IBM only supports S390x on the "s390x" architecture.
+		l.PrintfCtx(ctx, "IBM Cloud: forcing arch=%q (only supported), %s", vm.ArchS390x, testSpec.Name)
+		return vm.ArchS390x
+	}
+
 	// CPU architecture is unspecified, choose one according to the
 	// probability distribution.
 	var arch vm.CPUArch

pkg/cmd/roachtest/cluster_test.go

@@ -643,6 +643,134 @@ func TestAzureMachineType(t *testing.T) {
 	require.Error(t, err2)
 }
 
+func TestIBMMachineType(t *testing.T) {
+	testCases := []struct {
+		name            string
+		cpus            int
+		mem             spec.MemPerCPU
+		arch            vm.CPUArch
+		expectedMachine string
+		expectedArch    vm.CPUArch
+		expectedError   string
+	}{}
+
+	// Helper function to generate the expected machine type string
+	ibmMachineType := func(series string, cpus int, ramRatio int) string {
+		return fmt.Sprintf("%s-%dx%d", series, cpus, cpus*ramRatio)
+	}
+
+	// IBM Z only supports s390x architecture
+	arch := vm.ArchS390x
+
+	// Add test cases for each memory configuration
+	addTestCases := func(mem spec.MemPerCPU) {
+		var series string
+		var ramRatio int
+
+		switch mem {
+		case spec.Auto, spec.Standard:
+			series = "bz2" // balanced
+			ramRatio = 4
+		case spec.High:
+			series = "mz2" // memory optimized
+			ramRatio = 8
+		case spec.Low:
+			series = "cz2" // compute optimized
+			ramRatio = 2
+		}
+
+		// IBM Z only supports 2, 4, 8, or 16 CPUs
+		for _, cpus := range []int{2, 4, 8, 16} {
+			testName := fmt.Sprintf("valid_%dcpu_%s", cpus, mem)
+			testCases = append(testCases, struct {
+				name            string
+				cpus            int
+				mem             spec.MemPerCPU
+				arch            vm.CPUArch
+				expectedMachine string
+				expectedArch    vm.CPUArch
+				expectedError   string
+			}{
+				name:            testName,
+				cpus:            cpus,
+				mem:             mem,
+				arch:            arch,
+				expectedMachine: ibmMachineType(series, cpus, ramRatio),
+				expectedArch:    arch,
+				expectedError:   "",
+			})
+		}
+
+		// Add test cases for unsupported CPU counts
+		invalidCPUs := []int{1, 6, 10, 32, 96, 128}
+		for _, cpus := range invalidCPUs {
+			testName := fmt.Sprintf("invalid_%dcpu_%s", cpus, mem)
+			testCases = append(testCases, struct {
+				name            string
+				cpus            int
+				mem             spec.MemPerCPU
+				arch            vm.CPUArch
+				expectedMachine string
+				expectedArch    vm.CPUArch
+				expectedError   string
+			}{
+				name:            testName,
+				cpus:            cpus,
+				mem:             mem,
+				arch:            arch,
+				expectedMachine: "",
+				expectedArch:    arch,
+				expectedError:   fmt.Sprintf("invalid number of cpus %d for IBM", cpus),
+			})
+		}
+	}
+
+	// Add test cases for each memory configuration
+	for _, mem := range []spec.MemPerCPU{spec.Auto, spec.Standard, spec.High, spec.Low} {
+		addTestCases(mem)
+	}
+
+	// Add test cases for unsupported architectures
+	for _, invalidArch := range []vm.CPUArch{vm.ArchAMD64, vm.ArchARM64, vm.ArchFIPS} {
+		testName := fmt.Sprintf("invalid_arch_%s", invalidArch)
+		testCases = append(testCases, struct {
+			name            string
+			cpus            int
+			mem             spec.MemPerCPU
+			arch            vm.CPUArch
+			expectedMachine string
+			expectedArch    vm.CPUArch
+			expectedError   string
+		}{
+			name:            testName,
+			cpus:            4,
+			mem:             spec.Auto,
+			arch:            invalidArch,
+			expectedMachine: "",
+			expectedArch:    invalidArch,
+			expectedError:   fmt.Sprintf("invalid architecture %q for IBM", invalidArch),
+		})
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			machineType, selectedArch, err := spec.SelectIBMMachineType(tc.cpus, tc.mem, tc.arch)
+
+			if tc.expectedError != "" {
+				// We expect a specific error
+				require.Error(t, err)
+				require.Equal(t, tc.expectedError, err.Error())
+				require.Equal(t, tc.expectedMachine, machineType)
+				require.Equal(t, tc.expectedArch, selectedArch)
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, tc.expectedMachine, machineType)
+				require.Equal(t, tc.expectedArch, selectedArch)
+			}
+		})
+	}
+}
+
 func TestMachineTypes(t *testing.T) {
 	datadriven.Walk(t, datapathutils.TestDataPath(t, "cluster_test"), func(t *testing.T, path string) {
 		datadriven.RunTest(t, path, func(t *testing.T, d *datadriven.TestData) string {

pkg/cmd/roachtest/registry/test_spec.go

@@ -293,7 +293,7 @@ type CloudSet struct {
 }
 
 // AllClouds contains all clouds.
-var AllClouds = Clouds(spec.Local, spec.GCE, spec.AWS, spec.Azure)
+var AllClouds = Clouds(spec.Local, spec.GCE, spec.AWS, spec.Azure, spec.IBM)
 
 // AllExceptLocal contains all clouds except Local.
 var AllExceptLocal = AllClouds.NoLocal()
@@ -304,6 +304,9 @@ var AllExceptAWS = AllClouds.NoAWS()
 // AllExceptAzure contains all clouds except Azure.
 var AllExceptAzure = AllClouds.NoAzure()
 
+// AllExceptIBM contains all clouds except IBM.
+var AllExceptIBM = AllClouds.NoIBM()
+
 // OnlyAWS contains only the AWS cloud.
 var OnlyAWS = Clouds(spec.AWS)
 
@@ -313,14 +316,17 @@ var OnlyGCE = Clouds(spec.GCE)
 // OnlyAzure contains only the Azure cloud.
 var OnlyAzure = Clouds(spec.Azure)
 
+// OnlyIBM contains only the IBM cloud.
+var OnlyIBM = Clouds(spec.IBM)
+
 // OnlyLocal contains only the Local cloud.
 var OnlyLocal = Clouds(spec.Local)
 
 // CloudsWithServiceRegistration contains clouds that support service registration.
 var CloudsWithServiceRegistration = Clouds(spec.Local, spec.GCE)
 
 // Clouds creates a CloudSet for the given clouds. Cloud names must be one of:
-// spec.Local, spec.GCE, spec.AWS, spec.Azure.
+// spec.Local, spec.GCE, spec.AWS, spec.Azure, spec.IBM.
 func Clouds(clouds ...spec.Cloud) CloudSet {
 	return CloudSet{m: addToSet(nil, clouds...)}
 }
@@ -340,6 +346,11 @@ func (cs CloudSet) NoAzure() CloudSet {
 	return CloudSet{m: removeFromSet(cs.m, spec.Azure)}
 }
 
+// NoIBM removes the IBM cloud and returns the new set.
+func (cs CloudSet) NoIBM() CloudSet {
+	return CloudSet{m: removeFromSet(cs.m, spec.IBM)}
+}
+
 // Remove removes all clouds passed in and returns the new set.
 func (cs CloudSet) Remove(clouds CloudSet) CloudSet {
 	copyCs := CloudSet{m: cs.m}

pkg/cmd/roachtest/registry/test_spec_test.go

@@ -16,11 +16,12 @@ func TestCloudSet(t *testing.T) {
 	expect := func(c CloudSet, exp string) {
 		require.Equal(t, exp, c.String())
 	}
-	expect(AllClouds, "local,gce,aws,azure")
-	expect(AllExceptAWS, "local,gce,azure")
-	expect(AllExceptLocal, "gce,aws,azure")
-	expect(AllExceptLocal.NoAWS(), "gce,azure")
-	expect(AllClouds.NoAWS().NoAzure(), "local,gce")
+	expect(AllClouds, "local,gce,aws,azure,ibm")
+	expect(AllExceptAWS, "local,gce,azure,ibm")
+	expect(AllExceptLocal, "gce,aws,azure,ibm")
+	expect(AllExceptLocal.NoAWS(), "gce,azure,ibm")
+	expect(AllClouds.NoAWS().NoAzure(), "local,gce,ibm")
+	expect(AllClouds.NoIBM(), "local,gce,aws,azure")
 
 	require.True(t, AllExceptAWS.Contains(spec.GCE))
 	require.True(t, AllExceptAWS.Contains(spec.Local))