roachtest: provision 250 MB/s for 8TB restore test

The `restore/tpce/8TB/aws/nodes=10/cpus=8` test maxes out the default 125 MB/s
EBS throughput. This commit provisions throughput to be 250 MB/s so that the
test doesn't work at the edge of overload.

Epic: none
Release note: none

Author: pav-kv

pkg/cmd/roachtest/spec/cluster_spec.go

@@ -93,6 +93,10 @@ type ClusterSpec struct {
 	// generality. Make it easier to just inject cloud-specific arguments.
 	GCEMinCPUPlatform string
 	GCEVolumeType     string
+	// AWS-specific arguments.
+	//
+	// AWSVolumeThroughput is the min provisioned EBS volume throughput.
+	AWSVolumeThroughput int
 }
 
 // MakeClusterSpec makes a ClusterSpec.
@@ -140,11 +144,16 @@ func awsMachineSupportsSSD(machineType string) bool {
 	return false
 }
 
-func getAWSOpts(machineType string, zones []string, volumeSize int, localSSD bool) vm.ProviderOpts {
+func getAWSOpts(
+	machineType string, zones []string, volumeSize, ebsThroughput int, localSSD bool,
+) vm.ProviderOpts {
 	opts := aws.DefaultProviderOpts()
 	if volumeSize != 0 {
 		opts.DefaultEBSVolume.Disk.VolumeSize = volumeSize
 	}
+	if ebsThroughput != 0 {
+		opts.DefaultEBSVolume.Disk.Throughput = ebsThroughput
+	}
 	if localSSD {
 		opts.SSDMachineType = machineType
 	} else {
@@ -310,7 +319,8 @@ func (s *ClusterSpec) RoachprodOpts(
 	var providerOpts vm.ProviderOpts
 	switch s.Cloud {
 	case AWS:
-		providerOpts = getAWSOpts(machineType, zones, s.VolumeSize, createVMOpts.SSDOpts.UseLocalSSD)
+		providerOpts = getAWSOpts(machineType, zones, s.VolumeSize, s.AWSVolumeThroughput,
+			createVMOpts.SSDOpts.UseLocalSSD)
 	case GCE:
 		providerOpts = getGCEOpts(machineType, zones, s.VolumeSize, ssdCount,
 			createVMOpts.SSDOpts.UseLocalSSD, s.RAID0, s.TerminateOnMigration,

pkg/cmd/roachtest/tests/restore.go

@@ -323,7 +323,10 @@ func registerRestore(r registry.Registry) {
 		},
 		{
 			// The nightly 8TB Restore test.
-			hardware: makeHardwareSpecs(hardwareSpecs{nodes: 10, volumeSize: 2000}),
+			// NB: bump disk throughput because this load saturates the default 125
+			// MB/s. See https://github.com/cockroachdb/cockroach/issues/107609.
+			hardware: makeHardwareSpecs(hardwareSpecs{nodes: 10, volumeSize: 2000,
+				ebsThroughput: 250 /* MB/s */}),
 			backup: makeRestoringBackupSpecs(backupSpecs{
 				version:  "v22.2.1",
 				workload: tpceRestore{customers: 500000}}),
@@ -466,6 +469,9 @@ type hardwareSpecs struct {
 	// volumeSize indicates the size of per node block storage (pd-ssd for gcs,
 	// ebs for aws). If zero, local ssd's are used.
 	volumeSize int
+	// ebsThroughput is the min provisioned throughput of the EBS volume, in MB/s.
+	// TODO(pavelkalinnikov): support provisioning throughput not only on EBS.
+	ebsThroughput int
 
 	// mem is the memory per cpu.
 	mem spec.MemPerCPU
@@ -494,6 +500,10 @@ func (hw hardwareSpecs) makeClusterSpecs(r registry.Registry, backupCloud string
 	}
 	s := r.MakeClusterSpec(hw.nodes+addWorkloadNode, clusterOpts...)
 
+	if hw.ebsThroughput != 0 {
+		s.AWSVolumeThroughput = hw.ebsThroughput
+	}
+
 	if backupCloud == spec.AWS && s.Cloud == spec.AWS && s.VolumeSize != 0 {
 		// Work around an issue that RAID0s local NVMe and GP3 storage together:
 		// https://github.com/cockroachdb/cockroach/issues/98783.
@@ -554,6 +564,9 @@ func makeHardwareSpecs(override hardwareSpecs) hardwareSpecs {
 	if override.volumeSize != 0 {
 		specs.volumeSize = override.volumeSize
 	}
+	if override.ebsThroughput != 0 {
+		specs.ebsThroughput = override.ebsThroughput
+	}
 	specs.zones = override.zones
 	specs.workloadNode = override.workloadNode
 	return specs

pkg/roachprod/vm/gce/gcloud.go

@@ -1009,6 +1009,9 @@ func (p *Provider) Create(
 			fmt.Sprintf("size=%dGB", providerOpts.PDVolumeSize),
 			"auto-delete=yes",
 		}
+		// TODO(pavelkalinnikov): support disk types with "provisioned-throughput"
+		// option, such as Hyperdisk Throughput:
+		// https://cloud.google.com/compute/docs/disks/add-hyperdisk#hyperdisk-throughput.
 		args = append(args, "--create-disk", strings.Join(pdProps, ","))
 		// Enable DISCARD commands for persistent disks, as is advised in:
 		// https://cloud.google.com/compute/docs/disks/optimizing-pd-performance#formatting_parameters.