roachtest: add build branch to roachprod's labels

Up to v24.1, roachtests Prometheus targets were discovered in GCP via
the gce_sd_configs mechanism. Starting in v24.2, this mechanism has been
replaced by an internal helper service, and the gce_sd_configs discovery
was later removed from our Prometheus configuration.

This configuration update has led to missing scraping targets for all
versions prior to v24.2. And the discovery mechanism will need to be
reintroduced until all supported releases are EOL.

This patch introduces a new label `roachtest-branch` pushed by roachtest
during roachprod cluster creation containing the version that is being
tested. This will allow filtering in the Prometheus configuration in
order to reenable target discovery only for versions prior to v24.2.

Epic: none
Release note: None

Author: golgeek

pkg/cmd/roachtest/spec/cluster_spec.go

@@ -7,6 +7,7 @@ package spec
 
 import (
 	"fmt"
+	"os"
 	"reflect"
 	"strings"
 	"time"
@@ -31,6 +32,9 @@ const (
 
 	// Zfs file system.
 	Zfs fileSystemType = 1
+
+	// Extra labels added by roachtest
+	RoachtestBranch = "roachtest-branch"
 )
 
 type MemPerCPU int
@@ -385,7 +389,14 @@ func (s *ClusterSpec) RoachprodOpts(
 
 	createVMOpts := vm.DefaultCreateOpts()
 	// N.B. We set "usage=roachtest" as the default, custom label for billing tracking.
-	createVMOpts.CustomLabels = map[string]string{"usage": "roachtest"}
+	createVMOpts.CustomLabels = map[string]string{vm.TagUsage: "roachtest"}
+
+	branch := os.Getenv("TC_BUILD_BRANCH")
+	if branch != "" {
+		// If the branch is set, we add it as a custom label.
+		createVMOpts.CustomLabels[RoachtestBranch] = vm.SanitizeLabel(branch)
+	}
+
 	createVMOpts.ClusterName = "" // Will be set later.
 	if s.Lifetime != 0 {
 		createVMOpts.Lifetime = s.Lifetime