Merge #143737 #143813

143737: build/teamcity: add commit, os labels for nightlies r=rickystewart a=sambhav-jain-16

This change aims to add commit label, goos label and also renames cpu_arch label to goarch label to be more consistent for comparisons.

Epic: none

Release note: None

143813: roachtest: fix perf artifacts not processing r=DarrylWong,herkolategan a=sambhav-jain-16

This commit fixes an issue in the test runner that prevented performance
artifacts from being properly processed.
When perf artifcats directory for the benchmark is not defined, we
default to finding files in every node, however the current
implementation returns when the files are not found in the first node
rather than moving forward.
Also, this change makes sure we write to only those directory from where
the perf artifacts are fetched.

Epic: none

Release note: None

Co-authored-by: Sambhav Jain <[email protected]>

Author: craig[bot]

build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh

@@ -100,6 +100,6 @@ build/teamcity-roachtest-invoke.sh \
   --selective-tests="${selective_tests:-false}" \
   --side-eye-token="${SIDE_EYE_API_TOKEN}" \
   --export-openmetrics="${EXPORT_OPENMETRICS:-false}" \
-  --openmetrics-labels="branch=$(tc_build_branch), cpu-arch=${arch}, suite=nightly" \
+  --openmetrics-labels="branch=$(tc_build_branch), goarch=${arch}, goos=linux, commit=${COMMIT_SHA}, suite=nightly" \
   ${EXTRA_ROACHTEST_ARGS:+$EXTRA_ROACHTEST_ARGS} \
   "${TESTS}"

build/teamcity/cockroach/nightlies/roachtest_weekly_impl.sh

@@ -49,5 +49,5 @@ timeout -s INT $((7800*60)) build/teamcity-roachtest-invoke.sh \
   --slack-token="${SLACK_TOKEN}" \
   --side-eye-token="${SIDE_EYE_API_TOKEN}" \
   --export-openmetrics="${EXPORT_OPENMETRICS:-false}" \
-  --openmetrics-labels="branch=$(tc_build_branch), cpu-arch=${arch}, suite=weekly" \
+  --openmetrics-labels="branch=$(tc_build_branch), goarch=${arch}, goos=linux, commit=${COMMIT_SHA}, suite=weekly" \
   ${TESTS:-}

build/teamcity/util/roachtest_util.sh

@@ -35,6 +35,8 @@ if [[ "${EXPORT_OPENMETRICS}" == "true" ]]; then
   stats_file_name="stats.om"
 fi
 
+COMMIT_SHA=$(git rev-parse --short HEAD)
+
 # Set up a function we'll invoke at the end.
 function upload_stats {
   if tc_release_branch; then

pkg/cmd/roachtest/test_runner.go

@@ -181,9 +181,10 @@ type perfMetricsCollector struct {
 	// elapsed is the avg elapsed time of the run
 	elapsed int64
 	// count is the count of perf files
-	count int64
-	t     *testImpl
-	ctx   context.Context
+	count     int64
+	t         *testImpl
+	ctx       context.Context
+	perfNodes []int
 }
 
 // newTestRunner constructs a testRunner.
@@ -2024,26 +2025,27 @@ func (r *testRunner) postProcessPerfMetrics(
 	}
 
 	// Collect and aggregate metrics from all relevant nodes
-	if err := metrics.collectFromNodes(c, dstDirFn); err != nil {
+	if err := metrics.collectFromNodes(c, dstDirFn, t.L()); err != nil {
 		t.L().PrintfCtx(ctx, "failed to collect metrics: %v", err)
 		return
 	}
 
 	// Process and write aggregated metrics
-	if err := metrics.processAndWrite(c, dstDirFn); err != nil {
+	if err := metrics.processAndWrite(dstDirFn); err != nil {
 		t.L().PrintfCtx(ctx, "failed to process and write metrics: %v", err)
 	}
 }
 
 func (m *perfMetricsCollector) collectFromNodes(
-	c *clusterImpl, dstDirFn func(nodeIdx int) string,
+	c *clusterImpl, dstDirFn func(nodeIdx int) string, log *logger.Logger,
 ) error {
 	for _, node := range getPerfArtifactsNode(c) {
 		files, err := m.findMetricsFiles(dstDirFn(node))
 		if err != nil {
-			return errors.Wrapf(err, "finding metrics files")
+			log.Printf("failed to find metrics files for node %d will continue: %s", node, err)
+			continue
 		}
-
+		m.perfNodes = append(m.perfNodes, node)
 		if err := m.processFiles(files); err != nil {
 			return errors.Wrapf(err, "error while processing files")
 		}
@@ -2085,9 +2087,7 @@ func (m *perfMetricsCollector) processFiles(files []string) error {
 	return nil
 }
 
-func (m *perfMetricsCollector) processAndWrite(
-	c *clusterImpl, dstDirFn func(nodeIdx int) string,
-) error {
+func (m *perfMetricsCollector) processAndWrite(dstDirFn func(nodeIdx int) string) error {
 	if m.count == 0 {
 		return errors.New("no metrics files found")
 	}
@@ -2109,8 +2109,8 @@ func (m *perfMetricsCollector) processAndWrite(
 		return errors.Wrapf(err, "converting metrics to bytes")
 	}
 
-	// Write the file and take the first node of the cluster
-	outputPath := filepath.Join(dstDirFn(getPerfArtifactsNode(c)[0]), "aggregated_stats.om")
+	// Write the file to the first directory of any node where perf artifacts are present
+	outputPath := filepath.Join(dstDirFn(m.perfNodes[0]), "aggregated_stats.om")
 	return os.WriteFile(outputPath, finalMetricsBuffer.Bytes(), 0644)
 }