Merge #148443 #151317

148443: make and publish: tag staging repo early in process r=celiala a=rail

Currently, the “Make and Publish Build” process creates a staging tag and pushes it to the staging repository only after all builds and tests have been completed. This approach delays the initiation of the “Build and Sign Cockroach Release” build. In optimal scenarios, “Make and Publish Build” can take over 30 minutes to finish, and any flaky tests can reset this timer.

In urgent situations where fixes need to be deployed quickly, we want to initiate the release process without waiting for “Make and Publish Build” to complete. It’s important to note that the built binaries are merely staged and not published at this stage.

To enhance efficiency, we can shift the tagging process to the beginning of the “Make and Publish Build.” This adjustment will allow us to commence building release binaries without being hindered by the blocking build jobs. Furthermore, we should publish the metadata.json file immediately after tagging to avoid delaying the Pick SHA process.


Release note: none
Fixes: RE-958

151317: kvserver: deflake BenchmarkBumpSideTransportClosed r=iskettaneh a=pav-kv

Fixes #151295

Co-authored-by: Rail Aliiev <[email protected]>
Co-authored-by: Pavel Kalinnikov <[email protected]>

Author: 3 people

build/teamcity/internal/release/process/make-and-publish-build-start.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+# Copyright 2022 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+
+
+set -euo pipefail
+
+# This script can be skipped for dry run and customized builds.
+is_customized_build="$(echo "$TC_BUILD_BRANCH" | grep -Eo "^custombuild-" || echo "")"
+if [[ -n "${DRY_RUN:-}" ]] || [[ -n "${is_customized_build}" ]]; then
+  echo "Skipping for dry run or customized build."
+  exit 0
+fi
+
+dir="$(dirname $(dirname $(dirname $(dirname $(dirname "${0}")))))"
+source "$dir/release/teamcity-support.sh"
+
+github_ssh_key="${GITHUB_COCKROACH_TEAMCITY_PRIVATE_SSH_KEY:?GITHUB_COCKROACH_TEAMCITY_PRIVATE_SSH_KEY must be specified}"
+metadata_gcs_bucket="cockroach-release-qualification-prod"
+metadata_google_credentials="$GCS_CREDENTIALS_PROD"
+build_name="$(git describe --tags --dirty --match=v[0-9]* 2> /dev/null || git rev-parse --short HEAD;)"
+
+configure_git_ssh_key
+git tag "${build_name}"
+git_wrapped push ssh://[email protected]/cockroachlabs/release-staging.git "${build_name}"
+
+# Publish build metadata to a stable location.
+timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+metadata_file="artifacts/metadata.json"
+mkdir -p artifacts
+cat > "$metadata_file" << EOF
+{
+  "sha": "$BUILD_VCS_NUMBER",
+  "timestamp": "$timestamp",
+  "tag": "$build_name"
+}
+EOF
+# Run jq to pretty print and validate JSON
+jq . "$metadata_file"
+google_credentials=$metadata_google_credentials log_into_gcloud
+gsutil cp "$metadata_file" "gs://$metadata_gcs_bucket/builds/$BUILD_VCS_NUMBER.json"
+echo "Published to https://storage.googleapis.com/$metadata_gcs_bucket/builds/$BUILD_VCS_NUMBER.json"

build/teamcity/internal/release/process/make-and-publish-build-tagging.sh renamed to build/teamcity/internal/release/process/make-and-publish-build.sh

@@ -72,15 +72,6 @@ tc_end_block "Variable Setup"
 configure_git_ssh_key
 
 if [[ -z "${is_customized_build}" ]] ; then
-  tc_start_block "Tag the release"
-  git tag "${build_name}"
-  tc_end_block "Tag the release"
-
-  tc_start_block "Push release tag to github.com/cockroachlabs/release-staging"
-  git_wrapped push ssh://[email protected]/cockroachlabs/release-staging.git "${build_name}"
-  tc_end_block "Push release tag to github.com/cockroachlabs/release-staging"
-
-
   tc_start_block "Tag docker image as latest-build"
   # Only tag the image as "latest-vX.Y-build" if the tag is on a release branch
   # (or master for the alphas for the next major release).
@@ -120,22 +111,3 @@ if [[ -n "${is_customized_build}" ]] ; then
   git_wrapped push ssh://[email protected]/cockroachdb/cockroach.git --delete "${TC_BUILD_BRANCH}"
   tc_end_block "Delete custombuild tag"
 fi
-
-# Publish build metadata to a stable location.
-tc_start_block "Metadata"
-timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
-metadata_file="artifacts/metadata.json"
-mkdir -p artifacts
-cat > "$metadata_file" << EOF
-{
-  "sha": "$BUILD_VCS_NUMBER",
-  "timestamp": "$timestamp",
-  "tag": "$build_name"
-}
-EOF
-# Run jq to pretty print and validate JSON
-jq . "$metadata_file"
-google_credentials=$metadata_google_credentials log_into_gcloud
-gsutil cp "$metadata_file" "gs://$metadata_gcs_bucket/builds/$BUILD_VCS_NUMBER.json"
-echo "Published to https://storage.googleapis.com/$metadata_gcs_bucket/builds/$BUILD_VCS_NUMBER.json"
-tc_end_block "Metadata"

pkg/kv/kvserver/replica_closedts_test.go

@@ -694,6 +694,7 @@ func TestRejectedLeaseDoesntDictateClosedTimestamp(t *testing.T) {
 // we eliminate some memory copying and pass the node liveness record in to the
 // function so that we only have to grab it once instead of on each call to
 // BumpSideTransportClosed, we should be able to reach our target latency.
+// TODO(pav-kv): in 2025, this shows about 180-200ns.
 func BenchmarkBumpSideTransportClosed(b *testing.B) {
 	defer leaktest.AfterTest(b)()
 	defer log.Scope(b).Close(b)
@@ -721,6 +722,22 @@ func BenchmarkBumpSideTransportClosed(b *testing.B) {
 	now := s.Clock().NowAsClockTimestamp()
 	targets := map[ctpb.RangeClosedTimestampPolicy]hlc.Timestamp{}
 
+	// Wait for the scratch range's lease upgrade from expiration-based lease. The
+	// closed timestamp bumping requires for there to be no in-flight proposals,
+	// so a racing lease upgrade can cause a test failure like in #151295.
+	//
+	// TODO(pav-kv): share this code with TestCluster.MaybeWaitForLeaseUpgrade().
+	testutils.SucceedsSoon(b, func() error {
+		lease, _, err := s.GetRangeLease(ctx, key, roachpb.QueryLocalNodeOnly)
+		require.NoError(b, err)
+		typ := lease.Current().Type()
+		if typ == roachpb.LeaseExpiration {
+			return errors.Errorf("still on expiration-based lease")
+		}
+		b.Logf("lease is now of type: %s", typ)
+		return nil
+	})
+
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		// Advance time and the closed timestamp target.
@@ -730,7 +747,7 @@ func BenchmarkBumpSideTransportClosed(b *testing.B) {
 		// Perform the call.
 		res := r.BumpSideTransportClosed(ctx, now, targets)
 		if !res.OK {
-			b.Fatal("BumpSideTransportClosed unexpectedly failed")
+			b.Fatalf("BumpSideTransportClosed unexpectedly failed: %+v", res)
 		}
 	}
 }