Merge #155872 #157144 #157159

155872: changefeedccl: fix timestamp for desc fetches during planning r=andyyang890,aerfrei a=log-head

When a changefeed is planned and executed, there are several places where target table descriptors are fetched. With a db-level changefeed, the set of tables can change during changefeed startup. Now, the timestamp is set to the schema timestamp for retrieving table descriptors.

Adding a schema_ts to the protobuf for ChangeAggregatorSpec and ChangeFrontierSpec.

Fixes: #154549
Epic: CRDB-1421

Release note: None

157144: sql: add `ALTER TABLE ... SET/ADD IDENTITY` to the declarative schema… r=shghasemi a=shghasemi

… changer

This change adds support for `ALTER TABLE ... SET/ADD IDENTITY` to the declarative schema changer. This schema change operation runs using the legacy schema changer in versions older than 26.1.

Epic: CRDB-31283

Fixes #142918

Release note (sql change): `ALTER TABLE ... SET/ADD GENERATED AS IDENTITY` is supported by the declerative schema changer in 26.1 or later.

157159: build: add automated PGO profile update script r=rickystewart a=rail

This adds a unified TeamCity script that automates the entire PGO profile
update workflow into a single job.

The script orchestrates the following steps:
1. Runs baseline tpcc-nowait benchmarks with the current PGO profile
2. Generates a new PGO profile and uploads it to GCS
3. Updates the WORKSPACE file with the new profile URL and SHA256
4. Rebuilds with the new profile and runs benchmarks again
5. Compares before/after performance using benchstat
6. Creates a pull request with the changes and benchmark comparison

Fixes: DEVINF-1591
Fixes: DEVINF-1594
Release note: none

Co-authored-by: Matthew Lougheed <[email protected]>
Co-authored-by: Shadi Ghasemitaheri <[email protected]>
Co-authored-by: Rail Aliiev <[email protected]>

Author: 4 people

build/teamcity/cockroach/nightlies/roachtest_gce_validate_profile.sh

@@ -0,0 +1,221 @@
+#!/usr/bin/env bash
+
+# Copyright 2025 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+
+set -exuo pipefail
+
+dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
+
+source "$dir/teamcity-support.sh"  # For $root
+source "$dir/teamcity-bazel-support.sh"  # For run_bazel
+
+cleanup() {
+    rm -f ~/.config/gcloud/application_default_credentials.json
+}
+trap cleanup EXIT
+
+
+# Install `gh` CLI tool if not already available
+tc_start_block "Install gh CLI"
+if ! command -v gh &> /dev/null; then
+    echo "Installing gh CLI tool..."
+    wget -O /tmp/gh.tar.gz https://github.com/cli/cli/releases/download/v2.13.0/gh_2.13.0_linux_amd64.tar.gz
+    echo "9e833e02428cd49e0af73bc7dc4cafa329fe3ecba1bfe92f0859bf5b11916401  /tmp/gh.tar.gz" | sha256sum -c -
+    tar --strip-components 1 -xf /tmp/gh.tar.gz -C /tmp
+    export PATH=/tmp/bin:$PATH
+    echo "gh CLI installed successfully"
+fi
+tc_end_block "Install gh CLI"
+
+tc_start_block "Configuration"
+export TESTS="${TESTS:-tpcc-nowait/literal/w=1000/nodes=5/cpu=16}"
+export COUNT="${COUNT:-20}"
+export CLOUD="${CLOUD:-gce}"
+# Do not use spot instances for PGO profile generation to ensure consistent
+# performance and random VM preemptions.
+export USE_SPOT="${USE_SPOT:-never}"
+
+docker_env_args=(
+  SELECT_PROBABILITY=1.0
+  ARM_PROBABILITY=0.0
+  COCKROACH_EA_PROBABILITY=0.0
+  FIPS_PROBABILITY=0.0
+  LITERAL_ARTIFACTS_DIR=$root/artifacts
+  BUILD_VCS_NUMBER
+  CLOUD
+  COCKROACH_DEV_LICENSE
+  COCKROACH_RANDOM_SEED
+  COUNT
+  EXPORT_OPENMETRICS
+  GITHUB_API_TOKEN
+  GITHUB_ORG
+  GITHUB_REPO
+  GOOGLE_CREDENTIALS_ASSUME_ROLE
+  GOOGLE_EPHEMERAL_CREDENTIALS
+  GOOGLE_KMS_KEY_A
+  GOOGLE_KMS_KEY_B
+  GOOGLE_SERVICE_ACCOUNT
+  GRAFANA_SERVICE_ACCOUNT_AUDIENCE
+  GRAFANA_SERVICE_ACCOUNT_JSON
+  ROACHPERF_OPENMETRICS_CREDENTIALS
+  ROACHTEST_ASSERTIONS_ENABLED_SEED
+  ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH
+  SELECTIVE_TESTS
+  SLACK_TOKEN
+  SNOWFLAKE_PVT_KEY
+  SNOWFLAKE_USER
+  TC_BUILDTYPE_ID
+  TC_BUILD_BRANCH
+  TC_BUILD_ID
+  TC_SERVER_URL
+  TESTS
+  USE_SPOT
+)
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="$(printf -- "-e %s " "${docker_env_args[@]}")"
+
+benchstat_before="artifacts/benchstat_before.txt"
+benchstat_after="artifacts/benchstat_after.txt"
+tc_end_block "Configuration"
+
+tc_start_block "Baseline tpcc-nowait tests"
+
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="$BAZEL_SUPPORT_EXTRA_DOCKER_ARGS" \
+    run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh
+
+# Benchmark results are zipped in artifacts; extract them for analysis in a temp dir.
+tmp_artifacts_before="$(mktemp -d)"
+find "${root}/artifacts" -type f -name "artifacts.zip" -exec sh -c '
+  src="$1"
+  dest="'"$tmp_artifacts_before"'"
+  base="'"${root}/artifacts"'"
+  rel="${src#$base/}"
+  rel_dir="$(dirname "$rel")"
+  outdir="$dest/$rel_dir"
+  mkdir -p "$outdir"
+  unzip -o "$src" -d "$outdir"
+' _ {} \;
+
+bash -xe ./scripts/tpcc_results.sh "$tmp_artifacts_before" > "$benchstat_before"
+rm -rf "$tmp_artifacts_before"
+
+echo "=== Baseline benchstat saved to $benchstat_before ==="
+tc_end_block "Baseline tpcc-nowait tests"
+
+tc_start_block "Generate new PGO profile"
+echo "=== Step 2: Generating new PGO profile ==="
+filename="$(date +"%Y%m%d%H%M%S")-$(git rev-parse HEAD).pb.gz"
+bazel build //pkg/cmd/run-pgo-build
+"$(bazel info bazel-bin)"/pkg/cmd/run-pgo-build/run-pgo-build_/run-pgo-build -out "artifacts/$filename"
+sha256_hash=$(shasum -a 256 "artifacts/$filename" | awk '{print $1}')
+
+# Upload to GCS
+google_credentials="$GCS_GOOGLE_CREDENTIALS" log_into_gcloud
+gcs_url="gs://cockroach-profiles/$filename"
+gsutil cp "artifacts/$filename" "$gcs_url"
+
+# Convert GCS URL to HTTPS URL for WORKSPACE
+https_url="https://storage.googleapis.com/cockroach-profiles/$filename"
+
+echo "=== PGO profile uploaded to $gcs_url ==="
+echo "=== SHA256: $sha256_hash ==="
+
+# Use sed to update the sha256 and url in the pgo_profile block
+# This is more robust than string replacement
+sed -i.bak -e '/pgo_profile(/,/)/ {
+    s|sha256 = ".*"|sha256 = "'"$sha256_hash"'"|
+    s|url = ".*"|url = "'"$https_url"'"|
+}' WORKSPACE
+rm -f WORKSPACE.bak
+
+tc_end_block "Generate new PGO profile"
+
+tc_start_block "Rebuild with new PGO profile"
+# Clear build cache to ensure new profile is used
+bazel clean
+rm -rf "artifacts/tpcc-nowait"
+
+# Run the same tests again with the new PGO profile
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="$BAZEL_SUPPORT_EXTRA_DOCKER_ARGS" \
+    run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh
+
+# Benchmark results are zipped in artifacts; extract them for analysis in a temp dir.
+tmp_artifacts_after="$(mktemp -d)"
+find "${root}/artifacts" -type f -name "artifacts.zip" -exec sh -c '
+  src="$1"
+  dest="'"$tmp_artifacts_after"'"
+  base="'"${root}/artifacts"'"
+  rel="${src#$base/}"
+  rel_dir="$(dirname "$rel")"
+  outdir="$dest/$rel_dir"
+  mkdir -p "$outdir"
+  unzip -o "$src" -d "$outdir"
+' _ {} \;
+
+bash -xe ./scripts/tpcc_results.sh "$tmp_artifacts_after" > "$benchstat_after"
+rm -rf "$tmp_artifacts_after"
+
+echo "=== Post-PGO benchstat saved to $benchstat_after ==="
+tc_end_block "Rebuild with new PGO profile"
+
+tc_start_block "Benchstat comparison"
+# Run benchstat comparison using bazel and save to file
+benchstat_comparison="artifacts/benchstat_comparison.txt"
+
+bazel build @org_golang_x_perf//cmd/benchstat
+"$(bazel info bazel-bin)"/external/org_golang_x_perf/cmd/benchstat/benchstat_/benchstat \
+    "$benchstat_before" "$benchstat_after" | tee "$benchstat_comparison"
+tc_end_block "Benchstat comparison"
+
+tc_start_block "Create PR"
+export GIT_AUTHOR_NAME="Justin Beaver"
+export GIT_COMMITTER_NAME="Justin Beaver"
+export GIT_AUTHOR_EMAIL="[email protected]"
+export GIT_COMMITTER_EMAIL="[email protected]"
+gh_username="cockroach-teamcity"
+
+# Create a new branch for the PR
+branch_name="pgo-profile-update-$(date +"%Y%m%d%H%M%S")"
+git checkout -b "$branch_name"
+
+# Stage the WORKSPACE file changes
+git add WORKSPACE
+
+# Create commit with descriptive message
+# Note: Full benchstat comparison is in the PR description, not in commit message
+commit_message="build: update PGO profile to $filename
+
+This commit updates the PGO profile used for building CockroachDB.
+
+The new profile was generated from tpcc-nowait benchmarks.
+See PR description for full benchmark comparison results.
+
+Epic: none
+Release note: none"
+
+git commit -m "$commit_message"
+set +x
+git push "https://$gh_username:$GH_TOKEN@github.com/$gh_username/cockroach.git" "$branch_name"
+set -x
+
+# Create PR using gh CLI
+pr_body="This PR updates the PGO (Profile-Guided Optimization) profile used for building CockroachDB.
+
+The new profile was validated using tpcc-nowait benchmarks. Results comparing the old profile (before) vs new profile (after):
+
+\`\`\`
+$(cat "$benchstat_comparison")
+\`\`\`
+
+Epic: none
+Release note: none"
+
+gh pr create \
+    --head "$gh_username:$branch_name" \
+    --title "build: update PGO profile to $filename" \
+    --body "$pr_body" \
+    --reviewer "cockroachdb/release-eng-prs" \
+    --base master | tee artifacts/pgo_profile_pr_url.txt
+tc_end_block "Create PR"

build/teamcity/internal/cockroach/build/ci/generate-profile-impl.sh

@@ -448,7 +448,11 @@ func generateAndValidateNewTargets(
 		return nil, nil, hlc.Timestamp{}, nil, err
 	}
 
-	prevTargets, err := AllTargets(ctx, prevDetails, p.ExecCfg())
+	targetTS := prevProgress.GetHighWater()
+	if targetTS == nil || targetTS.IsEmpty() {
+		targetTS = &prevDetails.StatementTime
+	}
+	prevTargets, err := AllTargets(ctx, prevDetails, p.ExecCfg(), *targetTS)
 	if err != nil {
 		return nil, nil, hlc.Timestamp{}, nil, err
 	}

build/teamcity/internal/release/pgo_profile_update.sh

@@ -62,7 +62,10 @@ func makeChangefeedConfigFromJobDetails(
 // from the statement time name map in old protos
 // or the TargetSpecifications in new ones.
 func AllTargets(
-	ctx context.Context, cd jobspb.ChangefeedDetails, execCfg *sql.ExecutorConfig,
+	ctx context.Context,
+	cd jobspb.ChangefeedDetails,
+	execCfg *sql.ExecutorConfig,
+	timestamp hlc.Timestamp,
 ) (changefeedbase.Targets, error) {
 	targets := changefeedbase.Targets{}
 	var err error
@@ -76,7 +79,7 @@ func AllTargets(
 					if len(cd.TargetSpecifications) > 1 {
 						return changefeedbase.Targets{}, errors.AssertionFailedf("database-level changefeed is not supported with multiple targets")
 					}
-					targets, err = getTargetsFromDatabaseSpec(ctx, ts, execCfg)
+					targets, err = getTargetsFromDatabaseSpec(ctx, ts, execCfg, timestamp)
 					if err != nil {
 						return changefeedbase.Targets{}, err
 					}
@@ -110,11 +113,17 @@ func AllTargets(
 }
 
 func getTargetsFromDatabaseSpec(
-	ctx context.Context, ts jobspb.ChangefeedTargetSpecification, execCfg *sql.ExecutorConfig,
+	ctx context.Context,
+	ts jobspb.ChangefeedTargetSpecification,
+	execCfg *sql.ExecutorConfig,
+	timestamp hlc.Timestamp,
 ) (targets changefeedbase.Targets, err error) {
 	err = sql.DescsTxn(ctx, execCfg, func(
 		ctx context.Context, txn isql.Txn, descs *descs.Collection,
 	) error {
+		if err := txn.KV().SetFixedTimestamp(ctx, timestamp); err != nil {
+			return errors.Wrapf(err, "setting timestamp for table descriptor fetch")
+		}
 		databaseDescriptor, err := descs.ByIDWithLeased(txn.KV()).Get().Database(ctx, ts.DescID)
 		if err != nil {
 			return err