Merge #153474 #153531

153474: backupdest: remove old manifest format from 20.1 r=jeffswenson a=msbutler

backup: rename BackupManifestName DeprecatedBackupManifestName

Informs #139159

Release note: none

---
backup: always write slim manifest

Previously, backups would write the slim manifest if the a default true cluster
setting was true. This patch removes this cluster setting, so backups will
always write the slim manifest.

This work is part of an effort to remove deprecated backup manifest versions.
Since this patch will take effect on 25.4 clusters, backup _reads_ on 26.2
clusters can exclusively use the slim manifest without any failback logic to
the deprecated fat manifest.

Since there still exists logic to read the deprecated manifest exclusively
(e.g. SHOW BACKUPS), we cannot remove the deprecated manifest _write_ path
until all these read paths have fallback logic to the new slim manifest-- work
for an upcoming PR.

Informs #139159

Release note: none

---

backup: remove old manifest format from 20.1

Informs: #139159

Release note: none

153531: workload/ycsb: use crrand.Perm64 for pseudorandom keys r=RaduBerinde a=jbowens

This commit reworks the random ycsb key generation to use a pseudorandom
permutation of 64-bit integers constructed deterministically from the seed,
rather than hashing the key index. The pseudorandom permutation is provided by
the crrand.Perm64 type. The per-key computation is likely lighterweight than
the previous fnv hash, but more importantly, it avoids duplicates. This ensures
that a large IMPORT of random keys will not fail due to a key uniqueness
violation. This commit also accordingly renames the --insert-hash flag to
--insert-random (leaving an alias with the old name).

Epic: none
Release note: none

Co-authored-by: Michael Butler <[email protected]>
Co-authored-by: Jackson Owens <[email protected]>

Author: 3 people

DEPS.bzl

@@ -1740,10 +1740,10 @@ def go_deps():
         name = "com_github_cockroachdb_crlib",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/cockroachdb/crlib",
-        sha256 = "4fb4d32f15fb3c63decb9625406f291874f975518d5937493a7a9d1f3a644f18",
-        strip_prefix = "github.com/cockroachdb/[email protected]20250718215705-7ff5051265b9",
+        sha256 = "3481f073a07aad90b3e66e357ed9b1212fb649b2653d6ce91a7d6a345632a9e4",
+        strip_prefix = "github.com/cockroachdb/[email protected]20250914170308-eb2d55dbae2e",
         urls = [
-            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/crlib/com_github_cockroachdb_crlib-v0.0.0-20250718215705-7ff5051265b9.zip",
+            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/crlib/com_github_cockroachdb_crlib-v0.0.0-20250914170308-eb2d55dbae2e.zip",
         ],
     )
     go_repository(

build/bazelutil/distdir_files.bzl

@@ -345,7 +345,7 @@ DISTDIR_FILES = {
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/cmux/com_github_cockroachdb_cmux-v0.0.0-20250514152509-914d3bf9ec58.zip": "c1cf4cd99a1ad6a00f2ccd4188cbcf004cb0d56895670b2c171061ce564cd791",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/cockroach-go/v2/com_github_cockroachdb_cockroach_go_v2-v2.4.1.zip": "ba646db91152f3121a6812c7b74d12d8c0e126f7b4d3b927618b159692ceb424",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/crlfmt/com_github_cockroachdb_crlfmt-v0.0.0-20221214225007-b2fc5c302548.zip": "fedc01bdd6d964da0425d5eaac8efadc951e78e13f102292cc0774197f09ab63",
-    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/crlib/com_github_cockroachdb_crlib-v0.0.0-20250718215705-7ff5051265b9.zip": "4fb4d32f15fb3c63decb9625406f291874f975518d5937493a7a9d1f3a644f18",
+    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/crlib/com_github_cockroachdb_crlib-v0.0.0-20250914170308-eb2d55dbae2e.zip": "3481f073a07aad90b3e66e357ed9b1212fb649b2653d6ce91a7d6a345632a9e4",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/datadriven/com_github_cockroachdb_datadriven-v1.0.3-0.20250407164829-2945557346d5.zip": "251593cd9c040fe84a99a3919de7ce6f85030d522159a37d625dc2dea7a4d17f",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/drpc/com_github_cockroachdb_drpc-v0.0.0-20250807091527-65dcebaa113e.zip": "ebb4b3ca9a6c5944255506b4cf5091b86b76851e52211940b609c98e8bcc7aab",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/errors/com_github_cockroachdb_errors-v1.12.0.zip": "f73d8a5f4d8fcbc4ed61db2b47f17e2601d8b32e9a49c0665667489d9d9d6e7c",

go.mod

@@ -135,7 +135,7 @@ require (
 	github.com/cockroachdb/cmux v0.0.0-20250514152509-914d3bf9ec58
 	github.com/cockroachdb/cockroach-go/v2 v2.4.1
 	github.com/cockroachdb/crlfmt v0.0.0-20221214225007-b2fc5c302548
-	github.com/cockroachdb/crlib v0.0.0-20250718215705-7ff5051265b9
+	github.com/cockroachdb/crlib v0.0.0-20250914170308-eb2d55dbae2e
 	github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5
 	github.com/cockroachdb/errors v1.12.0
 	github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55

go.sum

@@ -553,8 +553,8 @@ github.com/cockroachdb/cockroach-go/v2 v2.4.1 h1:ACVT/zXsuK6waRPVYtDQpsM8pPA7IA/
 github.com/cockroachdb/cockroach-go/v2 v2.4.1/go.mod h1:9U179XbCx4qFWtNhc7BiWLPfuyMVQ7qdAhfrwLz1vH0=
 github.com/cockroachdb/crlfmt v0.0.0-20221214225007-b2fc5c302548 h1:i0bnjanlWAvM50wHMT7EFyxlt5HQusznWrkwl+HBIsU=
 github.com/cockroachdb/crlfmt v0.0.0-20221214225007-b2fc5c302548/go.mod h1:qtkxNlt5i3rrdirfJE/bQeW/IeLajKexErv7jEIV+Uc=
-github.com/cockroachdb/crlib v0.0.0-20250718215705-7ff5051265b9 h1:eNjHrT3pPlP1q/4VT/IRGzl8Jiq1XHrSpLJmrGFJDT8=
-github.com/cockroachdb/crlib v0.0.0-20250718215705-7ff5051265b9/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac=
+github.com/cockroachdb/crlib v0.0.0-20250914170308-eb2d55dbae2e h1:qn+p2EISsM2HjMDt6Ivx5W3CG/jCz/JSCeLnF9vJju8=
+github.com/cockroachdb/crlib v0.0.0-20250914170308-eb2d55dbae2e/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
 github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5 h1:UycK/E0TkisVrQbSoxvU827FwgBBcZ95nRRmpj/12QI=

pkg/backup/backup_job.go

@@ -427,24 +427,16 @@ func backup(
 		}
 	}
 
-	// Write a `BACKUP_MANIFEST` file to support backups in mixed-version clusters
-	// with 22.2 nodes.
-	//
-	// TODO(adityamaru): We can stop writing `BACKUP_MANIFEST` in 23.2
-	// because a mixed-version cluster with 23.1 nodes will read the
-	// `BACKUP_METADATA` instead.
-	if err := backupinfo.WriteBackupManifest(ctx, defaultStore, backupbase.BackupManifestName,
+	// TODO(msbutler): version gate writing the old manifest once we can guarantee
+	// a cluster version that will not read the old manifest.
+	if err := backupinfo.WriteBackupManifest(ctx, defaultStore, backupbase.DeprecatedBackupManifestName,
 		encryption, &kmsEnv, backupManifest); err != nil {
 		return roachpb.RowCount{}, 0, err
 	}
 
-	// Write a `BACKUP_METADATA` file along with SSTs for all the alloc heavy
-	// fields elided from the `BACKUP_MANIFEST`.
-	if backupinfo.WriteMetadataWithExternalSSTsEnabled.Get(&settings.SV) {
-		if err := backupinfo.WriteMetadataWithExternalSSTs(ctx, defaultStore, encryption,
-			&kmsEnv, backupManifest); err != nil {
-			return roachpb.RowCount{}, 0, err
-		}
+	if err := backupinfo.WriteMetadataWithExternalSSTs(ctx, defaultStore, encryption,
+		&kmsEnv, backupManifest); err != nil {
+		return roachpb.RowCount{}, 0, err
 	}
 
 	statsTable := getTableStatsForBackup(ctx, execCtx.ExecCfg().InternalDB.Executor(), backupManifest.Descriptors)

pkg/backup/backup_test.go

@@ -144,7 +144,7 @@ func TestBackupRestoreStatementResult(t *testing.T) {
 	// have been stored in the GZip compressed format.
 	t.Run("GZipBackupManifest", func(t *testing.T) {
 		backupDir := fmt.Sprintf("%s/foo", dir)
-		backupManifestFile := backupDir + backupPath + "/" + backupbase.BackupManifestName
+		backupManifestFile := backupDir + backupPath + "/" + backupbase.DeprecatedBackupManifestName
 		backupManifestBytes, err := os.ReadFile(backupManifestFile)
 		if err != nil {
 			t.Fatal(err)
@@ -668,7 +668,7 @@ func TestBackupRestoreAppend(t *testing.T) {
 	// Find the backup times in the collection and try RESTORE'ing to each, and
 	// within each also check if we can restore to individual times captured with
 	// incremental backups that were appended to that backup.
-	fullBackup1, fullBackup2 := findFullBackupPaths(tmpDir, path.Join(tmpDir, "*/*/*/"+backupbase.BackupManifestName))
+	fullBackup1, fullBackup2 := findFullBackupPaths(tmpDir, path.Join(tmpDir, "*/*/*/"+backupbase.DeprecatedBackupManifestName))
 
 	sqlDB.Exec(t, "DROP DATABASE data CASCADE")
 	sqlDB.Exec(t, "RESTORE DATABASE data FROM $4 IN ($1, $2, $3) AS OF SYSTEM TIME "+tsBefore,
@@ -740,7 +740,7 @@ func TestBackupAndRestoreJobDescription(t *testing.T) {
 	sqlDB.Exec(t, "BACKUP INTO ($1, $2, $3) AS OF SYSTEM TIME '-1s'", collections...)
 
 	// Find the subdirectory created by the full BACKUP INTO statement.
-	matches, err := filepath.Glob(path.Join(tmpDir, "full/*/*/*/"+backupbase.BackupManifestName))
+	matches, err := filepath.Glob(path.Join(tmpDir, "full/*/*/*/"+backupbase.DeprecatedBackupManifestName))
 	require.NoError(t, err)
 	require.Equal(t, 2, len(matches))
 	for i := range matches {
@@ -1786,7 +1786,7 @@ func TestBackupRestoreResume(t *testing.T) {
 
 				// If the backup properly took the (incorrect) checkpoint into account, it
 				// won't have tried to re-export any keys within backupCompletedSpan.
-				backupManifestFile := path.Join(dir, item.testName, subdir, backupbase.BackupManifestName)
+				backupManifestFile := path.Join(dir, item.testName, subdir, backupbase.DeprecatedBackupManifestName)
 				backupManifestBytes, err := os.ReadFile(backupManifestFile)
 				if err != nil {
 					t.Fatal(err)
@@ -4000,7 +4000,7 @@ func TestBackupRestoreChecksum(t *testing.T) {
 
 	var backupManifest backuppb.BackupManifest
 	{
-		backupManifestBytes, err := os.ReadFile(filepath.Join(dir, backupPath, backupbase.BackupManifestName))
+		backupManifestBytes, err := os.ReadFile(filepath.Join(dir, backupPath, backupbase.DeprecatedBackupManifestName))
 		if err != nil {
 			t.Fatalf("%+v", err)
 		}
@@ -8456,7 +8456,6 @@ func TestIncorrectAccessOfFilesInBackupMetadata(t *testing.T) {
 
 	_, sqlDB, rawDir, cleanupFn := backupRestoreTestSetup(t, singleNode, 1, InitManualReplication)
 	defer cleanupFn()
-	sqlDB.Exec(t, `SET CLUSTER SETTING backup.write_metadata_with_external_ssts.enabled=true`)
 	sqlDB.Exec(t, `CREATE DATABASE r1`)
 	sqlDB.Exec(t, `CREATE TABLE r1.foo ( id INT PRIMARY KEY)`)
 	sqlDB.Exec(t, `INSERT INTO r1.foo VALUES (1)`)
@@ -8502,7 +8501,6 @@ func TestRestoringAcrossVersions(t *testing.T) {
 	tc, sqlDB, rawDir, cleanupFn := backupRestoreTestSetup(t, singleNode, 1, InitManualReplication)
 	defer cleanupFn()
 
-	sqlDB.Exec(t, `SET CLUSTER SETTING backup.write_metadata_with_external_ssts.enabled=true`)
 	sqlDB.Exec(t, `CREATE DATABASE r1`)
 
 	sqlDB.Exec(t, `BACKUP DATABASE r1 INTO 'nodelocal://1/cross_version'`)
@@ -8615,7 +8613,6 @@ func TestManifestBitFlip(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 	_, sqlDB, rawDir, cleanupFn := backupRestoreTestSetup(t, singleNode, 1, InitManualReplication)
-	sqlDB.Exec(t, `SET CLUSTER SETTING backup.write_metadata_with_external_ssts.enabled=true`)
 	defer cleanupFn()
 	sqlDB.Exec(t, `CREATE DATABASE r1; CREATE DATABASE r2; CREATE DATABASE r3;`)
 	const checksumError = "checksum mismatch"
@@ -10422,7 +10419,7 @@ func TestBackupDoNotIncludeViewSpans(t *testing.T) {
 	// Read the backup manifest.
 	var backupManifest backuppb.BackupManifest
 	{
-		backupManifestBytes, err := os.ReadFile(filepath.Join(dir, "foo", res[0][0], backupbase.BackupManifestName))
+		backupManifestBytes, err := os.ReadFile(filepath.Join(dir, "foo", res[0][0], backupbase.DeprecatedBackupManifestName))
 		if err != nil {
 			t.Fatalf("%+v", err)
 		}

pkg/backup/backupbase/constants.go

@@ -35,19 +35,12 @@ const (
 	// Also exported for testing backup inspection tooling.
 	DateBasedIntoFolderName = "/2006/01/02-150405.00"
 
-	// BackupOldManifestName is an old name for the serialized BackupManifest
-	// proto. It is used by 20.1 nodes and earlier.
+	// DeprecatedBackupManifestName is the file name used for serialized
+	// BackupManifest protos.
 	//
-	// TODO(adityamaru): Remove this in 22.2 as part of disallowing backups
-	// from >1 major version in the past.
-	BackupOldManifestName = "BACKUP"
-
-	// BackupManifestName is the file name used for serialized BackupManifest
-	// protos.
-	//
-	// TODO(adityamaru): Remove in 23.2 since at that point all nodes will be
-	// writing a SlimBackupManifest instead.
-	BackupManifestName = "BACKUP_MANIFEST"
+	// TODO(msbutler): Remove 26.3 when we're guaranteed that no backup wrote
+	// exclusively the backup_manifest, and not the slim manifest.
+	DeprecatedBackupManifestName = "BACKUP_MANIFEST"
 
 	// BackupMetadataName is the file name used for serialized BackupManifest
 	// protos written by 23.1 nodes and later. This manifest has the alloc heavy

pkg/backup/backupdest/backup_destination.go

@@ -53,7 +53,7 @@ const (
 // On some cloud storage platforms (i.e. GS, S3), backups in a base bucket may
 // omit a leading slash. However, backups in a subdirectory of a base bucket
 // will contain one.
-var backupPathRE = regexp.MustCompile("^/?[^\\/]+/[^\\/]+/[^\\/]+/" + backupbase.BackupManifestName + "$")
+var backupPathRE = regexp.MustCompile("^/?[^\\/]+/[^\\/]+/[^\\/]+/" + backupbase.DeprecatedBackupManifestName + "$")
 
 // featureFullBackupUserSubdir, when true, will create a full backup at a user
 // specified subdirectory if no backup already exists at that subdirectory. As
@@ -68,7 +68,7 @@ var featureFullBackupUserSubdir = settings.RegisterBoolSetting(
 
 // TODO(adityamaru): Move this to the soon to be `backupinfo` package.
 func containsManifest(ctx context.Context, exportStore cloud.ExternalStorage) (bool, error) {
-	r, _, err := exportStore.ReadFile(ctx, backupbase.BackupManifestName, cloud.ReadOptions{NoFileSize: true})
+	r, _, err := exportStore.ReadFile(ctx, backupbase.DeprecatedBackupManifestName, cloud.ReadOptions{NoFileSize: true})
 	if err != nil {
 		if errors.Is(err, cloud.ErrFileDoesNotExist) {
 			return false, nil
@@ -534,7 +534,7 @@ func ListFullBackupsInCollection(
 		return nil, err
 	}
 	for i, backupPath := range backupPaths {
-		backupPaths[i] = strings.TrimSuffix(backupPath, "/"+backupbase.BackupManifestName)
+		backupPaths[i] = strings.TrimSuffix(backupPath, "/"+backupbase.DeprecatedBackupManifestName)
 	}
 	return backupPaths, nil
 }

pkg/backup/backupdest/backup_destination_test.go

@@ -56,7 +56,7 @@ func TestBackupRestoreResolveDestination(t *testing.T) {
 		require.NoError(t, err)
 		reader := bytes.NewReader(manifestBytes)
 		require.NoError(t, err)
-		require.NoError(t, cloud.WriteFile(ctx, storage, backupbase.BackupManifestName, reader))
+		require.NoError(t, cloud.WriteFile(ctx, storage, backupbase.DeprecatedBackupManifestName, reader))
 	}
 
 	// writeLatest writes latestBackupSuffix to the LATEST file in the given

pkg/backup/backupdest/incrementals.go

@@ -120,7 +120,7 @@ func FindAllIncrementalPaths(
 
 	if includeManifest {
 		paths = util.Map(paths, func(p string) string {
-			return path.Join(p, backupbase.BackupManifestName)
+			return path.Join(p, backupbase.DeprecatedBackupManifestName)
 		})
 	}
 	return paths, nil
@@ -141,32 +141,23 @@ func LegacyFindPriorBackups(
 
 	var prev []string
 	if err := store.List(ctx, "", backupbase.ListingDelimDataSlash, func(p string) error {
-		matchesGlob, err := path.Match(incBackupSubdirGlob+backupbase.BackupManifestName, p)
+		matchesGlob, err := path.Match(incBackupSubdirGlob+backupbase.DeprecatedBackupManifestName, p)
 		if err != nil {
 			return err
 		} else if !matchesGlob {
-			matchesGlob, err = path.Match(incBackupSubdirGlobWithSuffix+backupbase.BackupManifestName, p)
+			matchesGlob, err = path.Match(incBackupSubdirGlobWithSuffix+backupbase.DeprecatedBackupManifestName, p)
 			if err != nil {
 				return err
 			}
 		}
 
 		if matchesGlob {
 			if !includeManifest {
-				p = strings.TrimSuffix(p, "/"+backupbase.BackupManifestName)
+				p = strings.TrimSuffix(p, "/"+backupbase.DeprecatedBackupManifestName)
 			}
 			prev = append(prev, p)
 			return nil
 		}
-
-		if ok, err := path.Match(incBackupSubdirGlob+backupbase.BackupOldManifestName, p); err != nil {
-			return err
-		} else if ok {
-			if !includeManifest {
-				p = strings.TrimSuffix(p, "/"+backupbase.BackupOldManifestName)
-			}
-			prev = append(prev, p)
-		}
 		return nil
 	}); err != nil {
 		return nil, errors.Wrap(err, "reading previous backup layers")