Merge #159633 #159645

159633: storage: bump Pebble format r=RaduBerinde a=RaduBerinde

Epic: none
Release note: None

159645: importer: delete bulkio.import.write_import_epoch.enabled setting r=jeffswenson a=jeffswenson

This is an internal setting that has been on by default for a few versions now. We need to delete it because setting it to false can cause data corruption if online restore restores a non epoch based import.

Release note: none
Fixes: #153999

Co-authored-by: Radu Berinde <radu@cockroachlabs.com>
Co-authored-by: Jeff Swenson <jeffswenson@betterthannull.com>

Author: 3 people

pkg/backup/testdata/backup-restore/import-epoch

@@ -5,10 +5,6 @@
 new-cluster name=s1
 ----
 
-exec-sql
-SET CLUSTER SETTING bulkio.import.write_import_epoch.enabled=true;
-----
-
 exec-sql
 CREATE DATABASE d;
 USE d;
@@ -88,20 +84,3 @@ query-sql
 SELECT * FROM import_epoch
 ----
 3 <nil>
-
-
-exec-sql
-SET CLUSTER SETTING bulkio.import.write_import_epoch.enabled=false;
-----
-
-# Ensure that the correct ImportType is set when the cluster setting is disabled
-import expect-pausepoint tag=b
-IMPORT INTO foo (i,s) CSV DATA ('nodelocal://1/export3/export*-n*.0.csv')
-----
-job paused at pausepoint
-
-# We expect the zero-valued version here.
-query-sql
-SELECT * FROM import_epoch
-----
-3 <nil>

pkg/cli/testdata/ear-list

@@ -8,13 +8,13 @@ list
 000004.log:
   env type: Data, AES128_CTR
   keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef
-  nonce: 86 a7 78 ad 4b da 62 56 d5 e2 d1 70
-  counter: 798955289
+  nonce: d0 b1 31 4b 08 b9 f6 08 7e e6 af 40
+  counter: 2167389540
 000005.sst:
   env type: Data, AES128_CTR
   keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef
-  nonce: d1 05 79 53 68 35 a0 f1 44 01 22 79
-  counter: 1497766936
+  nonce: 13 78 ef e7 4f 95 15 83 24 d8 1a 9d
+  counter: 1330556971
 COCKROACHDB_DATA_KEYS_000001_monolith:
   env type: Store, AES128_CTR
   keyID: f594229216d81add7811c4360212eb7629b578ef4eab6e5d05679b3c5de48867
@@ -35,11 +35,11 @@ marker.datakeys.000001.COCKROACHDB_DATA_KEYS_000001_monolith:
   keyID: f594229216d81add7811c4360212eb7629b578ef4eab6e5d05679b3c5de48867
   nonce: 55 d7 d4 27 6c 97 9b dd f1 5d 40 c8
   counter: 467030050
-marker.format-version.000013.026:
+marker.format-version.000015.028:
   env type: Data, AES128_CTR
   keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef
-  nonce: 23 d9 b2 e1 39 b0 87 ed f9 6d 49 20
-  counter: 3481614039
+  nonce: d1 05 79 53 68 35 a0 f1 44 01 22 79
+  counter: 1497766936
 marker.manifest.000001.MANIFEST-000001:
   env type: Data, AES128_CTR
   keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef

pkg/settings/registry.go

@@ -276,6 +276,7 @@ var retiredSettings = map[InternalKey]struct{}{
 	"storage.columnar_blocks.enabled": {},
 
 	// removed as of 26.1
+	"bulkio.import.write_import_epoch.enabled":            {},
 	"sql.distsql_planning.use_gossip.enabled":             {},
 	"rocksdb.ingest_backpressure.l0_file_count_threshold": {},
 	"rocksdb.ingest_backpressure.max_delay":               {},

pkg/sql/importer/import_job.go

@@ -429,13 +429,8 @@ func (r *importResumer) prepareTableForIngestion(
 	// TODO(dt): audit everywhere we get table descs (leases or otherwise) to
 	// ensure that filtering by state handles IMPORTING correctly.
 
-	// We only use the new OfflineForImport on 24.1, which bumps
-	// the ImportEpoch, if we are completely on 24.1.
-	if importEpochs.Get(&p.ExecCfg().Settings.SV) {
-		importing.OfflineForImport()
-	} else {
-		importing.SetOffline(tabledesc.OfflineReasonImporting)
-	}
+	// Use OfflineForImport which bumps the ImportEpoch.
+	importing.OfflineForImport()
 
 	// TODO(dt): de-validate all the FKs.
 	if err := descsCol.WriteDesc(
@@ -709,13 +704,6 @@ var retryDuration = settings.RegisterDurationSetting(
 	settings.PositiveDuration,
 )
 
-var importEpochs = settings.RegisterBoolSetting(
-	settings.ApplicationLevel,
-	"bulkio.import.write_import_epoch.enabled",
-	"controls whether IMPORT will write ImportEpoch's to descriptors",
-	true,
-)
-
 func getFractionCompleted(job *jobs.Job) float64 {
 	p := job.Progress()
 	return float64(p.GetFractionCompleted())

pkg/sql/importer/import_mvcc_test.go

@@ -48,7 +48,6 @@ func TestMVCCValueHeaderImportEpoch(t *testing.T) {
 
 	// Create a table where the first row ( in sort order) comes from an IMPORT
 	// while the second comes from an INSERT.
-	sqlDB.Exec(t, `SET CLUSTER SETTING bulkio.import.write_import_epoch.enabled=true`)
 	sqlDB.Exec(t, `CREATE TABLE d.t (a INT8)`)
 	sqlDB.Exec(t, `INSERT INTO d.t VALUES ('2')`)
 	sqlDB.Exec(t, `IMPORT INTO d.t CSV DATA ($1)`, srv.URL)

pkg/storage/pebble.go

@@ -2506,6 +2506,7 @@ func (p *Pebble) CreateCheckpoint(dir string, spans []roachpb.Span) error {
 // named version, it can be assumed all *nodes* have ratcheted to the pebble
 // version associated with it, since they did so during the fence version.
 var pebbleFormatVersionMap = map[clusterversion.Key]pebble.FormatMajorVersion{
+	clusterversion.V26_1: pebble.FormatMarkForCompactionInVersionEdit,
 	clusterversion.V25_4: pebble.FormatV2BlobFiles,
 }