Merge #149371

149371: storage: remove 'storage.columnar_blocks.enabled' setting r=jbowens a=jbowens

Remove the 'storage.columnar_blocks.enabled' cluster setting, requiring that columnar blocks always be enabled. All recent sstable formats use columnar blocks. Eventually we'd like to remove support for row-block formats altogether once we have a guarantee that rowblk blocks no longer exist anywhere. This commit changes Pebble's behavior to treat columnar blocks as always enabled.

Epic: none
Release note (ops change): Removes the 'storage.columnar_blocks.enabled' cluster setting; columnar blocks are always enabled.

Co-authored-by: Jackson Owens <[email protected]>

Author: craig[bot]

docs/generated/settings/settings-for-tenants.txt

@@ -398,7 +398,6 @@ sql.txn.read_committed_isolation.enabled	boolean	true	set to true to allow trans
 sql.txn.repeatable_read_isolation.enabled (alias: sql.txn.snapshot_isolation.enabled)	boolean	false	set to true to allow transactions to use the REPEATABLE READ isolation level if specified by BEGIN/SET commands	application
 sql.txn_fingerprint_id_cache.capacity	integer	100	the maximum number of txn fingerprint IDs stored	application
 sql.vecindex.stalled_op.timeout	duration	100ms	amount of time before other vector index workers will assist with a stalled background fixup	application
-storage.columnar_blocks.enabled	boolean	true	set to true to enable columnar-blocks to store KVs in a columnar format	system-visible
 storage.delete_compaction_excise.enabled	boolean	true	set to false to direct Pebble to not partially excise sstables in delete-only compactions	system-visible
 storage.ingestion.value_blocks.enabled	boolean	true	set to true to enable writing of value blocks in ingestion sstables	application
 storage.max_sync_duration	duration	20s	maximum duration for disk operations; any operations that take longer than this setting trigger a warning log entry or process crash	system-visible

docs/generated/settings/settings.html

@@ -353,7 +353,6 @@
 <tr><td><div id="setting-sql-txn-snapshot-isolation-enabled" class="anchored"><code>sql.txn.repeatable_read_isolation.enabled<br />(alias: sql.txn.snapshot_isolation.enabled)</code></div></td><td>boolean</td><td><code>false</code></td><td>set to true to allow transactions to use the REPEATABLE READ isolation level if specified by BEGIN/SET commands</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-sql-txn-fingerprint-id-cache-capacity" class="anchored"><code>sql.txn_fingerprint_id_cache.capacity</code></div></td><td>integer</td><td><code>100</code></td><td>the maximum number of txn fingerprint IDs stored</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-sql-vecindex-stalled-op-timeout" class="anchored"><code>sql.vecindex.stalled_op.timeout</code></div></td><td>duration</td><td><code>100ms</code></td><td>amount of time before other vector index workers will assist with a stalled background fixup</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
-<tr><td><div id="setting-storage-columnar-blocks-enabled" class="anchored"><code>storage.columnar_blocks.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>set to true to enable columnar-blocks to store KVs in a columnar format</td><td>Dedicated/Self-hosted (read-write); Serverless (read-only)</td></tr>
 <tr><td><div id="setting-storage-delete-compaction-excise-enabled" class="anchored"><code>storage.delete_compaction_excise.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>set to false to direct Pebble to not partially excise sstables in delete-only compactions</td><td>Dedicated/Self-hosted (read-write); Serverless (read-only)</td></tr>
 <tr><td><div id="setting-storage-ingest-split-enabled" class="anchored"><code>storage.ingest_split.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>set to false to disable ingest-time splitting that lowers write-amplification</td><td>Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-storage-ingestion-value-blocks-enabled" class="anchored"><code>storage.ingestion.value_blocks.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>set to true to enable writing of value blocks in ingestion sstables</td><td>Serverless/Dedicated/Self-Hosted</td></tr>

pkg/cmd/roachtest/tests/tpcc.go

@@ -2453,7 +2453,6 @@ func runTPCCPublished(
 				_, _ = db.ExecContext(ctx, `SET CLUSTER SETTING admission.kv.enabled = false`)
 				_, _ = db.ExecContext(ctx, `SET CLUSTER SETTING kv.replication_reports.interval = '0s'`)
 				_, _ = db.ExecContext(ctx, `ALTER RANGE default CONFIGURE ZONE USING gc.ttlseconds = 600`)
-				_, _ = db.ExecContext(ctx, `SET CLUSTER SETTING storage.columnar_blocks.enabled = false;`)
 
 			}
 			require.NoError(t, db.Close())

pkg/kv/kvserver/kvstorage/snaprecv/sst_snapshot_storage_test.go

@@ -295,8 +295,6 @@ func testMultiSSTWriterInitSSTInner(t *testing.T, interesting bool) {
 
 	st := cluster.MakeTestingClusterSettings()
 
-	// Disabling columnar blocks causes stats changes.
-	storage.ColumnarBlocksEnabled.Override(context.Background(), &st.SV, true)
 	// The tests rely on specific SST sizes; we cannot use MinLZ as the
 	// compression can depend on the architecture.
 	storage.CompressionAlgorithmStorage.Override(context.Background(), &st.SV, storage.StoreCompressionSnappy)

pkg/kv/kvserver/rangefeed/event_size_test.go

@@ -96,7 +96,6 @@ func generateStaticTestdata() testData {
 func TestEventSizeCalculation(t *testing.T) {
 	st := cluster.MakeTestingClusterSettings()
 	data := generateStaticTestdata()
-	storage.ColumnarBlocksEnabled.Override(context.Background(), &st.SV, true)
 	storage.CompressionAlgorithmStorage.Override(context.Background(), &st.SV, storage.StoreCompressionSnappy)
 
 	key := data.key

pkg/settings/registry.go

@@ -271,6 +271,9 @@ var retiredSettings = map[InternalKey]struct{}{
 	// removed as of 25.3
 	"sql.metrics.max_stmt_fingerprints_per_explicit_txn": {},
 	"sql.jobs.legacy_per_job_access_via_details.enabled": {},
+
+	// removed as of 25.4
+	"storage.columnar_blocks.enabled": {},
 }
 
 // grandfatheredDefaultSettings is the list of "grandfathered" existing sql.defaults

pkg/storage/pebble.go

@@ -74,16 +74,6 @@ var IngestSplitEnabled = settings.RegisterBoolSetting(
 	settings.WithPublic,
 )
 
-// ColumnarBlocksEnabled controls whether columnar-blocks are enabled in Pebble.
-var ColumnarBlocksEnabled = settings.RegisterBoolSetting(
-	settings.SystemVisible,
-	"storage.columnar_blocks.enabled",
-	"set to true to enable columnar-blocks to store KVs in a columnar format",
-	metamorphic.ConstantWithTestBool(
-		"storage.columnar_blocks.enabled", true /* defaultValue */),
-	settings.WithPublic,
-)
-
 // deleteCompactionsCanExcise controls whether delete compactions can
 // apply rangedels/rangekeydels on sstables they partially apply to, through
 // an excise operation, instead of just applying the rangedels/rangekeydels
@@ -423,8 +413,7 @@ var (
 	valueSeparationEnabled = settings.RegisterBoolSetting(
 		settings.SystemVisible,
 		"storage.value_separation.enabled",
-		"whether or not values may be separated into blob files; "+
-			"requires columnar blocks to be enabled",
+		"whether or not values may be separated into blob files",
 		metamorphic.ConstantWithTestBool(
 			"storage.value_separation.enabled", true /* defaultValue */),
 	)
@@ -919,9 +908,7 @@ func newPebble(ctx context.Context, cfg engineConfig) (p *Pebble, err error) {
 	cfg.opts.Experimental.IngestSplit = func() bool {
 		return IngestSplitEnabled.Get(&cfg.settings.SV)
 	}
-	cfg.opts.Experimental.EnableColumnarBlocks = func() bool {
-		return ColumnarBlocksEnabled.Get(&cfg.settings.SV)
-	}
+	cfg.opts.Experimental.EnableColumnarBlocks = func() bool { return true }
 	cfg.opts.Experimental.EnableDeleteOnlyCompactionExcises = func() bool {
 		return deleteCompactionsCanExcise.Get(&cfg.settings.SV)
 	}

pkg/storage/sst_writer.go

@@ -82,13 +82,9 @@ func (*noopFinishAbort) Abort() {}
 // also used when constructing sstables for backups (because these sstables may
 // ultimately be ingested during online restore).
 func MakeIngestionWriterOptions(ctx context.Context, cs *cluster.Settings) sstable.WriterOptions {
-	// All supported versions understand TableFormatPebblev4. If columnar blocks
-	// are enabled and the active cluster version is at least 24.3, use
-	// TableFormatPebblev5.
-	format := sstable.TableFormatPebblev4
-	if ColumnarBlocksEnabled.Get(&cs.SV) {
-		format = sstable.TableFormatPebblev5
-	}
+	// TODO(jackson): Tie the table format to the cluster version using
+	// FormatMajorVersion.MaxTableFormat.
+	format := sstable.TableFormatPebblev5
 
 	opts := DefaultPebbleOptions().MakeWriterOptions(0, format)
 	// By default, compress with the algorithm used for L6 in a Pebble store.
@@ -123,13 +119,9 @@ func makeSSTRewriteOptions(
 // scanned and their keys inserted into new sstables (NB: constructed using
 // MakeIngestionSSTWriter) that ultimately are uploaded to object storage.
 func MakeTransportSSTWriter(ctx context.Context, cs *cluster.Settings, f io.Writer) SSTWriter {
-	// By default, take a conservative approach and assume we don't have newer
-	// table features available. Upgrade to an appropriate version only if the
-	// cluster supports it.
-	format := sstable.TableFormatPebblev4
-	if ColumnarBlocksEnabled.Get(&cs.SV) {
-		format = sstable.TableFormatPebblev5
-	}
+	// TODO(jackson): Tie the table format to the cluster version using
+	// FormatMajorVersion.MaxTableFormat.
+	format := sstable.TableFormatPebblev5
 
 	opts := DefaultPebbleOptions().MakeWriterOptions(0, format)
 

pkg/storage/sst_writer_test.go

@@ -87,11 +87,10 @@ func TestMakeIngestionWriterOptions(t *testing.T) {
 			st: func() *cluster.Settings {
 				st := cluster.MakeTestingClusterSettings()
 				IngestionValueBlocksEnabled.Override(context.Background(), &st.SV, true)
-				ColumnarBlocksEnabled.Override(context.Background(), &st.SV, false)
 				return st
 			}(),
 			want: want{
-				format:             sstable.TableFormatPebblev4,
+				format:             sstable.TableFormatPebblev5,
 				disableValueBlocks: false,
 			},
 		},
@@ -100,40 +99,13 @@ func TestMakeIngestionWriterOptions(t *testing.T) {
 			st: func() *cluster.Settings {
 				st := cluster.MakeTestingClusterSettings()
 				IngestionValueBlocksEnabled.Override(context.Background(), &st.SV, false)
-				ColumnarBlocksEnabled.Override(context.Background(), &st.SV, false)
-				return st
-			}(),
-			want: want{
-				format:             sstable.TableFormatPebblev4,
-				disableValueBlocks: true,
-			},
-		},
-		{
-			name: "enable columnar blocks",
-			st: func() *cluster.Settings {
-				st := cluster.MakeTestingClusterSettings()
-				IngestionValueBlocksEnabled.Override(context.Background(), &st.SV, false)
-				ColumnarBlocksEnabled.Override(context.Background(), &st.SV, true)
 				return st
 			}(),
 			want: want{
 				format:             sstable.TableFormatPebblev5,
 				disableValueBlocks: true,
 			},
 		},
-		{
-			name: "enable columnar blocks with value blocks",
-			st: func() *cluster.Settings {
-				st := cluster.MakeTestingClusterSettings()
-				IngestionValueBlocksEnabled.Override(context.Background(), &st.SV, true)
-				ColumnarBlocksEnabled.Override(context.Background(), &st.SV, true)
-				return st
-			}(),
-			want: want{
-				format:             sstable.TableFormatPebblev5,
-				disableValueBlocks: false,
-			},
-		},
 	}
 
 	for _, tc := range testCases {