db: enable new formats

Author: annrpom

cockroachkvs/cockroachkvs_bench_test.go

@@ -79,6 +79,18 @@ func BenchmarkRandSeekInSST(b *testing.B) {
 			valueLen: 128,        // ~200 KVs per data block
 			version:  sstable.TableFormatPebblev7,
 		},
+		{
+			name:     "v8/single-level",
+			numKeys:  200 * 100, // ~100 data blocks.
+			valueLen: 128,       // ~200 KVs per data block
+			version:  sstable.TableFormatPebblev8,
+		},
+		{
+			name:     "v8/two-level",
+			numKeys:  200 * 5000, // ~5000 data blocks
+			valueLen: 128,        // ~200 KVs per data block
+			version:  sstable.TableFormatPebblev8,
+		},
 	}
 	keyCfg := KeyGenConfig{
 		PrefixAlphabetLen: 26,

format_major_version.go

@@ -253,6 +253,12 @@ const (
 	// Previously, marking for compaction required a manifest rotation.
 	FormatMarkForCompactionInVersionEdit
 
+	// FormatTieredStorage is a format major version that adds support for
+	// tiered storage based on the age of a key-value pair. It introduces a new
+	// columnar block format (among other things) that is required for tracking
+	// the attribute used to derive the age.
+	FormatTieredStorage
+
 	// -- Add new versions here --
 
 	// FormatNewest is the most recent format major version.
@@ -293,6 +299,8 @@ func (v FormatMajorVersion) resolveDefault() FormatMajorVersion {
 func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat {
 	v = v.resolveDefault()
 	switch {
+	case v >= FormatTieredStorage:
+		return sstable.TableFormatPebblev8
 	case v >= formatFooterAttributes:
 		return sstable.TableFormatPebblev7
 	case v >= FormatTableFormatV6:
@@ -398,6 +406,9 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
 	FormatMarkForCompactionInVersionEdit: func(d *DB) error {
 		return d.finalizeFormatVersUpgrade(FormatMarkForCompactionInVersionEdit)
 	},
+	FormatTieredStorage: func(d *DB) error {
+		return d.finalizeFormatVersUpgrade(FormatTieredStorage)
+	},
 }
 
 const formatVersionMarkerName = `format-version`
@@ -511,7 +522,11 @@ func (d *DB) ratchetFormatMajorVersionLocked(formatVers FormatMajorVersion) erro
 	defer func() { d.mu.formatVers.ratcheting = false }()
 
 	for nextVers := d.FormatMajorVersion() + 1; nextVers <= formatVers; nextVers++ {
-		if err := formatMajorVersionMigrations[nextVers](d); err != nil {
+		migration, ok := formatMajorVersionMigrations[nextVers]
+		if !ok || migration == nil {
+			return errors.Errorf("pebble: no migration function defined for format version %d", nextVers)
+		}
+		if err := migration(d); err != nil {
 			return errors.Wrapf(err, "migrating to version %d", nextVers)
 		}
 

format_major_version_test.go

@@ -38,11 +38,12 @@ func TestFormatMajorVersionStableValues(t *testing.T) {
 	require.Equal(t, FormatV2BlobFiles, FormatMajorVersion(26))
 	require.Equal(t, FormatBackingValueSize, FormatMajorVersion(27))
 	require.Equal(t, FormatMarkForCompactionInVersionEdit, FormatMajorVersion(28))
+	require.Equal(t, FormatTieredStorage, FormatMajorVersion(29))
 
 	// When we add a new version, we should add a check for the new version above
 	// in addition to updating the expected values below.
-	require.Equal(t, FormatNewest, FormatMajorVersion(28))
-	require.Equal(t, internalFormatNewest, FormatMajorVersion(28))
+	require.Equal(t, FormatNewest, FormatMajorVersion(29))
+	require.Equal(t, internalFormatNewest, FormatMajorVersion(29))
 }
 
 func TestFormatMajorVersion_MigrationDefined(t *testing.T) {
@@ -233,6 +234,7 @@ func TestFormatMajorVersions_TableFormat(t *testing.T) {
 		FormatV2BlobFiles:                           {sstable.TableFormatPebblev1, sstable.TableFormatPebblev7},
 		FormatBackingValueSize:                      {sstable.TableFormatPebblev1, sstable.TableFormatPebblev7},
 		FormatMarkForCompactionInVersionEdit:        {sstable.TableFormatPebblev1, sstable.TableFormatPebblev7},
+		FormatTieredStorage:                         {sstable.TableFormatPebblev1, sstable.TableFormatPebblev8},
 	}
 
 	// Valid versions.
@@ -297,7 +299,7 @@ func TestFormatMajorVersions_MaxTableFormat(t *testing.T) {
 		},
 		{
 			fmv:  FormatNewest,
-			want: sstable.TableFormatPebblev7,
+			want: sstable.TableFormatPebblev8,
 		},
 	}
 	for _, tc := range testCases {

sstable/writer_test.go

@@ -50,6 +50,7 @@ func TestWriter(t *testing.T) {
 		{TableFormat: TableFormatPebblev5, File: "testdata/writer_v5"},
 		{TableFormat: TableFormatPebblev6, File: "testdata/writer_v6"},
 		{TableFormat: TableFormatPebblev7, File: "testdata/writer_v7"},
+		{TableFormat: TableFormatPebblev8, File: "testdata/writer_v8"},
 	}
 	for _, tff := range formatFiles {
 		t.Run(tff.TableFormat.String(), func(t *testing.T) {
@@ -66,6 +67,7 @@ func TestRewriter(t *testing.T) {
 		{TableFormat: TableFormatPebblev5, File: "testdata/rewriter_v5"},
 		{TableFormat: TableFormatPebblev6, File: "testdata/rewriter_v6"},
 		{TableFormat: TableFormatPebblev7, File: "testdata/rewriter_v7"},
+		{TableFormat: TableFormatPebblev8, File: "testdata/writer_v8"},
 	}
 	for _, tff := range formatFiles {
 		t.Run(tff.TableFormat.String(), func(t *testing.T) {

testdata/checkpoint

@@ -100,6 +100,11 @@ sync: db/marker.format-version.000015.028
 close: db/marker.format-version.000015.028
 remove: db/marker.format-version.000014.027
 sync: db
+create: db/marker.format-version.000016.029
+sync: db/marker.format-version.000016.029
+close: db/marker.format-version.000016.029
+remove: db/marker.format-version.000015.028
+sync: db
 get-disk-usage: db
 
 batch db

testdata/checkpoint_shared

@@ -85,6 +85,11 @@ sync: db/marker.format-version.000012.028
 close: db/marker.format-version.000012.028
 remove: db/marker.format-version.000011.027
 sync: db
+create: db/marker.format-version.000013.029
+sync: db/marker.format-version.000013.029
+close: db/marker.format-version.000013.029
+remove: db/marker.format-version.000012.028
+sync: db
 get-disk-usage: db
 create: db/REMOTE-OBJ-CATALOG-000001
 sync: db/REMOTE-OBJ-CATALOG-000001

testdata/event_listener

@@ -123,6 +123,12 @@ close: db/marker.format-version.000015.028
 remove: db/marker.format-version.000014.027
 sync: db
 upgraded to format version: 028
+create: db/marker.format-version.000016.029
+sync: db/marker.format-version.000016.029
+close: db/marker.format-version.000016.029
+remove: db/marker.format-version.000015.028
+sync: db
+upgraded to format version: 029
 get-disk-usage: db
 
 flush

tool/testdata/db_upgrade

@@ -27,7 +27,7 @@ db get foo yellow
 db upgrade foo
 ----
 ----
-Upgrading DB from internal version 16 to 28.
+Upgrading DB from internal version 16 to 29.
 WARNING!!!
 This DB will not be usable with older versions of Pebble!