upgrade to pebble v2.1.0

bench/store_bench_test.go

@@ -6,8 +6,8 @@ import (
 	"testing"
 	"time"
 
-	pb2 "github.com/cockroachdb/pebble"
-	"github.com/cockroachdb/pebble/bloom"
+	pb2 "github.com/cockroachdb/pebble/v2"
+	"github.com/cockroachdb/pebble/v2/bloom"
 	"github.com/ipni/go-indexer-core"
 	"github.com/ipni/go-indexer-core/store/memory"
 	"github.com/ipni/go-indexer-core/store/pebble"
@@ -85,7 +85,7 @@ func newPebbleSubject(b *testing.B) func() (indexer.Interface, error) {
 		pebbleOpts := &pb2.Options{
 			BytesPerSync:                10 << 20, // 10 MiB
 			WALBytesPerSync:             10 << 20, // 10 MiB
-			MaxConcurrentCompactions:    func() int { return 10 },
+			CompactionConcurrencyRange:  func() (int, int) { return 1, 10 },
 			MemTableSize:                64 << 20, // 64 MiB
 			MemTableStopWritesThreshold: 4,
 			LBaseMaxBytes:               64 << 20, // 64 MiB
@@ -96,21 +96,15 @@ func newPebbleSubject(b *testing.B) func() (indexer.Interface, error) {
 
 		pebbleOpts.Experimental.ReadCompactionRate = 10 << 20 // 20 MiB
 
-		const numLevels = 7
-		pebbleOpts.Levels = make([]pb2.LevelOptions, numLevels)
-		for i := 0; i < numLevels; i++ {
-			l := &pebbleOpts.Levels[i]
-			l.BlockSize = 32 << 10       // 32 KiB
-			l.IndexBlockSize = 256 << 10 // 256 KiB
-			l.FilterPolicy = bloom.FilterPolicy(10)
-			l.FilterType = pb2.TableFilter
-			if i > 0 {
-				l.TargetFileSize = pebbleOpts.Levels[i-1].TargetFileSize * 2
-			}
-			l.EnsureDefaults()
+		for i := range pebbleOpts.Levels {
+			pebbleOpts.Levels[i].BlockSize = 32 << 10       // 32 KiB
+			pebbleOpts.Levels[i].IndexBlockSize = 256 << 10 // 256 KiB
+			pebbleOpts.Levels[i].FilterPolicy = bloom.FilterPolicy(10)
+			pebbleOpts.Levels[i].FilterType = pb2.TableFilter
 		}
-		pebbleOpts.Levels[numLevels-1].FilterPolicy = nil
+		pebbleOpts.Levels[len(pebbleOpts.Levels)-1].FilterPolicy = nil
 		pebbleOpts.Cache = pb2.NewCache(1 << 30) // 1 GiB
+		pebbleOpts.EnsureDefaults()
 
 		return pebble.New(b.TempDir(), pebbleOpts)
 	}

go.mod

@@ -3,7 +3,7 @@ module github.com/ipni/go-indexer-core
 go 1.24.0
 
 require (
-	github.com/cockroachdb/pebble v1.1.5
+	github.com/cockroachdb/pebble/v2 v2.1.0
 	github.com/gammazero/radixtree v0.4.0
 	github.com/ipfs/go-cid v0.5.0
 	github.com/ipfs/go-log/v2 v2.8.1
@@ -18,14 +18,16 @@ require (
 )
 
 require (
-	github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e // indirect
+	github.com/DataDog/zstd v1.5.7 // indirect
+	github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657 // indirect
+	github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/cockroachdb/datadriven v1.0.3-0.20240530155848-7682d40af056 // indirect
+	github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b // indirect
 	github.com/cockroachdb/errors v1.11.3 // indirect
-	github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
 	github.com/cockroachdb/redact v1.1.5 // indirect
+	github.com/cockroachdb/swiss v0.0.0-20250624142022-d6e517c1d961 // indirect
 	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect
 	github.com/getsentry/sentry-go v0.27.0 // indirect
@@ -39,6 +41,7 @@ require (
 	github.com/kr/text v0.2.0 // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 // indirect
 	github.com/minio/sha256-simd v1.0.1 // indirect
 	github.com/multiformats/go-base32 v0.1.0 // indirect
 	github.com/multiformats/go-base36 v0.2.0 // indirect

go.sum

@@ -1,26 +1,36 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e h1:ZIWapoIRN1VqT8GR8jAwb1Ie9GyehWjVcGh32Y2MznE=
-github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
+github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE=
+github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
+github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657 h1:8XBWWQD+vFF+JqOsm16t0Kab1a7YWV8+GISVEP8AuZ8=
+github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657/go.mod h1:UHGJonU9z4YYGKJxSaC6/TNcLOBptpmM5m2Cksbnw0Y=
+github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 h1:bsU8Tzxr/PNz75ayvCnxKZWEYdLMPDkUgticP4a4Bvk=
+github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54/go.mod h1:0tr7FllbE9gJkHq7CVeeDDFAFKQVy5RnCSSNBOvdqbc=
+github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f h1:JjxwchlOepwsUWcQwD2mLUAGE9aCp0/ehy6yCHFBOvo=
+github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f/go.mod h1:tMDTce/yLLN/SK8gMOxQfnyeMeCg8KGzp0D1cbECEeo=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cockroachdb/datadriven v1.0.3-0.20240530155848-7682d40af056 h1:slXychO2uDM6hYRu4c0pD0udNI8uObfeKN6UInWViS8=
-github.com/cockroachdb/datadriven v1.0.3-0.20240530155848-7682d40af056/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
+github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b h1:SHlYZ/bMx7frnmeqCu+xm0TCxXLzX3jQIVuFbnFGtFU=
+github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac=
+github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5 h1:UycK/E0TkisVrQbSoxvU827FwgBBcZ95nRRmpj/12QI=
+github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5/go.mod h1:jsaKMvD3RBCATk1/jbUZM8C9idWBJME9+VRZ5+Liq1g=
 github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
 github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8=
-github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4=
-github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
-github.com/cockroachdb/pebble v1.1.5 h1:5AAWCBWbat0uE0blr8qzufZP5tBjkRyy/jWe1QWLnvw=
-github.com/cockroachdb/pebble v1.1.5/go.mod h1:17wO9el1YEigxkP/YtV8NtCivQDgoCyBg5c4VR/eOWo=
+github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA=
+github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA=
+github.com/cockroachdb/pebble/v2 v2.1.0 h1:6KZvjSpWcEXZUvlLzTRC7T1A2G7r+bFskIzggklxixo=
+github.com/cockroachdb/pebble/v2 v2.1.0/go.mod h1:Aza05DCCc05ghIJZkB4Q/axv/JK9wx5cFwWcnhG0eGw=
 github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
 github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
+github.com/cockroachdb/swiss v0.0.0-20250624142022-d6e517c1d961 h1:Nua446ru3juLHLZd4AwKNzClZgL1co3pUPGv3o8FlcA=
+github.com/cockroachdb/swiss v0.0.0-20250624142022-d6e517c1d961/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg=
 github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
 github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@@ -39,6 +49,8 @@ github.com/gammazero/radixtree v0.4.0 h1:e3TrzDzOvQ8aWwDEZKWNGPQeyS/9yFdaR57yVPO
 github.com/gammazero/radixtree v0.4.0/go.mod h1:ASdWnk7E8ZsLUjJeJAb4H3YZ0lzlWMGhXGgo8Hw7PCI=
 github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
 github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
+github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9 h1:r5GgOLGbza2wVHRzK7aAj6lWZjfbAwiu/RDCVOKjRyM=
+github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
 github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -95,6 +107,8 @@ github.com/libp2p/go-libp2p v0.43.0 h1:b2bg2cRNmY4HpLK8VHYQXLX2d3iND95OjodLFymvq
 github.com/libp2p/go-libp2p v0.43.0/go.mod h1:IiSqAXDyP2sWH+J2gs43pNmB/y4FOi2XQPbsb+8qvzc=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 h1:0lgqHvJWHLGW5TuObJrfyEi6+ASTKDBWikGvPqy9Yiw=
+github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec=
 github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
 github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
 github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=

metrics/pebble_metrics.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"time"
 
-	"github.com/cockroachdb/pebble"
+	"github.com/cockroachdb/pebble/v2"
 	logging "github.com/ipfs/go-log/v2"
 	"go.opencensus.io/stats"
 	"go.opencensus.io/stats/view"
@@ -62,11 +62,11 @@ var (
 		" compaction. Such files are compacted in a rewrite compaction"+
 		" when no other compactions are picked.", stats.UnitDimensionless)
 
-	// l0NumFiles is the total number of files in L0. The number of L0 files should not be in the high thousands.
-	// High values indicate heavy write load that is causing accumulation of files in level 0. These files are not
+	// l0TablesCount is the total count of sstables in L0. The number of L0 sstables should not be in the high thousands.
+	// High values indicate heavy write load that is causing accumulation of sstables in level 0. These sstables are not
 	// being compacted quickly enough to lower levels, resulting in a misshapen LSM.
-	l0NumFiles = stats.Int64("ipni/pebble/compact_l0_num_files", "The total number of files in L0. The number of L0 files should not be in the high thousands."+
-		" High values indicate heavy write load that is causing accumulation of files in level 0. These files are not"+
+	l0TablesCount = stats.Int64("ipni/pebble/compact_l0_tables_count", "The total count of sstables in L0. The number of L0 sstables should not be in the high thousands."+
+		" High values indicate heavy write load that is causing accumulation of sstables in level 0. These sstables are not"+
 		" being compacted quickly enough to lower levels, resulting in a misshapen LSM.", stats.UnitDimensionless)
 )
 
@@ -120,8 +120,8 @@ var (
 		Measure:     compactMarkedFiles,
 		Aggregation: view.LastValue(),
 	}
-	l0NumFilesView = &view.View{
-		Measure:     l0NumFiles,
+	l0TablesCountView = &view.View{
+		Measure:     l0TablesCount,
 		Aggregation: view.LastValue(),
 	}
 )
@@ -139,7 +139,7 @@ var PebbleViews = []*view.View{
 	compactInProgressBytesView,
 	compactNumInProgressView,
 	compactMarkedFilesView,
-	l0NumFilesView,
+	l0TablesCountView,
 }
 
 // ObservePebbleMetrics is used to periodically report metrics from the pebble database
@@ -176,7 +176,7 @@ func reportMetrics(m *pebble.Metrics) {
 		compactInProgressBytes.M(int64(m.Compact.InProgressBytes)),
 		compactNumInProgress.M(int64(m.Compact.NumInProgress)),
 		compactMarkedFiles.M(int64(m.Compact.MarkedFiles)),
-		l0NumFiles.M(int64(m.Levels[0].NumFiles)))
+		l0TablesCount.M(int64(m.Levels[0].TablesCount)))
 
 	// Block cache metrics
 	stats.RecordWithOptions(ctx,
@@ -195,20 +195,20 @@ func reportMetrics(m *pebble.Metrics) {
 		stats.WithMeasurements(cacheMisses.M(m.BlockCache.Misses)),
 		stats.WithTags(tag.Insert(cacheTag, "block")))
 
-	// Table cache metrics
+	// File cache metrics
 	stats.RecordWithOptions(ctx,
-		stats.WithMeasurements(cacheCount.M(m.TableCache.Count)),
-		stats.WithTags(tag.Insert(cacheTag, "table")))
+		stats.WithMeasurements(cacheCount.M(m.FileCache.TableCount+m.FileCache.BlobFileCount)),
+		stats.WithTags(tag.Insert(cacheTag, "file")))
 
 	stats.RecordWithOptions(ctx,
-		stats.WithMeasurements(cacheSize.M(m.TableCache.Size)),
-		stats.WithTags(tag.Insert(cacheTag, "table")))
+		stats.WithMeasurements(cacheSize.M(m.FileCache.Size)),
+		stats.WithTags(tag.Insert(cacheTag, "file")))
 
 	stats.RecordWithOptions(ctx,
-		stats.WithMeasurements(cacheHits.M(m.TableCache.Hits)),
-		stats.WithTags(tag.Insert(cacheTag, "table")))
+		stats.WithMeasurements(cacheHits.M(m.FileCache.Hits)),
+		stats.WithTags(tag.Insert(cacheTag, "file")))
 
 	stats.RecordWithOptions(ctx,
-		stats.WithMeasurements(cacheMisses.M(m.TableCache.Misses)),
-		stats.WithTags(tag.Insert(cacheTag, "table")))
+		stats.WithMeasurements(cacheMisses.M(m.FileCache.Misses)),
+		stats.WithTags(tag.Insert(cacheTag, "file")))
 }

store/pebble/pebble.go

@@ -8,7 +8,7 @@ import (
 	"slices"
 	"time"
 
-	"github.com/cockroachdb/pebble"
+	"github.com/cockroachdb/pebble/v2"
 	logging "github.com/ipfs/go-log/v2"
 	"github.com/ipni/go-indexer-core"
 	"github.com/ipni/go-indexer-core/metrics"

store/pebble/vk_merger.go

@@ -6,7 +6,7 @@ import (
 	"io"
 	"slices"
 
-	"github.com/cockroachdb/pebble"
+	"github.com/cockroachdb/pebble/v2"
 )
 
 const valueKeysMergerName = "indexer.v1.binary.valueKeysMerger"