From 72cf0870502bacbcc5eaea6d873642c230333054 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Tue, 1 Jul 2025 06:26:39 -1000 Subject: [PATCH 1/2] upgrade to pebble v2.0.6 --- bench/store_bench_test.go | 4 ++-- go.mod | 5 +++-- go.sum | 14 ++++++++++++-- metrics/pebble_metrics.go | 2 +- store/pebble/pebble.go | 2 +- store/pebble/vk_merger.go | 2 +- 6 files changed, 20 insertions(+), 9 deletions(-) diff --git a/bench/store_bench_test.go b/bench/store_bench_test.go index ed82d48..4984833 100644 --- a/bench/store_bench_test.go +++ b/bench/store_bench_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" - pb2 "github.com/cockroachdb/pebble" - "github.com/cockroachdb/pebble/bloom" + pb2 "github.com/cockroachdb/pebble/v2" + "github.com/cockroachdb/pebble/v2/bloom" "github.com/ipni/go-indexer-core" "github.com/ipni/go-indexer-core/store/memory" "github.com/ipni/go-indexer-core/store/pebble" diff --git a/go.mod b/go.mod index dc5e9be..643c15f 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ipni/go-indexer-core go 1.24.0 require ( - github.com/cockroachdb/pebble v1.1.5 + github.com/cockroachdb/pebble/v2 v2.0.6 github.com/gammazero/radixtree v0.4.0 github.com/ipfs/go-cid v0.5.0 github.com/ipfs/go-log/v2 v2.8.1 @@ -21,11 +21,12 @@ require ( github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cockroachdb/datadriven v1.0.3-0.20240530155848-7682d40af056 // indirect + github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94 // indirect github.com/cockroachdb/errors v1.11.3 // indirect github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.5 // indirect + github.com/cockroachdb/swiss v0.0.0-20250624142022-d6e517c1d961 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/getsentry/sentry-go v0.27.0 // indirect diff --git a/go.sum b/go.sum index 5b87099..69d12ef 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e h1:ZIWapoIRN1VqT8GR8jAwb1Ie9GyehWjVcGh32Y2MznE= github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f h1:JjxwchlOepwsUWcQwD2mLUAGE9aCp0/ehy6yCHFBOvo= +github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f/go.mod h1:tMDTce/yLLN/SK8gMOxQfnyeMeCg8KGzp0D1cbECEeo= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -9,6 +11,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94 h1:bvJv505UUfjzbaIPdNS4AEkHreDqQk6yuNpsdRHpwFA= +github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac= github.com/cockroachdb/datadriven v1.0.3-0.20240530155848-7682d40af056 h1:slXychO2uDM6hYRu4c0pD0udNI8uObfeKN6UInWViS8= github.com/cockroachdb/datadriven v1.0.3-0.20240530155848-7682d40af056/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= @@ -17,10 +21,14 @@ github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/e github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v1.1.5 h1:5AAWCBWbat0uE0blr8qzufZP5tBjkRyy/jWe1QWLnvw= -github.com/cockroachdb/pebble v1.1.5/go.mod h1:17wO9el1YEigxkP/YtV8NtCivQDgoCyBg5c4VR/eOWo= +github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA= +github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA= +github.com/cockroachdb/pebble/v2 v2.0.6 h1:eL54kX2AKp1ePJ/8vq4IO3xIEPpvVjlSP12dlLYilyE= +github.com/cockroachdb/pebble/v2 v2.0.6/go.mod h1:un1DXG73PKw3F7Ndd30YactyvsFviI9Fuhe0tENdnyA= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/swiss v0.0.0-20250624142022-d6e517c1d961 h1:Nua446ru3juLHLZd4AwKNzClZgL1co3pUPGv3o8FlcA= +github.com/cockroachdb/swiss v0.0.0-20250624142022-d6e517c1d961/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -39,6 +47,8 @@ github.com/gammazero/radixtree v0.4.0 h1:e3TrzDzOvQ8aWwDEZKWNGPQeyS/9yFdaR57yVPO github.com/gammazero/radixtree v0.4.0/go.mod h1:ASdWnk7E8ZsLUjJeJAb4H3YZ0lzlWMGhXGgo8Hw7PCI= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9 h1:r5GgOLGbza2wVHRzK7aAj6lWZjfbAwiu/RDCVOKjRyM= +github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= diff --git a/metrics/pebble_metrics.go b/metrics/pebble_metrics.go index 0fdccbc..45a2e1b 100644 --- a/metrics/pebble_metrics.go +++ b/metrics/pebble_metrics.go @@ -4,7 +4,7 @@ import ( "context" "time" - "github.com/cockroachdb/pebble" + "github.com/cockroachdb/pebble/v2" logging "github.com/ipfs/go-log/v2" "go.opencensus.io/stats" "go.opencensus.io/stats/view" diff --git a/store/pebble/pebble.go b/store/pebble/pebble.go index bf2d46a..dbabcf5 100644 --- a/store/pebble/pebble.go +++ b/store/pebble/pebble.go @@ -8,7 +8,7 @@ import ( "slices" "time" - "github.com/cockroachdb/pebble" + "github.com/cockroachdb/pebble/v2" logging "github.com/ipfs/go-log/v2" "github.com/ipni/go-indexer-core" "github.com/ipni/go-indexer-core/metrics" diff --git a/store/pebble/vk_merger.go b/store/pebble/vk_merger.go index eb5936d..f97f5fa 100644 --- a/store/pebble/vk_merger.go +++ b/store/pebble/vk_merger.go @@ -6,7 +6,7 @@ import ( "io" "slices" - "github.com/cockroachdb/pebble" + "github.com/cockroachdb/pebble/v2" ) const valueKeysMergerName = "indexer.v1.binary.valueKeysMerger" From cd185b84294fe99e204c1e9d907f3e299e53ed51 Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Tue, 9 Sep 2025 11:06:21 -1000 Subject: [PATCH 2/2] upgrade to pebble v2.1.0 --- bench/store_bench_test.go | 22 ++++++++-------------- go.mod | 10 ++++++---- go.sum | 24 ++++++++++++++---------- metrics/pebble_metrics.go | 34 +++++++++++++++++----------------- 4 files changed, 45 insertions(+), 45 deletions(-) diff --git a/bench/store_bench_test.go b/bench/store_bench_test.go index 4984833..2b7ab00 100644 --- a/bench/store_bench_test.go +++ b/bench/store_bench_test.go @@ -85,7 +85,7 @@ func newPebbleSubject(b *testing.B) func() (indexer.Interface, error) { pebbleOpts := &pb2.Options{ BytesPerSync: 10 << 20, // 10 MiB WALBytesPerSync: 10 << 20, // 10 MiB - MaxConcurrentCompactions: func() int { return 10 }, + CompactionConcurrencyRange: func() (int, int) { return 1, 10 }, MemTableSize: 64 << 20, // 64 MiB MemTableStopWritesThreshold: 4, LBaseMaxBytes: 64 << 20, // 64 MiB @@ -96,21 +96,15 @@ func newPebbleSubject(b *testing.B) func() (indexer.Interface, error) { pebbleOpts.Experimental.ReadCompactionRate = 10 << 20 // 20 MiB - const numLevels = 7 - pebbleOpts.Levels = make([]pb2.LevelOptions, numLevels) - for i := 0; i < numLevels; i++ { - l := &pebbleOpts.Levels[i] - l.BlockSize = 32 << 10 // 32 KiB - l.IndexBlockSize = 256 << 10 // 256 KiB - l.FilterPolicy = bloom.FilterPolicy(10) - l.FilterType = pb2.TableFilter - if i > 0 { - l.TargetFileSize = pebbleOpts.Levels[i-1].TargetFileSize * 2 - } - l.EnsureDefaults() + for i := range pebbleOpts.Levels { + pebbleOpts.Levels[i].BlockSize = 32 << 10 // 32 KiB + pebbleOpts.Levels[i].IndexBlockSize = 256 << 10 // 256 KiB + pebbleOpts.Levels[i].FilterPolicy = bloom.FilterPolicy(10) + pebbleOpts.Levels[i].FilterType = pb2.TableFilter } - pebbleOpts.Levels[numLevels-1].FilterPolicy = nil + pebbleOpts.Levels[len(pebbleOpts.Levels)-1].FilterPolicy = nil pebbleOpts.Cache = pb2.NewCache(1 << 30) // 1 GiB + pebbleOpts.EnsureDefaults() return pebble.New(b.TempDir(), pebbleOpts) } diff --git a/go.mod b/go.mod index 643c15f..8dd0061 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ipni/go-indexer-core go 1.24.0 require ( - github.com/cockroachdb/pebble/v2 v2.0.6 + github.com/cockroachdb/pebble/v2 v2.1.0 github.com/gammazero/radixtree v0.4.0 github.com/ipfs/go-cid v0.5.0 github.com/ipfs/go-log/v2 v2.8.1 @@ -18,12 +18,13 @@ require ( ) require ( - github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e // indirect + github.com/DataDog/zstd v1.5.7 // indirect + github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657 // indirect + github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94 // indirect + github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b // indirect github.com/cockroachdb/errors v1.11.3 // indirect - github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/swiss v0.0.0-20250624142022-d6e517c1d961 // indirect @@ -40,6 +41,7 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect diff --git a/go.sum b/go.sum index 69d12ef..80d2ad3 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,11 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e h1:ZIWapoIRN1VqT8GR8jAwb1Ie9GyehWjVcGh32Y2MznE= -github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE= +github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657 h1:8XBWWQD+vFF+JqOsm16t0Kab1a7YWV8+GISVEP8AuZ8= +github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657/go.mod h1:UHGJonU9z4YYGKJxSaC6/TNcLOBptpmM5m2Cksbnw0Y= +github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 h1:bsU8Tzxr/PNz75ayvCnxKZWEYdLMPDkUgticP4a4Bvk= +github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54/go.mod h1:0tr7FllbE9gJkHq7CVeeDDFAFKQVy5RnCSSNBOvdqbc= github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f h1:JjxwchlOepwsUWcQwD2mLUAGE9aCp0/ehy6yCHFBOvo= github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f/go.mod h1:tMDTce/yLLN/SK8gMOxQfnyeMeCg8KGzp0D1cbECEeo= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -11,20 +15,18 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94 h1:bvJv505UUfjzbaIPdNS4AEkHreDqQk6yuNpsdRHpwFA= -github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac= -github.com/cockroachdb/datadriven v1.0.3-0.20240530155848-7682d40af056 h1:slXychO2uDM6hYRu4c0pD0udNI8uObfeKN6UInWViS8= -github.com/cockroachdb/datadriven v1.0.3-0.20240530155848-7682d40af056/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b h1:SHlYZ/bMx7frnmeqCu+xm0TCxXLzX3jQIVuFbnFGtFU= +github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac= +github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5 h1:UycK/E0TkisVrQbSoxvU827FwgBBcZ95nRRmpj/12QI= +github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5/go.mod h1:jsaKMvD3RBCATk1/jbUZM8C9idWBJME9+VRZ5+Liq1g= github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= -github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4= -github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA= -github.com/cockroachdb/pebble/v2 v2.0.6 h1:eL54kX2AKp1ePJ/8vq4IO3xIEPpvVjlSP12dlLYilyE= -github.com/cockroachdb/pebble/v2 v2.0.6/go.mod h1:un1DXG73PKw3F7Ndd30YactyvsFviI9Fuhe0tENdnyA= +github.com/cockroachdb/pebble/v2 v2.1.0 h1:6KZvjSpWcEXZUvlLzTRC7T1A2G7r+bFskIzggklxixo= +github.com/cockroachdb/pebble/v2 v2.1.0/go.mod h1:Aza05DCCc05ghIJZkB4Q/axv/JK9wx5cFwWcnhG0eGw= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/swiss v0.0.0-20250624142022-d6e517c1d961 h1:Nua446ru3juLHLZd4AwKNzClZgL1co3pUPGv3o8FlcA= @@ -105,6 +107,8 @@ github.com/libp2p/go-libp2p v0.43.0 h1:b2bg2cRNmY4HpLK8VHYQXLX2d3iND95OjodLFymvq github.com/libp2p/go-libp2p v0.43.0/go.mod h1:IiSqAXDyP2sWH+J2gs43pNmB/y4FOi2XQPbsb+8qvzc= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 h1:0lgqHvJWHLGW5TuObJrfyEi6+ASTKDBWikGvPqy9Yiw= +github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= diff --git a/metrics/pebble_metrics.go b/metrics/pebble_metrics.go index 45a2e1b..10ff24c 100644 --- a/metrics/pebble_metrics.go +++ b/metrics/pebble_metrics.go @@ -62,11 +62,11 @@ var ( " compaction. Such files are compacted in a rewrite compaction"+ " when no other compactions are picked.", stats.UnitDimensionless) - // l0NumFiles is the total number of files in L0. The number of L0 files should not be in the high thousands. - // High values indicate heavy write load that is causing accumulation of files in level 0. These files are not + // l0TablesCount is the total count of sstables in L0. The number of L0 sstables should not be in the high thousands. + // High values indicate heavy write load that is causing accumulation of sstables in level 0. These sstables are not // being compacted quickly enough to lower levels, resulting in a misshapen LSM. - l0NumFiles = stats.Int64("ipni/pebble/compact_l0_num_files", "The total number of files in L0. The number of L0 files should not be in the high thousands."+ - " High values indicate heavy write load that is causing accumulation of files in level 0. These files are not"+ + l0TablesCount = stats.Int64("ipni/pebble/compact_l0_tables_count", "The total count of sstables in L0. The number of L0 sstables should not be in the high thousands."+ + " High values indicate heavy write load that is causing accumulation of sstables in level 0. These sstables are not"+ " being compacted quickly enough to lower levels, resulting in a misshapen LSM.", stats.UnitDimensionless) ) @@ -120,8 +120,8 @@ var ( Measure: compactMarkedFiles, Aggregation: view.LastValue(), } - l0NumFilesView = &view.View{ - Measure: l0NumFiles, + l0TablesCountView = &view.View{ + Measure: l0TablesCount, Aggregation: view.LastValue(), } ) @@ -139,7 +139,7 @@ var PebbleViews = []*view.View{ compactInProgressBytesView, compactNumInProgressView, compactMarkedFilesView, - l0NumFilesView, + l0TablesCountView, } // ObservePebbleMetrics is used to periodically report metrics from the pebble database @@ -176,7 +176,7 @@ func reportMetrics(m *pebble.Metrics) { compactInProgressBytes.M(int64(m.Compact.InProgressBytes)), compactNumInProgress.M(int64(m.Compact.NumInProgress)), compactMarkedFiles.M(int64(m.Compact.MarkedFiles)), - l0NumFiles.M(int64(m.Levels[0].NumFiles))) + l0TablesCount.M(int64(m.Levels[0].TablesCount))) // Block cache metrics stats.RecordWithOptions(ctx, @@ -195,20 +195,20 @@ func reportMetrics(m *pebble.Metrics) { stats.WithMeasurements(cacheMisses.M(m.BlockCache.Misses)), stats.WithTags(tag.Insert(cacheTag, "block"))) - // Table cache metrics + // File cache metrics stats.RecordWithOptions(ctx, - stats.WithMeasurements(cacheCount.M(m.TableCache.Count)), - stats.WithTags(tag.Insert(cacheTag, "table"))) + stats.WithMeasurements(cacheCount.M(m.FileCache.TableCount+m.FileCache.BlobFileCount)), + stats.WithTags(tag.Insert(cacheTag, "file"))) stats.RecordWithOptions(ctx, - stats.WithMeasurements(cacheSize.M(m.TableCache.Size)), - stats.WithTags(tag.Insert(cacheTag, "table"))) + stats.WithMeasurements(cacheSize.M(m.FileCache.Size)), + stats.WithTags(tag.Insert(cacheTag, "file"))) stats.RecordWithOptions(ctx, - stats.WithMeasurements(cacheHits.M(m.TableCache.Hits)), - stats.WithTags(tag.Insert(cacheTag, "table"))) + stats.WithMeasurements(cacheHits.M(m.FileCache.Hits)), + stats.WithTags(tag.Insert(cacheTag, "file"))) stats.RecordWithOptions(ctx, - stats.WithMeasurements(cacheMisses.M(m.TableCache.Misses)), - stats.WithTags(tag.Insert(cacheTag, "table"))) + stats.WithMeasurements(cacheMisses.M(m.FileCache.Misses)), + stats.WithTags(tag.Insert(cacheTag, "file"))) }