Skip to content

Commit a4e8d27

Browse files
craig[bot]RaduBerinde
craig[bot]
and
RaduBerinde
committed
Merge #156747
156747: go.mod: bump Pebble to 951fae603dfe r=RaduBerinde a=RaduBerinde Update code to use the new metrics. EnvStats now includes the blob file count (see #156746). Changes: * [`951fae60`](cockroachdb/pebble@951fae60) metrics: redesign disk usage metrics * [`9e6937d2`](cockroachdb/pebble@9e6937d2) db: add validation for WALFailover options Release note: none. Epic: none. Co-authored-by: Radu Berinde <[email protected]>
2 parents fa5512c + 71d7360 commit a4e8d27

File tree

8 files changed

+29
-21
lines changed

8 files changed

+29
-21
lines changed

‎DEPS.bzl‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1834,10 +1834,10 @@ def go_deps():
18341834
patches = [
18351835
"@com_github_cockroachdb_cockroach//build/patches:com_github_cockroachdb_pebble.patch",
18361836
],
1837-
sha256 = "9b2a84419f999a1a0b37fccb761dca75dd2cd2ec4fd4875dbe900c9d14e12b79",
1838-
strip_prefix = "github.com/cockroachdb/[email protected]20251029205532-e8d7ecb38559",
1837+
sha256 = "0ffbfa9854e45c7fa775c37af3e80f439c078d28ffa348a6a2bdb9af4e10ad37",
1838+
strip_prefix = "github.com/cockroachdb/[email protected]20251102183433-951fae603dfe",
18391839
urls = [
1840-
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20251029205532-e8d7ecb38559.zip",
1840+
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20251102183433-951fae603dfe.zip",
18411841
],
18421842
)
18431843
go_repository(

‎build/bazelutil/distdir_files.bzl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -352,7 +352,7 @@ DISTDIR_FILES = {
352352
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/gostdlib/com_github_cockroachdb_gostdlib-v1.19.0.zip": "c4d516bcfe8c07b6fc09b8a9a07a95065b36c2855627cb3514e40c98f872b69e",
353353
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/logtags/com_github_cockroachdb_logtags-v0.0.0-20241215232642-bb51bb14a506.zip": "920068af09e3846d9ebb4e4a7787ff1dd10f3989c5f940ad861b0f6a9f824f6e",
354354
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/metamorphic/com_github_cockroachdb_metamorphic-v0.0.0-20231108215700-4ba948b56895.zip": "28c8cf42192951b69378cf537be5a9a43f2aeb35542908cc4fe5f689505853ea",
355-
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20251029205532-e8d7ecb38559.zip": "9b2a84419f999a1a0b37fccb761dca75dd2cd2ec4fd4875dbe900c9d14e12b79",
355+
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20251102183433-951fae603dfe.zip": "0ffbfa9854e45c7fa775c37af3e80f439c078d28ffa348a6a2bdb9af4e10ad37",
356356
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/redact/com_github_cockroachdb_redact-v1.1.6.zip": "018eccb5fb9ca52d43ec9eaf213539d01c1f2b94e0e822406ebfb2e9321ef6cf",
357357
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/returncheck/com_github_cockroachdb_returncheck-v0.0.0-20200612231554-92cdbca611dd.zip": "ce92ba4352deec995b1f2eecf16eba7f5d51f5aa245a1c362dfe24c83d31f82b",
358358
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/stress/com_github_cockroachdb_stress-v0.0.0-20220803192808-1806698b1b7b.zip": "3fda531795c600daf25532a4f98be2a1335cd1e5e182c72789bca79f5f69fcc1",

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ require (
137137
github.com/cockroachdb/errors v1.12.1-0.20251010171200-64801262cd6f
138138
github.com/cockroachdb/gostdlib v1.19.0
139139
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506
140-
github.com/cockroachdb/pebble v0.0.0-20251029205532-e8d7ecb38559
140+
github.com/cockroachdb/pebble v0.0.0-20251102183433-951fae603dfe
141141
github.com/cockroachdb/redact v1.1.6
142142
github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd
143143
github.com/cockroachdb/tokenbucket v0.0.0-20250429170803-42689b6311bb

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -572,8 +572,8 @@ github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 h1:ASDL+UJcILM
572572
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506/go.mod h1:Mw7HqKr2kdtu6aYGn3tPmAftiP3QPX63LdK/zcariIo=
573573
github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA=
574574
github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA=
575-
github.com/cockroachdb/pebble v0.0.0-20251029205532-e8d7ecb38559 h1:ov6vATG9hZ7bwMYnrrmRrSRvaqAzCGlrNMjOqc8ivzA=
576-
github.com/cockroachdb/pebble v0.0.0-20251029205532-e8d7ecb38559/go.mod h1:GflN9NRwberlF+gFRJd6r6jrUYdWR/QD+gtgtH0LRK8=
575+
github.com/cockroachdb/pebble v0.0.0-20251102183433-951fae603dfe h1:HUrVJv2RUIT2L9Gl4STK1rS0AvlubDvHgreuc3f+Boc=
576+
github.com/cockroachdb/pebble v0.0.0-20251102183433-951fae603dfe/go.mod h1:GflN9NRwberlF+gFRJd6r6jrUYdWR/QD+gtgtH0LRK8=
577577
github.com/cockroachdb/redact v1.1.6 h1:zXJBwDZ84xJNlHl1rMyCojqyIxv+7YUpQiJLQ7n4314=
578578
github.com/cockroachdb/redact v1.1.6/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
579579
github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd h1:KFOt5I9nEKZgCnOSmy8r4Oykh8BYQO8bFOTgHDS8YZA=

‎pkg/kv/kvserver/metrics.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4331,8 +4331,9 @@ func (sm *StoreMetrics) updateEngineMetrics(m storage.Metrics) {
43314331
sm.IngestCount.Update(int64(m.Ingest.Count))
43324332
sm.ValueSeparationBytesReferenced.Update(int64(m.BlobFiles.ReferencedValueSize))
43334333
sm.ValueSeparationBytesUnreferenced.Update(int64(m.BlobFiles.ValueSize - m.BlobFiles.ReferencedValueSize))
4334-
sm.ValueSeparationBlobFileCount.Update(int64(m.BlobFiles.Live.All.Count))
4335-
sm.ValueSeparationBlobFileSize.Update(int64(m.BlobFiles.Live.All.Bytes))
4334+
liveBlobFiles := m.BlobFiles.Live.Total()
4335+
sm.ValueSeparationBlobFileCount.Update(int64(liveBlobFiles.Count))
4336+
sm.ValueSeparationBlobFileSize.Update(int64(liveBlobFiles.Bytes))
43364337
sm.ValueSeparationValueRetrievalCount.Update(int64(m.Iterator.ValueRetrievalCount))
43374338
// NB: `UpdateIfHigher` is used here since there is a race in pebble where
43384339
// sometimes the WAL is rotated but metrics are retrieved prior to the update
@@ -4350,7 +4351,7 @@ func (sm *StoreMetrics) updateEngineMetrics(m storage.Metrics) {
43504351
sm.BatchCommitL0StallDuration.Update(int64(m.BatchCommitStats.L0ReadAmpWriteStallDuration))
43514352
sm.BatchCommitWALRotWaitDuration.Update(int64(m.BatchCommitStats.WALRotationDuration))
43524353
sm.BatchCommitCommitWaitDuration.Update(int64(m.BatchCommitStats.CommitWaitDuration))
4353-
sm.SSTableZombieBytes.Update(int64(m.Table.Zombie.All.Bytes))
4354+
sm.SSTableZombieBytes.Update(int64(m.Table.Physical.Zombie.Total().Bytes))
43544355
remoteTables := m.RemoteTablesTotal()
43554356
sm.SSTableRemoteBytes.Update(int64(remoteTables.Bytes))
43564357
sm.SSTableRemoteCount.Update(int64(remoteTables.Count))

‎pkg/storage/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ go_library(
9999
"@com_github_cockroachdb_pebble//batchrepr",
100100
"@com_github_cockroachdb_pebble//bloom",
101101
"@com_github_cockroachdb_pebble//cockroachkvs",
102+
"@com_github_cockroachdb_pebble//metrics",
102103
"@com_github_cockroachdb_pebble//objstorage",
103104
"@com_github_cockroachdb_pebble//objstorage/objstorageprovider",
104105
"@com_github_cockroachdb_pebble//objstorage/remote",

‎pkg/storage/engine.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1401,10 +1401,10 @@ func (m *Metrics) AsStoreStatsEvent() eventpb.StoreStats {
14011401
WalPhysicalSize: m.WAL.PhysicalSize,
14021402
WalBytesIn: m.WAL.BytesIn,
14031403
WalBytesWritten: m.WAL.BytesWritten,
1404-
TableObsoleteCount: int64(m.Table.Obsolete.All.Count),
1405-
TableObsoleteSize: m.Table.Obsolete.All.Bytes,
1406-
TableZombieCount: int64(m.Table.Zombie.All.Count),
1407-
TableZombieSize: m.Table.Zombie.All.Bytes,
1404+
TableObsoleteCount: int64(m.Table.Physical.Obsolete.Total().Count),
1405+
TableObsoleteSize: m.Table.Physical.Obsolete.Total().Bytes,
1406+
TableZombieCount: int64(m.Table.Physical.Zombie.Total().Count),
1407+
TableZombieSize: m.Table.Physical.Zombie.Total().Bytes,
14081408
RangeKeySetsCount: m.Keys.RangeKeySetsCount,
14091409
}
14101410
e.CacheHits, e.CacheMisses = m.BlockCache.HitsAndMisses.Aggregate()

‎pkg/storage/pebble.go‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ import (
5050
"github.com/cockroachdb/pebble"
5151
"github.com/cockroachdb/pebble/bloom"
5252
"github.com/cockroachdb/pebble/cockroachkvs"
53+
"github.com/cockroachdb/pebble/metrics"
5354
"github.com/cockroachdb/pebble/objstorage/objstorageprovider"
5455
"github.com/cockroachdb/pebble/objstorage/remote"
5556
"github.com/cockroachdb/pebble/rangekey"
@@ -2074,13 +2075,18 @@ func (p *Pebble) GetEnvStats() (*fs.EnvStats, error) {
20742075
}
20752076

20762077
m := p.db.Metrics()
2077-
stats.TotalFiles = 3 /* CURRENT, MANIFEST, OPTIONS */
2078-
stats.TotalFiles += uint64(m.WAL.Files) + m.Table.Zombie.All.Count + uint64(m.WAL.ObsoleteFiles) + m.Table.Obsolete.All.Count
2079-
stats.TotalBytes = m.WAL.Size + m.Table.Zombie.All.Bytes + m.Table.Obsolete.All.Bytes
2080-
for _, l := range m.Levels {
2081-
stats.TotalFiles += l.Tables.Count
2082-
stats.TotalBytes += l.Tables.Bytes
2083-
}
2078+
var cs metrics.CountAndSize
2079+
cs.Count += 3 /* CURRENT, MANIFEST, OPTIONS */
2080+
cs.Count += uint64(m.WAL.Files + m.WAL.ObsoleteFiles)
2081+
cs.Bytes += m.WAL.Size
2082+
cs.Accumulate(m.Table.Physical.Live.Total())
2083+
cs.Accumulate(m.Table.Physical.Zombie.Total())
2084+
cs.Accumulate(m.Table.Physical.Obsolete.Total())
2085+
cs.Accumulate(m.BlobFiles.Live.Total())
2086+
cs.Accumulate(m.BlobFiles.Zombie.Total())
2087+
cs.Accumulate(m.BlobFiles.Obsolete.Total())
2088+
stats.TotalFiles = cs.Count
2089+
stats.TotalBytes = cs.Bytes
20842090

20852091
sstSizes := make(map[pebble.TableNum]uint64)
20862092
sstInfos, err := p.db.SSTables()

0 commit comments

Comments
 (0)