Skip to content

Commit 5ccd93e

Browse files
craig[bot]RaduBerinde
andcommitted
Merge #148117
148117: storage: bump Pebble and use fastest profile for default compression r=RaduBerinde a=RaduBerinde #### go.mod: bump Pebble to 24730ad60ca6 Changes: * [`24730ad6`](cockroachdb/pebble@24730ad6) reduce TestFlushDelayStress runs in slow builds * [`a277627d`](cockroachdb/pebble@a277627d) db.ScanStatistics: sanity check LimitBytesPerSecond * [`dc175270`](cockroachdb/pebble@dc175270) block: add FastestCompression profile * [`819cf246`](cockroachdb/pebble@819cf246) manifest: fix bug in accumulating blob file deletions * [`ea3743eb`](cockroachdb/pebble@ea3743eb) db: remove format check for shared objects * [`ace1ff84`](cockroachdb/pebble@ace1ff84) scripts: run-crossversion-meta fix aliasing of test binaries * [`df305e42`](cockroachdb/pebble@df305e42) manifest: include L0 files in output when L0Sublevels unpopulated * [`37926692`](cockroachdb/pebble@37926692) *: prefix BlobFileIDs with a 'B' * [`6715148f`](cockroachdb/pebble@6715148f) manifest: explicitly encode BlobFileID in manifest * [`491b500c`](cockroachdb/pebble@491b500c) metamorphic: fix WAL recovery dirs logic * [`e14f6834`](cockroachdb/pebble@e14f6834) metamorphic: support --initial-state with --try-to-reduce * [`ac01b429`](cockroachdb/pebble@ac01b429) metamorphic: support --initial-state with --run/compare * [`433156b4`](cockroachdb/pebble@433156b4) db: create shared objects only once the format is new enough * [`bcf89f7e`](cockroachdb/pebble@bcf89f7e) crossversion: increase op timeout Release note: none. Fixes: #147740 #### storage: use fastest profile for default compression This change fixes the issue of having different cluster setting defaults on different platforms, which is problematic for auto-generated documentation. Epic: none Release note: None Co-authored-by: Radu Berinde <[email protected]>
2 parents 2c5c35a + 043ba64 commit 5ccd93e

File tree

7 files changed

+27
-34
lines changed

7 files changed

+27
-34
lines changed

DEPS.bzl

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1885,10 +1885,10 @@ def go_deps():
18851885
patches = [
18861886
"@com_github_cockroachdb_cockroach//build/patches:com_github_cockroachdb_pebble.patch",
18871887
],
1888-
sha256 = "aa3192e19d3a56ab0eebb4bc6d2950dd392daa3f6aafdccba8ec69e75c478ef3",
1889-
strip_prefix = "github.com/cockroachdb/[email protected]20250607160804-056234f760e0",
1888+
sha256 = "66a7caf0461fd298e3208b6ef9d225b956fbd997eb086d1c464c743f3204dd44",
1889+
strip_prefix = "github.com/cockroachdb/[email protected]20250610195649-24730ad60ca6",
18901890
urls = [
1891-
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250607160804-056234f760e0.zip",
1891+
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250610195649-24730ad60ca6.zip",
18921892
],
18931893
)
18941894
go_repository(

build/bazelutil/distdir_files.bzl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -366,7 +366,7 @@ DISTDIR_FILES = {
366366
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/gostdlib/com_github_cockroachdb_gostdlib-v1.19.0.zip": "c4d516bcfe8c07b6fc09b8a9a07a95065b36c2855627cb3514e40c98f872b69e",
367367
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/logtags/com_github_cockroachdb_logtags-v0.0.0-20241215232642-bb51bb14a506.zip": "920068af09e3846d9ebb4e4a7787ff1dd10f3989c5f940ad861b0f6a9f824f6e",
368368
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/metamorphic/com_github_cockroachdb_metamorphic-v0.0.0-20231108215700-4ba948b56895.zip": "28c8cf42192951b69378cf537be5a9a43f2aeb35542908cc4fe5f689505853ea",
369-
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250607160804-056234f760e0.zip": "aa3192e19d3a56ab0eebb4bc6d2950dd392daa3f6aafdccba8ec69e75c478ef3",
369+
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250610195649-24730ad60ca6.zip": "66a7caf0461fd298e3208b6ef9d225b956fbd997eb086d1c464c743f3204dd44",
370370
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/redact/com_github_cockroachdb_redact-v1.1.6.zip": "018eccb5fb9ca52d43ec9eaf213539d01c1f2b94e0e822406ebfb2e9321ef6cf",
371371
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/returncheck/com_github_cockroachdb_returncheck-v0.0.0-20200612231554-92cdbca611dd.zip": "ce92ba4352deec995b1f2eecf16eba7f5d51f5aa245a1c362dfe24c83d31f82b",
372372
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/stress/com_github_cockroachdb_stress-v0.0.0-20220803192808-1806698b1b7b.zip": "3fda531795c600daf25532a4f98be2a1335cd1e5e182c72789bca79f5f69fcc1",

docs/generated/settings/settings-for-tenants.txt

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -401,9 +401,9 @@ storage.delete_compaction_excise.enabled boolean true set to false to direct Peb
401401
storage.ingestion.value_blocks.enabled boolean true set to true to enable writing of value blocks in ingestion sstables application
402402
storage.max_sync_duration duration 20s maximum duration for disk operations; any operations that take longer than this setting trigger a warning log entry or process crash system-visible
403403
storage.max_sync_duration.fatal.enabled boolean true if true, fatal the process when a disk operation exceeds storage.max_sync_duration application
404-
storage.sstable.compression_algorithm enumeration minlz determines the compression algorithm to use when compressing sstable data blocks for use in a Pebble store; [snappy = 1, zstd = 2, none = 3, minlz = 4] system-visible
405-
storage.sstable.compression_algorithm_backup_storage enumeration minlz determines the compression algorithm to use when compressing sstable data blocks for backup row data storage; [snappy = 1, zstd = 2, none = 3, minlz = 4] system-visible
406-
storage.sstable.compression_algorithm_backup_transport enumeration minlz determines the compression algorithm to use when compressing sstable data blocks for backup transport; [snappy = 1, zstd = 2, none = 3, minlz = 4] system-visible
404+
storage.sstable.compression_algorithm enumeration fastest determines the compression algorithm to use when compressing sstable data blocks for use in a Pebble store; [snappy = 1, zstd = 2, none = 3, minlz = 4, fastest = 5] system-visible
405+
storage.sstable.compression_algorithm_backup_storage enumeration fastest determines the compression algorithm to use when compressing sstable data blocks for backup row data storage; [snappy = 1, zstd = 2, none = 3, minlz = 4, fastest = 5] system-visible
406+
storage.sstable.compression_algorithm_backup_transport enumeration fastest determines the compression algorithm to use when compressing sstable data blocks for backup transport; [snappy = 1, zstd = 2, none = 3, minlz = 4, fastest = 5] system-visible
407407
timeseries.storage.resolution_10s.ttl duration 240h0m0s the maximum age of time series data stored at the 10 second resolution. Data older than this is subject to rollup and deletion. system-visible
408408
timeseries.storage.resolution_30m.ttl duration 2160h0m0s the maximum age of time series data stored at the 30 minute resolution. Data older than this is subject to deletion. system-visible
409409
trace.debug_http_endpoint.enabled (alias: trace.debug.enable) boolean false if set, traces for recent requests can be seen at https://<ui>/debug/requests application

docs/generated/settings/settings.html

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -357,9 +357,9 @@
357357
<tr><td><div id="setting-storage-ingestion-value-blocks-enabled" class="anchored"><code>storage.ingestion.value_blocks.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>set to true to enable writing of value blocks in ingestion sstables</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
358358
<tr><td><div id="setting-storage-max-sync-duration" class="anchored"><code>storage.max_sync_duration</code></div></td><td>duration</td><td><code>20s</code></td><td>maximum duration for disk operations; any operations that take longer than this setting trigger a warning log entry or process crash</td><td>Dedicated/Self-hosted (read-write); Serverless (read-only)</td></tr>
359359
<tr><td><div id="setting-storage-max-sync-duration-fatal-enabled" class="anchored"><code>storage.max_sync_duration.fatal.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>if true, fatal the process when a disk operation exceeds storage.max_sync_duration</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
360-
<tr><td><div id="setting-storage-sstable-compression-algorithm" class="anchored"><code>storage.sstable.compression_algorithm</code></div></td><td>enumeration</td><td><code>minlz</code></td><td>determines the compression algorithm to use when compressing sstable data blocks for use in a Pebble store; [snappy = 1, zstd = 2, none = 3, minlz = 4]</td><td>Dedicated/Self-hosted (read-write); Serverless (read-only)</td></tr>
361-
<tr><td><div id="setting-storage-sstable-compression-algorithm-backup-storage" class="anchored"><code>storage.sstable.compression_algorithm_backup_storage</code></div></td><td>enumeration</td><td><code>minlz</code></td><td>determines the compression algorithm to use when compressing sstable data blocks for backup row data storage; [snappy = 1, zstd = 2, none = 3, minlz = 4]</td><td>Dedicated/Self-hosted (read-write); Serverless (read-only)</td></tr>
362-
<tr><td><div id="setting-storage-sstable-compression-algorithm-backup-transport" class="anchored"><code>storage.sstable.compression_algorithm_backup_transport</code></div></td><td>enumeration</td><td><code>minlz</code></td><td>determines the compression algorithm to use when compressing sstable data blocks for backup transport; [snappy = 1, zstd = 2, none = 3, minlz = 4]</td><td>Dedicated/Self-hosted (read-write); Serverless (read-only)</td></tr>
360+
<tr><td><div id="setting-storage-sstable-compression-algorithm" class="anchored"><code>storage.sstable.compression_algorithm</code></div></td><td>enumeration</td><td><code>fastest</code></td><td>determines the compression algorithm to use when compressing sstable data blocks for use in a Pebble store; [snappy = 1, zstd = 2, none = 3, minlz = 4, fastest = 5]</td><td>Dedicated/Self-hosted (read-write); Serverless (read-only)</td></tr>
361+
<tr><td><div id="setting-storage-sstable-compression-algorithm-backup-storage" class="anchored"><code>storage.sstable.compression_algorithm_backup_storage</code></div></td><td>enumeration</td><td><code>fastest</code></td><td>determines the compression algorithm to use when compressing sstable data blocks for backup row data storage; [snappy = 1, zstd = 2, none = 3, minlz = 4, fastest = 5]</td><td>Dedicated/Self-hosted (read-write); Serverless (read-only)</td></tr>
362+
<tr><td><div id="setting-storage-sstable-compression-algorithm-backup-transport" class="anchored"><code>storage.sstable.compression_algorithm_backup_transport</code></div></td><td>enumeration</td><td><code>fastest</code></td><td>determines the compression algorithm to use when compressing sstable data blocks for backup transport; [snappy = 1, zstd = 2, none = 3, minlz = 4, fastest = 5]</td><td>Dedicated/Self-hosted (read-write); Serverless (read-only)</td></tr>
363363
<tr><td><div id="setting-storage-wal-failover-unhealthy-op-threshold" class="anchored"><code>storage.wal_failover.unhealthy_op_threshold</code></div></td><td>duration</td><td><code>100ms</code></td><td>the latency of a WAL write considered unhealthy and triggers a failover to a secondary WAL location</td><td>Dedicated/Self-Hosted</td></tr>
364364
<tr><td><div id="setting-timeseries-storage-enabled" class="anchored"><code>timeseries.storage.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>if set, periodic timeseries data is stored within the cluster; disabling is not recommended unless you are storing the data elsewhere</td><td>Dedicated/Self-Hosted</td></tr>
365365
<tr><td><div id="setting-timeseries-storage-resolution-10s-ttl" class="anchored"><code>timeseries.storage.resolution_10s.ttl</code></div></td><td>duration</td><td><code>240h0m0s</code></td><td>the maximum age of time series data stored at the 10 second resolution. Data older than this is subject to rollup and deletion.</td><td>Dedicated/Self-hosted (read-write); Serverless (read-only)</td></tr>

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,7 @@ require (
139139
github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55
140140
github.com/cockroachdb/gostdlib v1.19.0
141141
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506
142-
github.com/cockroachdb/pebble v0.0.0-20250607160804-056234f760e0
142+
github.com/cockroachdb/pebble v0.0.0-20250610195649-24730ad60ca6
143143
github.com/cockroachdb/redact v1.1.6
144144
github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd
145145
github.com/cockroachdb/stress v0.0.0-20220803192808-1806698b1b7b

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -597,8 +597,8 @@ github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 h1:ASDL+UJcILM
597597
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506/go.mod h1:Mw7HqKr2kdtu6aYGn3tPmAftiP3QPX63LdK/zcariIo=
598598
github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA=
599599
github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA=
600-
github.com/cockroachdb/pebble v0.0.0-20250607160804-056234f760e0 h1:n3qVQukPDHgfy3IRUt9D2zdU2BDc25xnetEzhMwbIFo=
601-
github.com/cockroachdb/pebble v0.0.0-20250607160804-056234f760e0/go.mod h1:cnPe7/4+ZaEoSSlr4P3QT2CBbd6fdnP+JQ6On5PEpoE=
600+
github.com/cockroachdb/pebble v0.0.0-20250610195649-24730ad60ca6 h1:J9+QrMWdFk+LCg2Ht2wZo+clzTxEv6yiucRVSSQuUVI=
601+
github.com/cockroachdb/pebble v0.0.0-20250610195649-24730ad60ca6/go.mod h1:cnPe7/4+ZaEoSSlr4P3QT2CBbd6fdnP+JQ6On5PEpoE=
602602
github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
603603
github.com/cockroachdb/redact v1.1.6 h1:zXJBwDZ84xJNlHl1rMyCojqyIxv+7YUpQiJLQ7n4314=
604604
github.com/cockroachdb/redact v1.1.6/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=

pkg/storage/pebble.go

Lines changed: 14 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ import (
1212
"math"
1313
"os"
1414
"path/filepath"
15-
"runtime"
1615
"sort"
1716
"strconv"
1817
"strings"
@@ -184,17 +183,22 @@ type CompressionAlgorithm int64
184183
// These values end up being the underlying value of the cluster setting, so
185184
// they must be stable across releases.
186185
const (
187-
CompressionAlgorithmSnappy CompressionAlgorithm = 1
188-
CompressionAlgorithmZstd CompressionAlgorithm = 2
189-
CompressionAlgorithmNone CompressionAlgorithm = 3
190-
CompressionAlgorithmMinLZ CompressionAlgorithm = 4
186+
CompressionAlgorithmSnappy CompressionAlgorithm = 1
187+
CompressionAlgorithmZstd CompressionAlgorithm = 2
188+
CompressionAlgorithmNone CompressionAlgorithm = 3
189+
CompressionAlgorithmMinLZ CompressionAlgorithm = 4
190+
CompressionAlgorithmFastest CompressionAlgorithm = 5
191191
)
192192

193193
var compressionAlgorithmToString = map[CompressionAlgorithm]string{
194194
CompressionAlgorithmSnappy: "snappy",
195195
CompressionAlgorithmMinLZ: "minlz",
196196
CompressionAlgorithmNone: "none",
197197
CompressionAlgorithmZstd: "zstd",
198+
199+
// CompressionAlgorithmFastest uses either Snappy or MinLZ, depending on the
200+
// architecture.
201+
CompressionAlgorithmFastest: "fastest",
198202
}
199203

200204
// String implements fmt.Stringer for CompressionAlgorithm.
@@ -225,19 +229,6 @@ func RegisterCompressionAlgorithmClusterSetting(
225229
)
226230
}
227231

228-
var defaultCompressionAlgorithm = func() CompressionAlgorithm {
229-
if runtime.GOARCH == "amd64" {
230-
// We prefer MinLZ on amd64 because it is slightly superior to Snappy in
231-
// almost all cases (both in terms of speed and compression ratio).
232-
//
233-
// Only amd64 has an optimized assembly MinLZ implementation; the Go
234-
// implementation is significantly slower, especially when decompressing;
235-
// see https://github.com/minio/minlz#protobuf-sample
236-
return CompressionAlgorithmMinLZ
237-
}
238-
return CompressionAlgorithmSnappy
239-
}()
240-
241232
// CompressionAlgorithmStorage determines the compression algorithm used to
242233
// compress data blocks when writing sstables for use in a Pebble store (written
243234
// directly, or constructed for ingestion on a remote store via AddSSTable).
@@ -246,7 +237,7 @@ var defaultCompressionAlgorithm = func() CompressionAlgorithm {
246237
var CompressionAlgorithmStorage = RegisterCompressionAlgorithmClusterSetting(
247238
"storage.sstable.compression_algorithm",
248239
`determines the compression algorithm to use when compressing sstable data blocks for use in a Pebble store;`,
249-
defaultCompressionAlgorithm,
240+
CompressionAlgorithmFastest,
250241
)
251242

252243
// CompressionAlgorithmBackupStorage determines the compression algorithm used
@@ -256,7 +247,7 @@ var CompressionAlgorithmStorage = RegisterCompressionAlgorithmClusterSetting(
256247
var CompressionAlgorithmBackupStorage = RegisterCompressionAlgorithmClusterSetting(
257248
"storage.sstable.compression_algorithm_backup_storage",
258249
`determines the compression algorithm to use when compressing sstable data blocks for backup row data storage;`,
259-
defaultCompressionAlgorithm,
250+
CompressionAlgorithmFastest,
260251
)
261252

262253
// CompressionAlgorithmBackupTransport determines the compression algorithm used
@@ -269,7 +260,7 @@ var CompressionAlgorithmBackupStorage = RegisterCompressionAlgorithmClusterSetti
269260
var CompressionAlgorithmBackupTransport = RegisterCompressionAlgorithmClusterSetting(
270261
"storage.sstable.compression_algorithm_backup_transport",
271262
`determines the compression algorithm to use when compressing sstable data blocks for backup transport;`,
272-
defaultCompressionAlgorithm,
263+
CompressionAlgorithmFastest,
273264
)
274265

275266
func getCompressionProfile(
@@ -286,6 +277,8 @@ func getCompressionProfile(
286277
return pebble.NoCompression
287278
case CompressionAlgorithmMinLZ:
288279
return pebble.MinLZCompression
280+
case CompressionAlgorithmFastest:
281+
return pebble.FastestCompression
289282
default:
290283
return pebble.DefaultCompression
291284
}

0 commit comments

Comments
 (0)