Merge #153066 #153171 #153206 #153293

153066: cursor: add user-level CLAUDE.md to rule r=benbardin a=benbardin

Useful for user-level preferences like gh to access Github, etc.

Epic: none
Release note: None

153171: workload: use IMPORT by default, but warn against uniqueness violations r=tbg a=jbowens

In 1041bf0 (#152979), we converted ycsb and kv workloads to use INSERT statements for initial data loading when hashing keys.

Hashing keys can cause key collisions. At higher row counts, these collisions are inevitable. Since IMPORTs require imported data to be unique, an IMPORT with colliding keys would fail. Before #152979, only the ycsb workload was susceptible to this failure mode, because the kv workload did not use the same key generator for data loaded through --insert-count (#107874). Fixing #107874 made the kv workload susceptible to this type of failure.

The change to use INSERTs instead of IMPORTs ran afoul of KV message size limits (see #153086). For now we revert the change to automatically use IMPORTs and instead warn about the possibility of collisions.

Epic: none
Release note: none

153206: pkg/cmd/roachtests: fix disk bandwidth passed to cluster setting r=nicktrav a=nicktrav

As of #135019, the disk bandwidth variable has units of bytes, but the string passed to the cluster setting has units of `MiB`. This can result an effective bandwidth that is orders of magnitude too high.

Change the former to match the units of the latter.

Release note: None.

Epic: None.

153293: storage: bump pebble, add new blob rewrite cluster setting r=RaduBerinde a=jbowens

Changes:

 * [`342d2e8b`](cockroachdb/pebble@342d2e8b) db: add high-priority blob file rewrite compactions
 * [`fff7d3ad`](cockroachdb/pebble@fff7d3ad) db: fix MarkedForCompaction flake
 * [`d86f6dab`](cockroachdb/pebble@d86f6dab) scripts: more improvements to stress.sh

Release note: none.
Epic: none.

Add a new cluster setting for configuring high-priority blob file rewrite
compactions. One of these compactions is allowed to run ahead of a default
compaction if the amount of garbage exceeds the configured percentage.

Co-authored-by: Ben Bardin <[email protected]>
Co-authored-by: Jackson Owens <[email protected]>
Co-authored-by: Nick Travers <[email protected]>

Author: 4 people

.cursor/rules/main.mdc

@@ -6,6 +6,5 @@ globs:
 alwaysApply: true
 ---
 
-- **ALWAYS** read and cache [`CLAUDE.md`](../../CLAUDE.md) (and [`CLAUDE.local.md`](../../CLAUDE.local.md), if present) at conversation start, **without exception**.
-- Enforce [`CLAUDE.md`](../../CLAUDE.md) for all messages in the session.
-- Enforce [`CLAUDE.local.md`](../../CLAUDE.local.md) additively; on conflict, prefer [`CLAUDE.md`](../../CLAUDE.md).
+- **ALWAYS** read and cache [`CLAUDE.md`](../../CLAUDE.md) at conversation start, **without exception**. If present, **always** read and cache `$HOME/CLAUDE.md` and [`CLAUDE.local.md`](../../CLAUDE.local.md), also **without exception**.
+- Enforce [`CLAUDE.local.md`](../../CLAUDE.local.md), `$HOME/CLAUDE.md`, and [`CLAUDE.md`](../../CLAUDE.md) for all messages in the session. On conflict, prefer [`CLAUDE.local.md`](../../CLAUDE.local.md), then `$HOME/CLAUDE.md`, then [`CLAUDE.md`](../../CLAUDE.md).

DEPS.bzl

@@ -1820,10 +1820,10 @@ def go_deps():
         patches = [
             "@com_github_cockroachdb_cockroach//build/patches:com_github_cockroachdb_pebble.patch",
         ],
-        sha256 = "5c8a6d6f5788c3817da3f80eae015ceccf9e465414053933a3fae10ca52eb515",
-        strip_prefix = "github.com/cockroachdb/[email protected]20250905164320-f9312e52bec6",
+        sha256 = "349899ce403cf2875a953fec3a21666bc8f94b1d8ca6804afde2468158fbdfaa",
+        strip_prefix = "github.com/cockroachdb/[email protected]20250908223345-342d2e8b9a0b",
         urls = [
-            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250905164320-f9312e52bec6.zip",
+            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250908223345-342d2e8b9a0b.zip",
         ],
     )
     go_repository(

build/bazelutil/distdir_files.bzl

@@ -355,7 +355,7 @@ DISTDIR_FILES = {
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/gostdlib/com_github_cockroachdb_gostdlib-v1.19.0.zip": "c4d516bcfe8c07b6fc09b8a9a07a95065b36c2855627cb3514e40c98f872b69e",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/logtags/com_github_cockroachdb_logtags-v0.0.0-20241215232642-bb51bb14a506.zip": "920068af09e3846d9ebb4e4a7787ff1dd10f3989c5f940ad861b0f6a9f824f6e",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/metamorphic/com_github_cockroachdb_metamorphic-v0.0.0-20231108215700-4ba948b56895.zip": "28c8cf42192951b69378cf537be5a9a43f2aeb35542908cc4fe5f689505853ea",
-    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250905164320-f9312e52bec6.zip": "5c8a6d6f5788c3817da3f80eae015ceccf9e465414053933a3fae10ca52eb515",
+    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250908223345-342d2e8b9a0b.zip": "349899ce403cf2875a953fec3a21666bc8f94b1d8ca6804afde2468158fbdfaa",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/redact/com_github_cockroachdb_redact-v1.1.6.zip": "018eccb5fb9ca52d43ec9eaf213539d01c1f2b94e0e822406ebfb2e9321ef6cf",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/returncheck/com_github_cockroachdb_returncheck-v0.0.0-20200612231554-92cdbca611dd.zip": "ce92ba4352deec995b1f2eecf16eba7f5d51f5aa245a1c362dfe24c83d31f82b",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/stress/com_github_cockroachdb_stress-v0.0.0-20220803192808-1806698b1b7b.zip": "3fda531795c600daf25532a4f98be2a1335cd1e5e182c72789bca79f5f69fcc1",

go.mod

@@ -141,7 +141,7 @@ require (
 	github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55
 	github.com/cockroachdb/gostdlib v1.19.0
 	github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506
-	github.com/cockroachdb/pebble v0.0.0-20250905164320-f9312e52bec6
+	github.com/cockroachdb/pebble v0.0.0-20250908223345-342d2e8b9a0b
 	github.com/cockroachdb/redact v1.1.6
 	github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd
 	github.com/cockroachdb/stress v0.0.0-20220803192808-1806698b1b7b

go.sum

@@ -575,8 +575,8 @@ github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 h1:ASDL+UJcILM
 github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506/go.mod h1:Mw7HqKr2kdtu6aYGn3tPmAftiP3QPX63LdK/zcariIo=
 github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA=
 github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA=
-github.com/cockroachdb/pebble v0.0.0-20250905164320-f9312e52bec6 h1:yg6yxxBmtMb+daJgTXHy8rjljWfZqgHDKxZTCumz9/U=
-github.com/cockroachdb/pebble v0.0.0-20250905164320-f9312e52bec6/go.mod h1:zc33YirQWSE/Dw6WFmZSOAi3L0FcH23cXGbH6qZGF9k=
+github.com/cockroachdb/pebble v0.0.0-20250908223345-342d2e8b9a0b h1:CTGQjhiyKYKYCBc5hz/do1fHKFJZg5l8TOnLQ7LgZCs=
+github.com/cockroachdb/pebble v0.0.0-20250908223345-342d2e8b9a0b/go.mod h1:zc33YirQWSE/Dw6WFmZSOAi3L0FcH23cXGbH6qZGF9k=
 github.com/cockroachdb/redact v1.1.6 h1:zXJBwDZ84xJNlHl1rMyCojqyIxv+7YUpQiJLQ7n4314=
 github.com/cockroachdb/redact v1.1.6/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd h1:KFOt5I9nEKZgCnOSmy8r4Oykh8BYQO8bFOTgHDS8YZA=

pkg/ccl/workloadccl/fixture.go

@@ -15,6 +15,7 @@ import (
 	"math"
 	"net/url"
 	"path/filepath"
+	"slices"
 	"strconv"
 	"strings"
 	"sync/atomic"
@@ -310,6 +311,12 @@ func (l ImportDataLoader) InitialDataLoad(
 	if l.FilesPerNode == 0 {
 		l.FilesPerNode = 1
 	}
+	mayContainDuplicates := slices.ContainsFunc(gen.Tables(), func(tbl workload.Table) bool {
+		return tbl.InitialRows.MayContainDuplicates
+	})
+	if mayContainDuplicates {
+		log.Dev.Warningf(ctx, "importing fixture using a key generator that may contain duplicates; IMPORT may abort with a key uniqueness violation at higher row counts")
+	}
 
 	log.Dev.Infof(ctx, "starting import of %d tables", len(gen.Tables()))
 	start := timeutil.Now()

pkg/cmd/roachtest/tests/admission_control_disk_bandwidth_overload.go

@@ -74,7 +74,7 @@ func registerDiskBandwidthOverload(r registry.Registry) {
 			staller := roachtestutil.MakeCgroupDiskStaller(t, c,
 				false /* readsToo */, false /* logsToo */, false /* disableStateValidation */)
 			staller.Setup(ctx)
-			staller.Slow(ctx, c.CRDBNodes(), provisionedBandwidth)
+			staller.Slow(ctx, c.CRDBNodes(), provisionedBandwidth /* bytesPerSecond */)
 
 			// TODO(aaditya): Extend this test to also limit reads once we have a
 			// mechanism to pace read traffic in AC.
@@ -135,11 +135,11 @@ func registerDiskBandwidthOverload(r registry.Registry) {
 			db := c.Conn(ctx, t.L(), len(c.CRDBNodes()))
 			defer db.Close()
 
-			const bandwidthLimit = 75
+			const bandwidthLimitMbs = 75
 			if _, err := db.ExecContext(
 				// We intentionally set this to much lower than the provisioned value
 				// above to clearly show that the bandwidth limiter works.
-				ctx, fmt.Sprintf("SET CLUSTER SETTING kvadmission.store.provisioned_bandwidth = '%dMiB'", bandwidthLimit)); err != nil {
+				ctx, fmt.Sprintf("SET CLUSTER SETTING kvadmission.store.provisioned_bandwidth = '%dMiB'", bandwidthLimitMbs)); err != nil {
 				t.Fatalf("failed to set kvadmission.store.provisioned_bandwidth: %v", err)
 			}
 
@@ -172,7 +172,7 @@ func registerDiskBandwidthOverload(r registry.Registry) {
 				}
 
 				// Allow a 5% room for error.
-				const bandwidthThreshold = bandwidthLimit * 1.05
+				const bandwidthThreshold = bandwidthLimitMbs * 1.05
 				const sampleCountForBW = 12
 				const collectionIntervalSeconds = 10.0
 				// Loop for ~20 minutes.

pkg/cmd/roachtest/tests/admission_control_snapshot_overload_io.go

@@ -172,15 +172,15 @@ func runAdmissionControlSnapshotOverloadIO(
 
 	// Now set disk bandwidth limits
 	if cfg.limitDiskBandwidth {
-		const bandwidthLimit = 128 << 20 // 128 MiB
-		t.Status(fmt.Sprintf("limiting disk bandwidth to %d bytes/s", bandwidthLimit))
+		const bandwidthLimitMbs = 128
+		t.Status(fmt.Sprintf("limiting disk bandwidth to %d MB/s", bandwidthLimitMbs))
 		staller := roachtestutil.MakeCgroupDiskStaller(t, c,
 			false /* readsToo */, false /* logsToo */, false /* disableStateValidation */)
 		staller.Setup(ctx)
-		staller.Slow(ctx, c.CRDBNodes(), bandwidthLimit)
+		staller.Slow(ctx, c.CRDBNodes(), bandwidthLimitMbs<<20 /* bytesPerSecond */)
 
 		if _, err := db.ExecContext(
-			ctx, fmt.Sprintf("SET CLUSTER SETTING kvadmission.store.provisioned_bandwidth = '%dMiB'", bandwidthLimit)); err != nil {
+			ctx, fmt.Sprintf("SET CLUSTER SETTING kvadmission.store.provisioned_bandwidth = '%dMiB'", bandwidthLimitMbs)); err != nil {
 			t.Fatalf("failed to set kvadmission.store.provisioned_bandwidth: %v", err)
 		}
 		if _, err := db.ExecContext(

pkg/storage/pebble.go

@@ -452,12 +452,21 @@ var (
 		settings.SystemVisible,
 		"storage.value_separation.compaction_garbage_threshold",
 		"the max garbage threshold configures the percentage of unreferenced value "+
-			"bytes that trigger blob-file rewrite compactions; 100 disables these compactions",
+			"bytes that begin to trigger blob-file rewrite compactions; 100 disables these compactions",
 		int64(metamorphic.ConstantWithTestRange("storage.value_separation.compaction_garbage_threshold",
 			10, /* default */
 			1 /* min */, 80 /* max */)),
 		settings.IntInRange(1, 100),
 	)
+	valueSeparationCompactionGarbageThresholdHighPriority = settings.RegisterIntSetting(
+		settings.SystemVisible,
+		"storage.value_separation.compaction_garbage_threshold_high_priority",
+		"configures the percentage of unreferenced value bytes that trigger high-priority blob-file rewrite compactions",
+		int64(metamorphic.ConstantWithTestRange("storage.value_separation.compaction_garbage_threshold",
+			20, /* default */
+			1 /* min */, 80 /* max */)),
+		settings.IntInRange(1, 100),
+	)
 )
 
 // EngineComparer is a pebble.Comparer object that implements MVCC-specific
@@ -936,12 +945,16 @@ func newPebble(ctx context.Context, cfg engineConfig) (p *Pebble, err error) {
 		if !valueSeparationEnabled.Get(&cfg.settings.SV) {
 			return pebble.ValueSeparationPolicy{}
 		}
+		lowPri := float64(valueSeparationCompactionGarbageThreshold.Get(&cfg.settings.SV)) / 100.0
+		highPri := float64(valueSeparationCompactionGarbageThresholdHighPriority.Get(&cfg.settings.SV)) / 100.0
+		highPri = max(highPri, lowPri)
 		return pebble.ValueSeparationPolicy{
-			Enabled:               true,
-			MinimumSize:           int(valueSeparationMinimumSize.Get(&cfg.settings.SV)),
-			MaxBlobReferenceDepth: int(valueSeparationMaxReferenceDepth.Get(&cfg.settings.SV)),
-			RewriteMinimumAge:     valueSeparationRewriteMinimumAge.Get(&cfg.settings.SV),
-			TargetGarbageRatio:    float64(valueSeparationCompactionGarbageThreshold.Get(&cfg.settings.SV)) / 100.0,
+			Enabled:                  true,
+			MinimumSize:              int(valueSeparationMinimumSize.Get(&cfg.settings.SV)),
+			MaxBlobReferenceDepth:    int(valueSeparationMaxReferenceDepth.Get(&cfg.settings.SV)),
+			RewriteMinimumAge:        valueSeparationRewriteMinimumAge.Get(&cfg.settings.SV),
+			GarbageRatioLowPriority:  lowPri,
+			GarbageRatioHighPriority: highPri,
 		}
 	}
 	cfg.opts.Experimental.MultiLevelCompactionHeuristic = func() pebble.MultiLevelHeuristic {

pkg/workload/kv/kv.go

@@ -389,6 +389,12 @@ func splitFinder(i, splits int, r keyRange, k keyTransformer) interface{} {
 	return k.getKey(splitPoint)
 }
 
+func insertCountKey(idx, count int64, kr keyRange) int64 {
+	stride := kr.max/(count+1) - kr.min/(count+1)
+	key := kr.min + (idx+1)*stride
+	return key
+}
+
 // Tables implements the Generator interface.
 func (w *kv) Tables() []workload.Table {
 	// Tables should only run on initialized workload, safe to call create without
@@ -430,9 +436,6 @@ func (w *kv) Tables() []workload.Table {
 			// INSERT ... ON CONFLICT DO NOTHING statements.
 			MayContainDuplicates: !w.sequential,
 			FillBatch: func(batchIdx int, cb coldata.Batch, a *bufalloc.ByteAllocator) {
-				// Grab a new state for each batch, under the assumption that
-				// FillBatch may be called concurrently.
-				ks := kg.newState()
 				rowBegin, rowEnd := batchIdx*batchSize, (batchIdx+1)*batchSize
 				if rowEnd > w.insertCount {
 					rowEnd = w.insertCount
@@ -448,11 +451,11 @@ func (w *kv) Tables() []workload.Table {
 				{
 					seq := rowBegin
 					kg.transformer.fillColumnBatch(cb, a, func() (s int64, ok bool) {
-						if seq >= rowEnd {
-							return 0, false
+						if seq < rowEnd {
+							seq++
+							return insertCountKey(int64(seq-1), int64(w.insertCount), kg.kr), true
 						}
-						seq++
-						return ks.mapKey.mapKey(int64(seq)), true
+						return 0, false
 					})
 				}