Skip to content

Commit d210117

Browse files
craig[bot]KeithChwenyihu6jbowens
committed
148230: changefeedccl: modify changefeed job payload to support db-level changefeeds r=asg0451 a=KeithCh Update and populate changefeed job payload to support db-level changefeeds. Epic: CRDB-1421 Resolves: #147370 Release note: None 149725: asim: fix shorter than expected replica change delay r=tbg a=wenyihu6 Previously, we used integer division to compute the delay for adding a replica, which led to shorter than expected delays. We previously attempted to fix this by switching to float division but didn't notice we are later casting to time.Duration which truncated the result. This commit fixes the issue by casting time.Second to a float instead. Before: ``` I250709 03:24:05.016545 47 kv/kvserver/asim/op/settings.go:158 ⋮ [-] 54117 estimatedTimeToAddReplica: 0.80, s.RebalancingSnapshotRate: 33554432, base: 100000000 I250709 03:24:05.016549 47 kv/kvserver/asim/op/settings.go:161 ⋮ [-] 54118 calculated replica change delay: ‹100ms› for range size 26843545 ``` After: ``` I250709 03:27:53.914048 43 kv/kvserver/asim/op/settings.go:159 ⋮ [-] 11096 estimatedTimeToAddReplica: 0.80, s.RebalancingSnapshotRate: 33554432, base: 100000000 I250709 03:27:53.914053 43 kv/kvserver/asim/op/settings.go:162 ⋮ [-] 11097 calculated replica change delay: ‹899.999982ms› for range size 26843545 ``` See slack https://cockroachlabs.slack.com/archives/C048HDZJSAY/p1751935031605719 for more context. Epic: none Release note: none 149995: kvserver: add LBRebalancingMultiMetric r=tbg a=wenyihu6 **settings: support enum cluster setting validation** Previously, enum cluster settings did not support validation check. This commit adds one. I considered whether validation should take a string or an enum. If we validate using the enum directly, we’d need separate validator function signatures for each enum type, since `EnumSetting[T constraints.Integer]` is generic but `SettingOption` isn’t. If we take a string, `ParseEnum` happens earlier and `SET` only sees the enum's integer value, validating directly would require converting the int back to a string. I ended up taking in a string since it aligns with the existing design pattern around enum settings. 1. `RegisterEnumSetting` uses strings for default values. 2. It feels more intuitive to set the cluster setting with the string input. It did require us adding `parseEnumInt64`, which converts the enum's integer value back to its string representation for validation. Epic: none Release note: none --- **kvserver: add LBRebalancingMultiMetric** This commit adds a new option to kvserver.LoadBasedRebalancingMode for the multi-metric allocator. Note that there is a validation check in place to prevent users from enabling it since it is still an unimplemented feature. Epic: none Release note: none 149998: go.mod: bump Pebble to 837c19b81bab r=xinhaoz a=jbowens Changes: * [`837c19b8`](cockroachdb/pebble@837c19b8) db: fix double Close of directory lock during failed Open * [`b35ff004`](cockroachdb/pebble@b35ff004) db: re-export base.LockDirectory, base.DirLock * [`7d7e2f4b`](cockroachdb/pebble@7d7e2f4b) sstable: set up proper iterator transforms during EstimateDiskUsage * [`20b0451b`](cockroachdb/pebble@20b0451b) db: validate sstable's blob value liveness during DebugCheckLevels * [`ba961483`](cockroachdb/pebble@ba961483) pebble: fix wal recovery dir lock acquisition in Open * [`6feecba1`](cockroachdb/pebble@6feecba1) pebble: add blob rewrite compaction count to metrics string * [`42327e92`](cockroachdb/pebble@42327e92) fix: avoid reducing the last op in reduction process * [`33e6fedc`](cockroachdb/pebble@33e6fedc) fix: use the correct key format in reduction * [`4de9b7c5`](cockroachdb/pebble@4de9b7c5) db: reformat Metrics.String * [`0ef4b16e`](cockroachdb/pebble@0ef4b16e) sstable: add separated value retrieval internal iterator stats * [`dd0f8384`](cockroachdb/pebble@dd0f8384) pebble: lock wal directories on Open * [`c8cfc93a`](cockroachdb/pebble@c8cfc93a) db: use internal iterator stats to populate compaction bytes read * [`1dd9cda1`](cockroachdb/pebble@1dd9cda1) db: maintain per-block kind internal iterator stats * [`41687897`](cockroachdb/pebble@41687897) compression: deflake TestAdaptiveCompressorCompressible Release note: none. Epic: none. Co-authored-by: Keith Chow <[email protected]> Co-authored-by: wenyihu6 <[email protected]> Co-authored-by: Jackson Owens <[email protected]>
5 parents 691f7a8 + 39234ec + 19223af + 6d90941 + 39153bb commit d210117

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+417
-150
lines changed

DEPS.bzl

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1835,10 +1835,10 @@ def go_deps():
18351835
patches = [
18361836
"@com_github_cockroachdb_cockroach//build/patches:com_github_cockroachdb_pebble.patch",
18371837
],
1838-
sha256 = "d3a727e03593def35072caac3e097b4cbc58ebafc3f8dc7294f9acfb7c4d79c0",
1839-
strip_prefix = "github.com/cockroachdb/[email protected]20250708172732-a370be5e7e50",
1838+
sha256 = "b854ba6802c929910b6c5c2572bf9b8b408e1d4cb80db9c62af8fc19fc74dd14",
1839+
strip_prefix = "github.com/cockroachdb/[email protected]20250711175034-837c19b81bab",
18401840
urls = [
1841-
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250708172732-a370be5e7e50.zip",
1841+
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250711175034-837c19b81bab.zip",
18421842
],
18431843
)
18441844
go_repository(

build/bazelutil/distdir_files.bzl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -358,7 +358,7 @@ DISTDIR_FILES = {
358358
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/gostdlib/com_github_cockroachdb_gostdlib-v1.19.0.zip": "c4d516bcfe8c07b6fc09b8a9a07a95065b36c2855627cb3514e40c98f872b69e",
359359
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/logtags/com_github_cockroachdb_logtags-v0.0.0-20241215232642-bb51bb14a506.zip": "920068af09e3846d9ebb4e4a7787ff1dd10f3989c5f940ad861b0f6a9f824f6e",
360360
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/metamorphic/com_github_cockroachdb_metamorphic-v0.0.0-20231108215700-4ba948b56895.zip": "28c8cf42192951b69378cf537be5a9a43f2aeb35542908cc4fe5f689505853ea",
361-
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250708172732-a370be5e7e50.zip": "d3a727e03593def35072caac3e097b4cbc58ebafc3f8dc7294f9acfb7c4d79c0",
361+
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250711175034-837c19b81bab.zip": "b854ba6802c929910b6c5c2572bf9b8b408e1d4cb80db9c62af8fc19fc74dd14",
362362
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/redact/com_github_cockroachdb_redact-v1.1.6.zip": "018eccb5fb9ca52d43ec9eaf213539d01c1f2b94e0e822406ebfb2e9321ef6cf",
363363
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/returncheck/com_github_cockroachdb_returncheck-v0.0.0-20200612231554-92cdbca611dd.zip": "ce92ba4352deec995b1f2eecf16eba7f5d51f5aa245a1c362dfe24c83d31f82b",
364364
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/stress/com_github_cockroachdb_stress-v0.0.0-20220803192808-1806698b1b7b.zip": "3fda531795c600daf25532a4f98be2a1335cd1e5e182c72789bca79f5f69fcc1",

docs/generated/settings/settings.html

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@
8585
<tr><td><div id="setting-jobs-retention-time" class="anchored"><code>jobs.retention_time</code></div></td><td>duration</td><td><code>336h0m0s</code></td><td>the amount of time for which records for completed jobs are retained</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
8686
<tr><td><div id="setting-kv-allocator-lease-rebalance-threshold" class="anchored"><code>kv.allocator.lease_rebalance_threshold</code></div></td><td>float</td><td><code>0.05</code></td><td>minimum fraction away from the mean a store&#39;s lease count can be before it is considered for lease-transfers</td><td>Dedicated/Self-Hosted</td></tr>
8787
<tr><td><div id="setting-kv-allocator-load-based-lease-rebalancing-enabled" class="anchored"><code>kv.allocator.load_based_lease_rebalancing.enabled</code></div></td><td>boolean</td><td><code>true</code></td><td>set to enable rebalancing of range leases based on load and latency</td><td>Dedicated/Self-Hosted</td></tr>
88-
<tr><td><div id="setting-kv-allocator-load-based-rebalancing" class="anchored"><code>kv.allocator.load_based_rebalancing</code></div></td><td>enumeration</td><td><code>leases and replicas</code></td><td>whether to rebalance based on the distribution of load across stores [off = 0, leases = 1, leases and replicas = 2]</td><td>Dedicated/Self-Hosted</td></tr>
88+
<tr><td><div id="setting-kv-allocator-load-based-rebalancing" class="anchored"><code>kv.allocator.load_based_rebalancing</code></div></td><td>enumeration</td><td><code>leases and replicas</code></td><td>whether to rebalance based on the distribution of load across stores [off = 0, leases = 1, leases and replicas = 2, multi-metric = 3]</td><td>Dedicated/Self-Hosted</td></tr>
8989
<tr><td><div id="setting-kv-allocator-load-based-rebalancing-objective" class="anchored"><code>kv.allocator.load_based_rebalancing.objective</code></div></td><td>enumeration</td><td><code>cpu</code></td><td>what objective does the cluster use to rebalance; if set to `qps` the cluster will attempt to balance qps among stores, if set to `cpu` the cluster will attempt to balance cpu usage among stores [qps = 0, cpu = 1]</td><td>Dedicated/Self-Hosted</td></tr>
9090
<tr><td><div id="setting-kv-allocator-load-based-rebalancing-interval" class="anchored"><code>kv.allocator.load_based_rebalancing_interval</code></div></td><td>duration</td><td><code>1m0s</code></td><td>the rough interval at which each store will check for load-based lease / replica rebalancing opportunities</td><td>Dedicated/Self-Hosted</td></tr>
9191
<tr><td><div id="setting-kv-allocator-qps-rebalance-threshold" class="anchored"><code>kv.allocator.qps_rebalance_threshold</code></div></td><td>float</td><td><code>0.1</code></td><td>minimum fraction away from the mean a store&#39;s QPS (such as queries per second) can be before it is considered overfull or underfull</td><td>Dedicated/Self-Hosted</td></tr>

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ require (
140140
github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55
141141
github.com/cockroachdb/gostdlib v1.19.0
142142
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506
143-
github.com/cockroachdb/pebble v0.0.0-20250708172732-a370be5e7e50
143+
github.com/cockroachdb/pebble v0.0.0-20250711175034-837c19b81bab
144144
github.com/cockroachdb/redact v1.1.6
145145
github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd
146146
github.com/cockroachdb/stress v0.0.0-20220803192808-1806698b1b7b

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -582,8 +582,8 @@ github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 h1:ASDL+UJcILM
582582
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506/go.mod h1:Mw7HqKr2kdtu6aYGn3tPmAftiP3QPX63LdK/zcariIo=
583583
github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA=
584584
github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA=
585-
github.com/cockroachdb/pebble v0.0.0-20250708172732-a370be5e7e50 h1:nURMd/syIIPsnG2yFt0rx4kUFmjSuCIuGys74/YQxEY=
586-
github.com/cockroachdb/pebble v0.0.0-20250708172732-a370be5e7e50/go.mod h1:LLa+I2vZrOsrsy4QcuZ6wSXBNQll7erNg3GCr8NpE90=
585+
github.com/cockroachdb/pebble v0.0.0-20250711175034-837c19b81bab h1:TL4dKn78L2rqTb2bOMnLJjYoyx7NQUOyi/BD0UDYAjo=
586+
github.com/cockroachdb/pebble v0.0.0-20250711175034-837c19b81bab/go.mod h1:LLa+I2vZrOsrsy4QcuZ6wSXBNQll7erNg3GCr8NpE90=
587587
github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
588588
github.com/cockroachdb/redact v1.1.6 h1:zXJBwDZ84xJNlHl1rMyCojqyIxv+7YUpQiJLQ7n4314=
589589
github.com/cockroachdb/redact v1.1.6/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=

pkg/ccl/changefeedccl/alter_changefeed_stmt.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -450,15 +450,15 @@ func generateAndValidateNewTargets(
450450
prevTargets := AllTargets(prevDetails)
451451
noLongerExist := make(map[string]descpb.ID)
452452
if err := prevTargets.EachTarget(func(targetSpec changefeedbase.Target) error {
453-
k := targetKey{TableID: targetSpec.TableID, FamilyName: tree.Name(targetSpec.FamilyName)}
453+
k := targetKey{TableID: targetSpec.DescID, FamilyName: tree.Name(targetSpec.FamilyName)}
454454
var desc catalog.TableDescriptor
455-
if d, exists := descResolver.DescByID[targetSpec.TableID]; exists {
455+
if d, exists := descResolver.DescByID[targetSpec.DescID]; exists {
456456
desc = d.(catalog.TableDescriptor)
457457
} else {
458458
// Table was dropped; that's okay since the changefeed likely
459459
// will handle DROP alter command below; and if not, then we'll resume
460460
// the changefeed, which will promptly fail if the table no longer exist.
461-
noLongerExist[string(targetSpec.StatementTimeName)] = targetSpec.TableID
461+
noLongerExist[string(targetSpec.StatementTimeName)] = targetSpec.DescID
462462
return nil
463463
}
464464

@@ -477,11 +477,11 @@ func generateAndValidateNewTargets(
477477
FamilyName: tree.Name(targetSpec.FamilyName),
478478
}
479479
newTargets[k] = newTarget
480-
newTableDescs[targetSpec.TableID] = descResolver.DescByID[targetSpec.TableID]
480+
newTableDescs[targetSpec.DescID] = descResolver.DescByID[targetSpec.DescID]
481481

482482
originalSpecs[newTarget] = jobspb.ChangefeedTargetSpecification{
483483
Type: targetSpec.Type,
484-
TableID: targetSpec.TableID,
484+
DescID: targetSpec.DescID,
485485
FamilyName: targetSpec.FamilyName,
486486
StatementTimeName: string(targetSpec.StatementTimeName),
487487
}

pkg/ccl/changefeedccl/cdceval/compat_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ func TestCompatRewrite(t *testing.T) {
3535
execCfg := s.ExecutorConfig().(sql.ExecutorConfig)
3636
fooDesc := cdctest.GetHydratedTableDescriptor(t, s.ExecutorConfig(), "foo")
3737
target := changefeedbase.Target{
38-
TableID: fooDesc.GetID(),
38+
DescID: fooDesc.GetID(),
3939
}
4040
fooRef := fmt.Sprintf("[%d AS foo]", fooDesc.GetID())
4141

pkg/ccl/changefeedccl/cdceval/expr_eval_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -545,7 +545,7 @@ $$`)
545545
targets := changefeedbase.Targets{}
546546
target := changefeedbase.Target{
547547
Type: targetType,
548-
TableID: desc.GetID(),
548+
DescID: desc.GetID(),
549549
FamilyName: tc.familyName,
550550
}
551551
targets.Add(target)
@@ -647,7 +647,7 @@ func TestUnsupportedCDCFunctions(t *testing.T) {
647647
sqlDB.Exec(t, "CREATE TABLE foo (a INT PRIMARY KEY, b STRING)")
648648
desc := cdctest.GetHydratedTableDescriptor(t, s.ExecutorConfig(), "foo")
649649
target := changefeedbase.Target{
650-
TableID: desc.GetID(),
650+
DescID: desc.GetID(),
651651
FamilyName: desc.GetFamilies()[0].Name,
652652
}
653653
for fnCall, errFn := range map[string]string{
@@ -776,7 +776,7 @@ func newEvaluatorWithNormCheck(
776776
context.Background(), execCfg, username.RootUserName(), defaultDBSessionData, desc, schemaTS,
777777
jobspb.ChangefeedTargetSpecification{
778778
Type: target.Type,
779-
TableID: target.TableID,
779+
DescID: target.DescID,
780780
FamilyName: target.FamilyName,
781781
},
782782
sc, splitFamilies,

pkg/ccl/changefeedccl/cdceval/plan_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,7 @@ FAMILY extra (extra)
261261
return
262262
}
263263
target := jobspb.ChangefeedTargetSpecification{
264-
TableID: tc.desc.GetID(),
264+
DescID: tc.desc.GetID(),
265265
StatementTimeName: tc.desc.GetName(),
266266
FamilyName: tc.targetFamily,
267267
}

pkg/ccl/changefeedccl/cdceval/validation.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,9 +83,9 @@ func normalizeAndValidateSelectForTarget(
8383
}
8484

8585
// Sanity check target and descriptor refer to the same table.
86-
if target.TableID != desc.GetID() {
86+
if target.DescID != desc.GetID() {
8787
return nil, errors.AssertionFailedf("target table id (%d) does not match descriptor id (%d)",
88-
target.TableID, desc.GetID())
88+
target.DescID, desc.GetID())
8989
}
9090

9191
columnVisitor := checkColumnsVisitor{

0 commit comments

Comments
 (0)