Skip to content

Commit 5183cb9

Browse files
committed
kvserver: add BaseScorerOptions
This commit introduces `BaseScorerOptions`, a base struct embedded by other `ScorerOptions` implementations. It provides the default implementation methods, while specific implementations can override individual methods to apply their own scoring heuristics.
1 parent 05e0e07 commit 5183cb9

File tree

7 files changed

+79
-74
lines changed

7 files changed

+79
-74
lines changed

pkg/kv/kvserver/allocator/allocatorimpl/allocator.go

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2056,9 +2056,11 @@ func (a Allocator) RebalanceNonVoter(
20562056
// machinery to achieve range count convergence.
20572057
func (a *Allocator) ScorerOptions(ctx context.Context) *RangeCountScorerOptions {
20582058
return &RangeCountScorerOptions{
2059-
IOOverloadOptions: a.IOOverloadOptions(),
2060-
DiskCapacityOptions: a.DiskOptions(),
2061-
deterministic: a.deterministic,
2059+
BaseScorerOptions: BaseScorerOptions{
2060+
IOOverload: a.IOOverloadOptions(),
2061+
DiskCapacity: a.DiskOptions(),
2062+
Deterministic: a.deterministic,
2063+
},
20622064
rangeRebalanceThreshold: RangeRebalanceThreshold.Get(&a.st.SV),
20632065
}
20642066
}
@@ -2067,9 +2069,11 @@ func (a *Allocator) ScorerOptions(ctx context.Context) *RangeCountScorerOptions
20672069
func (a *Allocator) ScorerOptionsForScatter(ctx context.Context) *ScatterScorerOptions {
20682070
return &ScatterScorerOptions{
20692071
RangeCountScorerOptions: RangeCountScorerOptions{
2070-
IOOverloadOptions: a.IOOverloadOptions(),
2071-
DiskCapacityOptions: a.DiskOptions(),
2072-
deterministic: a.deterministic,
2072+
BaseScorerOptions: BaseScorerOptions{
2073+
IOOverload: a.IOOverloadOptions(),
2074+
DiskCapacity: a.DiskOptions(),
2075+
Deterministic: a.deterministic,
2076+
},
20732077
rangeRebalanceThreshold: 0,
20742078
},
20752079
// We set jitter to be equal to the padding around replica-count rebalancing
@@ -2518,9 +2522,11 @@ func (a *Allocator) TransferLeaseTarget(
25182522
candidates,
25192523
storeDescMap,
25202524
&LoadScorerOptions{
2521-
IOOverloadOptions: a.IOOverloadOptions(),
2522-
DiskOptions: a.DiskOptions(),
2523-
Deterministic: a.deterministic,
2525+
BaseScorerOptions: BaseScorerOptions{
2526+
IOOverload: a.IOOverloadOptions(),
2527+
DiskCapacity: a.DiskOptions(),
2528+
Deterministic: a.deterministic,
2529+
},
25242530
LoadDims: opts.LoadDimensions,
25252531
LoadThreshold: LoadThresholds(&a.st.SV, opts.LoadDimensions...),
25262532
MinLoadThreshold: LoadMinThresholds(opts.LoadDimensions...),

pkg/kv/kvserver/allocator/allocatorimpl/allocator_scorer.go

Lines changed: 30 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -342,10 +342,6 @@ type ScatterScorerOptions struct {
342342

343343
var _ ScorerOptions = &ScatterScorerOptions{}
344344

345-
func (o *ScatterScorerOptions) getIOOverloadOptions() IOOverloadOptions {
346-
return o.RangeCountScorerOptions.IOOverloadOptions
347-
}
348-
349345
func (o *ScatterScorerOptions) maybeJitterStoreStats(
350346
sl storepool.StoreList, allocRand allocatorRand,
351347
) (perturbedSL storepool.StoreList) {
@@ -360,37 +356,46 @@ func (o *ScatterScorerOptions) maybeJitterStoreStats(
360356
return storepool.MakeStoreList(perturbedStoreDescs)
361357
}
362358

363-
// RangeCountScorerOptions is used by the replicateQueue to tell the Allocator's
364-
// rebalancing machinery to base its balance/convergence scores on range counts.
365-
// This means that the resulting rebalancing decisions will further the goal of
366-
// converging range counts across stores in the cluster.
367-
type RangeCountScorerOptions struct {
368-
IOOverloadOptions
369-
DiskCapacityOptions
370-
deterministic bool
371-
rangeRebalanceThreshold float64
359+
// BaseScorerOptions is the base scorer options that is embedded in
360+
// every scorer option.
361+
type BaseScorerOptions struct {
362+
IOOverload IOOverloadOptions
363+
DiskCapacity DiskCapacityOptions
364+
Deterministic bool
372365
}
373366

374-
var _ ScorerOptions = &RangeCountScorerOptions{}
367+
func (bo BaseScorerOptions) getIOOverloadOptions() IOOverloadOptions {
368+
return bo.IOOverload
369+
}
375370

376-
func (o *RangeCountScorerOptions) getIOOverloadOptions() IOOverloadOptions {
377-
return o.IOOverloadOptions
371+
func (bo BaseScorerOptions) getDiskOptions() DiskCapacityOptions {
372+
return bo.DiskCapacity
378373
}
379374

380-
func (o *RangeCountScorerOptions) getDiskOptions() DiskCapacityOptions {
381-
return o.DiskCapacityOptions
375+
func (bo BaseScorerOptions) deterministicForTesting() bool {
376+
return bo.Deterministic
382377
}
383378

384-
func (o *RangeCountScorerOptions) maybeJitterStoreStats(
379+
// maybeJitterStoreStats returns the provided store list since that is the
380+
// default behavior. ScatterScorerOptions is the only scorer option that jitters
381+
// store stats to rebalance replicas across stores randomly.
382+
func (bo BaseScorerOptions) maybeJitterStoreStats(
385383
sl storepool.StoreList, _ allocatorRand,
386-
) (perturbedSL storepool.StoreList) {
384+
) storepool.StoreList {
387385
return sl
388386
}
389387

390-
func (o *RangeCountScorerOptions) deterministicForTesting() bool {
391-
return o.deterministic
388+
// RangeCountScorerOptions is used by the replicateQueue to tell the Allocator's
389+
// rebalancing machinery to base its balance/convergence scores on range counts.
390+
// This means that the resulting rebalancing decisions will further the goal of
391+
// converging range counts across stores in the cluster.
392+
type RangeCountScorerOptions struct {
393+
BaseScorerOptions
394+
rangeRebalanceThreshold float64
392395
}
393396

397+
var _ ScorerOptions = &RangeCountScorerOptions{}
398+
394399
func (o RangeCountScorerOptions) shouldRebalanceBasedOnThresholds(
395400
ctx context.Context, eqClass equivalenceClass, metrics AllocatorMetrics,
396401
) bool {
@@ -491,10 +496,8 @@ func (o *RangeCountScorerOptions) removalMaximallyConvergesScore(
491496
// queries-per-second. This means that the resulting rebalancing decisions will
492497
// further the goal of converging QPS across stores in the cluster.
493498
type LoadScorerOptions struct {
494-
IOOverloadOptions IOOverloadOptions
495-
DiskOptions DiskCapacityOptions
496-
Deterministic bool
497-
LoadDims []load.Dimension
499+
BaseScorerOptions
500+
LoadDims []load.Dimension
498501

499502
// LoadThreshold and MinLoadThreshold track the threshold beyond which a
500503
// store should be considered under/overfull and the minimum absolute
@@ -528,23 +531,7 @@ type LoadScorerOptions struct {
528531
RebalanceImpact load.Load
529532
}
530533

531-
func (o *LoadScorerOptions) getIOOverloadOptions() IOOverloadOptions {
532-
return o.IOOverloadOptions
533-
}
534-
535-
func (o *LoadScorerOptions) getDiskOptions() DiskCapacityOptions {
536-
return o.DiskOptions
537-
}
538-
539-
func (o *LoadScorerOptions) maybeJitterStoreStats(
540-
sl storepool.StoreList, _ allocatorRand,
541-
) storepool.StoreList {
542-
return sl
543-
}
544-
545-
func (o *LoadScorerOptions) deterministicForTesting() bool {
546-
return o.Deterministic
547-
}
534+
var _ ScorerOptions = &LoadScorerOptions{}
548535

549536
// shouldRebalanceBasedOnThresholds tries to determine if, within the given
550537
// equivalenceClass `eqClass`, rebalancing a replica from one of the existing

pkg/kv/kvserver/allocator/allocatorimpl/allocator_scorer_test.go

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1557,7 +1557,9 @@ func TestShouldRebalanceDiversity(t *testing.T) {
15571557
defer log.Scope(t).Close(t)
15581558

15591559
options := &RangeCountScorerOptions{
1560-
DiskCapacityOptions: defaultDiskCapacityOptions(),
1560+
BaseScorerOptions: BaseScorerOptions{
1561+
DiskCapacity: defaultDiskCapacityOptions(),
1562+
},
15611563
}
15621564
newStore := func(id int, locality roachpb.Locality) roachpb.StoreDescriptor {
15631565
return roachpb.StoreDescriptor{
@@ -2008,7 +2010,9 @@ func TestBalanceScoreByRangeCount(t *testing.T) {
20082010
defer log.Scope(t).Close(t)
20092011

20102012
options := RangeCountScorerOptions{
2011-
DiskCapacityOptions: defaultDiskCapacityOptions(),
2013+
BaseScorerOptions: BaseScorerOptions{
2014+
DiskCapacity: defaultDiskCapacityOptions(),
2015+
},
20122016
rangeRebalanceThreshold: 0.1,
20132017
}
20142018
storeList := storepool.StoreList{
@@ -2094,7 +2098,9 @@ func TestRebalanceConvergesRangeCountOnMean(t *testing.T) {
20942098
}
20952099

20962100
options := RangeCountScorerOptions{
2097-
DiskCapacityOptions: defaultDiskCapacityOptions(),
2101+
BaseScorerOptions: BaseScorerOptions{
2102+
DiskCapacity: defaultDiskCapacityOptions(),
2103+
},
20982104
}
20992105
eqClass := equivalenceClass{
21002106
candidateSL: storeList,

pkg/kv/kvserver/allocator/allocatorimpl/allocator_test.go

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1310,7 +1310,7 @@ func TestAllocatorRebalanceBasedOnRangeCount(t *testing.T) {
13101310
gossiputil.NewStoreGossiper(g).GossipStores(stores, t)
13111311

13121312
options := a.ScorerOptions(ctx)
1313-
options.DiskCapacityOptions = testingDiskCapacityOptions
1313+
options.BaseScorerOptions.DiskCapacity = testingDiskCapacityOptions
13141314

13151315
// Every rebalance target must be either store 1 or 2.
13161316
for i := 0; i < 10; i++ {
@@ -1708,7 +1708,7 @@ func TestAllocatorRebalanceByQPS(t *testing.T) {
17081708
gossiputil.NewStoreGossiper(g).GossipStores(subtest.testStores, t)
17091709
var rangeUsageInfo allocator.RangeUsageInfo
17101710
options := TestingQPSLoadScorerOptions(100, 0.2)
1711-
options.IOOverloadOptions = IOOverloadOptions{ReplicaEnforcementLevel: IOOverloadThresholdIgnore}
1711+
options.BaseScorerOptions.IOOverload = IOOverloadOptions{ReplicaEnforcementLevel: IOOverloadThresholdIgnore}
17121712
add, remove, _, ok := a.RebalanceVoter(
17131713
ctx,
17141714
sp,
@@ -1822,7 +1822,7 @@ func TestAllocatorRemoveBasedOnQPS(t *testing.T) {
18221822
defer stopper.Stop(ctx)
18231823
gossiputil.NewStoreGossiper(g).GossipStores(subtest.testStores, t)
18241824
options := TestingQPSLoadScorerOptions(0, 0.1)
1825-
options.IOOverloadOptions = IOOverloadOptions{ReplicaEnforcementLevel: IOOverloadThresholdIgnore}
1825+
options.BaseScorerOptions.IOOverload = IOOverloadOptions{ReplicaEnforcementLevel: IOOverloadThresholdIgnore}
18261826
remove, _, err := a.RemoveVoter(
18271827
ctx,
18281828
sp,
@@ -4924,7 +4924,7 @@ func TestAllocatorRebalanceIOOverloadCheck(t *testing.T) {
49244924
sg.GossipStores(test.stores, t)
49254925
// Enable read disk health checking in candidate exclusion.
49264926
options := a.ScorerOptions(ctx)
4927-
options.IOOverloadOptions = IOOverloadOptions{
4927+
options.BaseScorerOptions.IOOverload = IOOverloadOptions{
49284928
ReplicaEnforcementLevel: test.enforcement,
49294929
ReplicaIOOverloadThreshold: 1,
49304930
UseIOThresholdMax: true,
@@ -9071,7 +9071,7 @@ func qpsBasedRebalanceFn(
90719071
)
90729072
if ok {
90739073
log.Dev.Infof(ctx, "rebalancing from %v to %v; details: %s", remove, add, details)
9074-
candidate.rebalance(&testStores[int(add.StoreID)], alloc.randGen.Int63n(1<<20), jitteredQPS, opts.DiskOptions)
9074+
candidate.rebalance(&testStores[int(add.StoreID)], alloc.randGen.Int63n(1<<20), jitteredQPS, opts.BaseScorerOptions.DiskCapacity)
90759075
}
90769076
}
90779077

@@ -9476,8 +9476,10 @@ func TestingQPSLoadScorerOptions(
94769476
qpsPerReplica float64, qpsRebalanceThreshold float64,
94779477
) *LoadScorerOptions {
94789478
options := &LoadScorerOptions{
9479-
DiskOptions: defaultDiskCapacityOptions(),
9480-
Deterministic: true,
9479+
BaseScorerOptions: BaseScorerOptions{
9480+
DiskCapacity: defaultDiskCapacityOptions(),
9481+
Deterministic: true,
9482+
},
94819483
LoadDims: []load.Dimension{load.Queries},
94829484
LoadThreshold: MakeQPSOnlyDim(qpsRebalanceThreshold),
94839485
MinLoadThreshold: LoadMinThresholds(load.Queries),

pkg/kv/kvserver/asim/storerebalancer/store_rebalancer.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -132,9 +132,11 @@ func (s simRebalanceObjectiveProvider) Objective() kvserver.LBRebalancingObjecti
132132
func (src *storeRebalancerControl) scorerOptions() *allocatorimpl.LoadScorerOptions {
133133
dim := kvserver.LBRebalancingObjective(src.settings.LBRebalancingObjective).ToDimension()
134134
return &allocatorimpl.LoadScorerOptions{
135-
IOOverloadOptions: src.allocator.IOOverloadOptions(),
136-
DiskOptions: src.allocator.DiskOptions(),
137-
Deterministic: true,
135+
BaseScorerOptions: allocatorimpl.BaseScorerOptions{
136+
IOOverload: src.allocator.IOOverloadOptions(),
137+
DiskCapacity: src.allocator.DiskOptions(),
138+
Deterministic: true,
139+
},
138140
LoadDims: []load.Dimension{dim},
139141
LoadThreshold: allocatorimpl.LoadThresholds(&src.settings.ST.SV, dim),
140142
MinLoadThreshold: allocatorimpl.LoadMinThresholds(dim),

pkg/kv/kvserver/store_rebalancer.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -319,9 +319,11 @@ func (sr *StoreRebalancer) scorerOptions(
319319
ctx context.Context, lbDimension load.Dimension,
320320
) *allocatorimpl.LoadScorerOptions {
321321
return &allocatorimpl.LoadScorerOptions{
322-
IOOverloadOptions: sr.allocator.IOOverloadOptions(),
323-
DiskOptions: sr.allocator.DiskOptions(),
324-
Deterministic: sr.storePool.IsDeterministic(),
322+
BaseScorerOptions: allocatorimpl.BaseScorerOptions{
323+
IOOverload: sr.allocator.IOOverloadOptions(),
324+
DiskCapacity: sr.allocator.DiskOptions(),
325+
Deterministic: sr.storePool.IsDeterministic(),
326+
},
325327
LoadDims: []load.Dimension{lbDimension},
326328
LoadThreshold: allocatorimpl.LoadThresholds(&sr.st.SV, lbDimension),
327329
MinLoadThreshold: allocatorimpl.LoadMinThresholds(lbDimension),

pkg/kv/kvserver/store_rebalancer_test.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -932,7 +932,7 @@ func TestChooseRangeToRebalanceRandom(t *testing.T) {
932932
hottestRanges := sr.replicaRankings.TopLoad(lbRebalanceDimension)
933933
options := sr.scorerOptions(ctx, lbRebalanceDimension)
934934
rctx := sr.NewRebalanceContext(ctx, options, hottestRanges, sr.RebalanceMode())
935-
rctx.options.IOOverloadOptions = allocatorimpl.IOOverloadOptions{ReplicaEnforcementLevel: allocatorimpl.IOOverloadThresholdIgnore}
935+
rctx.options.IOOverload = allocatorimpl.IOOverloadOptions{ReplicaEnforcementLevel: allocatorimpl.IOOverloadThresholdIgnore}
936936
rctx.options.LoadThreshold = allocatorimpl.WithAllDims(rebalanceThreshold)
937937

938938
_, voterTargets, nonVoterTargets := sr.chooseRangeToRebalance(ctx, rctx)
@@ -1273,7 +1273,7 @@ func TestChooseRangeToRebalanceAcrossHeterogeneousZones(t *testing.T) {
12731273
hottestRanges := sr.replicaRankings.TopLoad(lbRebalanceDimension)
12741274
options := sr.scorerOptions(ctx, lbRebalanceDimension)
12751275
rctx := sr.NewRebalanceContext(ctx, options, hottestRanges, kvserverbase.LBRebalancingLeasesAndReplicas)
1276-
rctx.options.IOOverloadOptions = allocatorimpl.IOOverloadOptions{
1276+
rctx.options.IOOverload = allocatorimpl.IOOverloadOptions{
12771277
ReplicaEnforcementLevel: allocatorimpl.IOOverloadThresholdBlockTransfers,
12781278
UseIOThresholdMax: true,
12791279
}
@@ -1363,7 +1363,7 @@ func TestChooseRangeToRebalanceIgnoresRangeOnBestStores(t *testing.T) {
13631363
hottestRanges := sr.replicaRankings.TopLoad(lbRebalanceDimension)
13641364
options := sr.scorerOptions(ctx, lbRebalanceDimension)
13651365
rctx := sr.NewRebalanceContext(ctx, options, hottestRanges, sr.RebalanceMode())
1366-
rctx.options.IOOverloadOptions = allocatorimpl.IOOverloadOptions{
1366+
rctx.options.IOOverload = allocatorimpl.IOOverloadOptions{
13671367
ReplicaEnforcementLevel: allocatorimpl.IOOverloadThresholdIgnore}
13681368
rctx.options.LoadThreshold = allocatorimpl.WithAllDims(0.05)
13691369

@@ -1530,7 +1530,7 @@ func TestChooseRangeToRebalanceOffHotNodes(t *testing.T) {
15301530
hottestRanges := sr.replicaRankings.TopLoad(lbRebalanceDimension)
15311531
options := sr.scorerOptions(ctx, lbRebalanceDimension)
15321532
rctx := sr.NewRebalanceContext(ctx, options, hottestRanges, sr.RebalanceMode())
1533-
rctx.options.IOOverloadOptions = allocatorimpl.IOOverloadOptions{
1533+
rctx.options.IOOverload = allocatorimpl.IOOverloadOptions{
15341534
ReplicaEnforcementLevel: allocatorimpl.IOOverloadThresholdIgnore}
15351535
rctx.options.LoadThreshold = allocatorimpl.WithAllDims(tc.rebalanceThreshold)
15361536

@@ -1637,7 +1637,7 @@ func TestNoLeaseTransferToBehindReplicas(t *testing.T) {
16371637
hottestRanges = sr.replicaRankings.TopLoad(lbRebalanceDimension)
16381638
options = sr.scorerOptions(ctx, lbRebalanceDimension)
16391639
rctx = sr.NewRebalanceContext(ctx, options, hottestRanges, sr.RebalanceMode())
1640-
rctx.options.IOOverloadOptions = allocatorimpl.IOOverloadOptions{
1640+
rctx.options.IOOverload = allocatorimpl.IOOverloadOptions{
16411641
ReplicaEnforcementLevel: allocatorimpl.IOOverloadThresholdIgnore}
16421642
rctx.options.LoadThreshold = allocatorimpl.WithAllDims(0.05)
16431643
rctx.options.Deterministic = true
@@ -1799,7 +1799,7 @@ func TestStoreRebalancerIOOverloadCheck(t *testing.T) {
17991799
rctx := sr.NewRebalanceContext(ctx, options, hottestRanges, sr.RebalanceMode())
18001800
require.Greater(t, len(rctx.hottestRanges), 0)
18011801

1802-
rctx.options.IOOverloadOptions = allocatorimpl.IOOverloadOptions{
1802+
rctx.options.IOOverload = allocatorimpl.IOOverloadOptions{
18031803
ReplicaEnforcementLevel: test.enforcement, ReplicaIOOverloadThreshold: allocatorimpl.DefaultReplicaIOOverloadThreshold}
18041804
rctx.options.LoadThreshold = allocatorimpl.WithAllDims(0.05)
18051805

0 commit comments

Comments
 (0)