Skip to content

Commit 6806a43

Browse files
Evgenik2Evgenik2
authored
Add replicas specific volume option (#26305)
1 parent 48e4209 commit 6806a43

12 files changed

+60
-21
lines changed

‎ydb/core/blobstorage/nodewarden/distconf.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -472,6 +472,7 @@ namespace NKikimr::NStorage {
472472
, const NKikimrConfig::TDomainsConfig::TStateStorage& oldConfig = {}
473473
, ui32 overrideReplicasInRingCount = 0
474474
, ui32 overrideRingsCount = 0
475+
, ui32 replicasSpecificVolume = 200
475476
);
476477

477478
////////////////////////////////////////////////////////////////////////////////////////////////////////////////

‎ydb/core/blobstorage/nodewarden/distconf_generate.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -586,6 +586,7 @@ namespace NKikimr::NStorage {
586586
, const NKikimrConfig::TDomainsConfig::TStateStorage& oldConfig
587587
, ui32 overrideReplicasInRingCount
588588
, ui32 overrideRingsCount
589+
, ui32 replicasSpecificVolume
589590
) {
590591
std::map<TBridgePileId, THashMap<TString, std::vector<std::tuple<ui32, TNodeLocation>>>> nodes;
591592
bool goodConfig = true;
@@ -595,7 +596,7 @@ namespace NKikimr::NStorage {
595596
nodes[pileId][location.GetDataCenterId()].emplace_back(node.GetNodeId(), location);
596597
}
597598
for (auto& [pileId, nodesByDataCenter] : nodes) {
598-
TStateStoragePerPileGenerator generator(nodesByDataCenter, SelfHealNodesState, pileId, usedNodes, oldConfig, overrideReplicasInRingCount, overrideRingsCount);
599+
TStateStoragePerPileGenerator generator(nodesByDataCenter, SelfHealNodesState, pileId, usedNodes, oldConfig, overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume);
599600
generator.AddRingGroup(ss);
600601
goodConfig &= generator.IsGoodConfig();
601602
}

‎ydb/core/blobstorage/nodewarden/distconf_invoke.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,12 +107,12 @@ namespace NKikimr::NStorage {
107107
void ReassignStateStorageNode(const TQuery::TReassignStateStorageNode& cmd);
108108
void ReconfigStateStorage(const NKikimrBlobStorage::TStateStorageConfig& cmd);
109109
void SelfHealStateStorage(const TQuery::TSelfHealStateStorage& cmd);
110-
void SelfHealStateStorage(ui32 waitForConfigStep, bool forceHeal, bool pileupReplicas, ui32 overrideReplicasInRingCount, ui32 overrideRingsCount);
110+
void SelfHealStateStorage(ui32 waitForConfigStep, bool forceHeal, bool pileupReplicas, ui32 overrideReplicasInRingCount, ui32 overrideRingsCount, ui32 replicasSpecificVolume);
111111
void SelfHealNodesStateUpdate(const TQuery::TSelfHealNodesStateUpdate& cmd);
112112
void GetStateStorageConfig(const TQuery::TGetStateStorageConfig& cmd);
113113

114114
void GetCurrentStateStorageConfig(NKikimrBlobStorage::TStateStorageConfig* currentConfig, bool getNodesState);
115-
bool GetRecommendedStateStorageConfig(NKikimrBlobStorage::TStateStorageConfig* currentConfig, bool pileupReplicas, ui32 overrideReplicasInRingCount, ui32 overrideRingsCount);
115+
bool GetRecommendedStateStorageConfig(NKikimrBlobStorage::TStateStorageConfig* currentConfig, bool pileupReplicas, ui32 overrideReplicasInRingCount, ui32 overrideRingsCount, ui32 replicasSpecificVolume);
116116
void AdjustRingGroupActorIdOffsetInRecommendedStateStorageConfig(NKikimrBlobStorage::TStateStorageConfig* currentConfig);
117117
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
118118
// Storage configuration YAML manipulation

‎ydb/core/blobstorage/nodewarden/distconf_invoke_state_storage.cpp‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -9,19 +9,19 @@ namespace NKikimr::NStorage {
99

1010
using TInvokeRequestHandlerActor = TDistributedConfigKeeper::TInvokeRequestHandlerActor;
1111

12-
bool TInvokeRequestHandlerActor::GetRecommendedStateStorageConfig(NKikimrBlobStorage::TStateStorageConfig* currentConfig, bool pileupReplicas, ui32 overrideReplicasInRingCount, ui32 overrideRingsCount) {
12+
bool TInvokeRequestHandlerActor::GetRecommendedStateStorageConfig(NKikimrBlobStorage::TStateStorageConfig* currentConfig, bool pileupReplicas, ui32 overrideReplicasInRingCount, ui32 overrideRingsCount, ui32 replicasSpecificVolume) {
1313
const NKikimrBlobStorage::TStorageConfig& config = *Self->StorageConfig;
1414
bool result = true;
1515
std::unordered_set<ui32> usedNodes;
16-
result &= Self->GenerateStateStorageConfig(currentConfig->MutableStateStorageConfig(), config, usedNodes, config.GetStateStorageConfig(), overrideReplicasInRingCount, overrideRingsCount);
16+
result &= Self->GenerateStateStorageConfig(currentConfig->MutableStateStorageConfig(), config, usedNodes, config.GetStateStorageConfig(), overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume);
1717
if (pileupReplicas) {
1818
usedNodes.clear();
1919
}
20-
result &= Self->GenerateStateStorageConfig(currentConfig->MutableStateStorageBoardConfig(), config, usedNodes, config.GetStateStorageBoardConfig(), overrideReplicasInRingCount, overrideRingsCount);
20+
result &= Self->GenerateStateStorageConfig(currentConfig->MutableStateStorageBoardConfig(), config, usedNodes, config.GetStateStorageBoardConfig(), overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume);
2121
if (pileupReplicas) {
2222
usedNodes.clear();
2323
}
24-
result &= Self->GenerateStateStorageConfig(currentConfig->MutableSchemeBoardConfig(), config, usedNodes, config.GetSchemeBoardConfig(), overrideReplicasInRingCount, overrideRingsCount);
24+
result &= Self->GenerateStateStorageConfig(currentConfig->MutableSchemeBoardConfig(), config, usedNodes, config.GetSchemeBoardConfig(), overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume);
2525
return result;
2626
}
2727

@@ -97,7 +97,7 @@ namespace NKikimr::NStorage {
9797
auto* currentConfig = record->MutableStateStorageConfig();
9898

9999
if (cmd.GetRecommended()) {
100-
GetRecommendedStateStorageConfig(currentConfig, cmd.GetPileupReplicas(), cmd.GetOverrideReplicasInRingCount(), cmd.GetOverrideRingsCount());
100+
GetRecommendedStateStorageConfig(currentConfig, cmd.GetPileupReplicas(), cmd.GetOverrideReplicasInRingCount(), cmd.GetOverrideRingsCount(), cmd.GetReplicasSpecificVolume());
101101
AdjustRingGroupActorIdOffsetInRecommendedStateStorageConfig(currentConfig);
102102
} else {
103103
GetCurrentStateStorageConfig(currentConfig, cmd.GetNodesState());
@@ -112,20 +112,20 @@ namespace NKikimr::NStorage {
112112
Self->SelfHealNodesState[node.GetNodeId()] = node.GetState();
113113
}
114114
if (cmd.GetEnableSelfHealStateStorage()) {
115-
SelfHealStateStorage(cmd.GetWaitForConfigStep(), true, cmd.GetPileupReplicas(), cmd.GetOverrideReplicasInRingCount(), cmd.GetOverrideRingsCount());
115+
SelfHealStateStorage(cmd.GetWaitForConfigStep(), true, cmd.GetPileupReplicas(), cmd.GetOverrideReplicasInRingCount(), cmd.GetOverrideRingsCount(), cmd.GetReplicasSpecificVolume());
116116
}
117117
}
118118

119119
void TInvokeRequestHandlerActor::SelfHealStateStorage(const TQuery::TSelfHealStateStorage& cmd) {
120-
SelfHealStateStorage(cmd.GetWaitForConfigStep(), cmd.GetForceHeal(), cmd.GetPileupReplicas(), cmd.GetOverrideReplicasInRingCount(), cmd.GetOverrideRingsCount());
120+
SelfHealStateStorage(cmd.GetWaitForConfigStep(), cmd.GetForceHeal(), cmd.GetPileupReplicas(), cmd.GetOverrideReplicasInRingCount(), cmd.GetOverrideRingsCount(), cmd.GetReplicasSpecificVolume());
121121
}
122122

123-
void TInvokeRequestHandlerActor::SelfHealStateStorage(ui32 waitForConfigStep, bool forceHeal, bool pileupReplicas, ui32 overrideReplicasInRingCount, ui32 overrideRingsCount) {
123+
void TInvokeRequestHandlerActor::SelfHealStateStorage(ui32 waitForConfigStep, bool forceHeal, bool pileupReplicas, ui32 overrideReplicasInRingCount, ui32 overrideRingsCount, ui32 replicasSpecificVolume) {
124124
RunCommonChecks();
125125
STLOG(PRI_DEBUG, BS_NODE, NW105, "TInvokeRequestHandlerActor::SelfHealStateStorage", (waitForConfigStep, waitForConfigStep),
126-
(forceHeal, forceHeal), (pileupReplicas, pileupReplicas), (overrideReplicasInRingCount, overrideReplicasInRingCount), (overrideRingsCount, overrideRingsCount));
126+
(forceHeal, forceHeal), (pileupReplicas, pileupReplicas), (overrideReplicasInRingCount, overrideReplicasInRingCount), (overrideRingsCount, overrideRingsCount), (replicasSpecificVolume, replicasSpecificVolume));
127127
NKikimrBlobStorage::TStateStorageConfig targetConfig;
128-
if (!GetRecommendedStateStorageConfig(&targetConfig, pileupReplicas, overrideReplicasInRingCount, overrideRingsCount) && !forceHeal) {
128+
if (!GetRecommendedStateStorageConfig(&targetConfig, pileupReplicas, overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume) && !forceHeal) {
129129
throw TExError() << "Recommended configuration has faulty nodes and can not be applyed";
130130
}
131131

‎ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.cpp‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#include <library/cpp/streams/zstd/zstd.h>
1010

1111
namespace NKikimr::NStorage {
12+
constexpr ui32 defaultReplicasSpecificVolume = 200;
1213

1314
TStateStoragePerPileGenerator::TStateStoragePerPileGenerator(THashMap<TString, std::vector<std::tuple<ui32, TNodeLocation>>>& nodes
1415
, const std::unordered_map<ui32, ui32>& selfHealNodesState
@@ -17,13 +18,15 @@ namespace NKikimr::NStorage {
1718
, const NKikimrConfig::TDomainsConfig::TStateStorage& oldConfig
1819
, ui32 overrideReplicasInRingCount
1920
, ui32 overrideRingsCount
21+
, ui32 replicasSpecificVolume
2022
)
2123
: PileId(pileId)
2224
, SelfHealNodesState(selfHealNodesState)
2325
, UsedNodes(usedNodes)
2426
, OldConfig(oldConfig)
2527
, OverrideReplicasInRingCount(overrideReplicasInRingCount)
2628
, OverrideRingsCount(overrideRingsCount)
29+
, ReplicasSpecificVolume(replicasSpecificVolume == 0 ? defaultReplicasSpecificVolume : replicasSpecificVolume)
2730
{
2831
FillNodeGroups(nodes);
2932
CalculateRingsParameters();
@@ -67,7 +70,7 @@ namespace NKikimr::NStorage {
6770
RingsInGroupCount = minNodesInGroup;
6871
}
6972
NToSelect = RingsInGroupCount < 3 ? 1 : (RingsInGroupCount < 5 ? 3 : 5);
70-
ReplicasInRingCount = OverrideReplicasInRingCount > 0 ? OverrideReplicasInRingCount : (1 + minNodesInGroup / 1000);
73+
ReplicasInRingCount = OverrideReplicasInRingCount > 0 ? OverrideReplicasInRingCount : (1 + minNodesInGroup / ReplicasSpecificVolume);
7174
} else {
7275
if (OverrideRingsCount == 3 || OverrideRingsCount == 9) {
7376
RingsInGroupCount = OverrideRingsCount / 3;
@@ -79,7 +82,7 @@ namespace NKikimr::NStorage {
7982
for (auto& n : NodeGroups) {
8083
nodesCnt += n.Nodes.size();
8184
}
82-
ReplicasInRingCount = OverrideReplicasInRingCount > 0 ? OverrideReplicasInRingCount : (1 + nodesCnt / 1000);
85+
ReplicasInRingCount = OverrideReplicasInRingCount > 0 ? OverrideReplicasInRingCount : (1 + nodesCnt / ReplicasSpecificVolume);
8386
}
8487
if (ReplicasInRingCount * RingsInGroupCount > minNodesInGroup) {
8588
ReplicasInRingCount = 1;

‎ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@ namespace NKikimr::NStorage {
1414
std::unordered_set<ui32>& usedNodes,
1515
const NKikimrConfig::TDomainsConfig::TStateStorage& oldConfig,
1616
ui32 overrideReplicasInRingCount,
17-
ui32 overrideRingsCount
17+
ui32 overrideRingsCount,
18+
ui32 replicasSpecificVolume
1819
);
1920
bool IsGoodConfig() const;
2021
void AddRingGroup(NKikimrConfig::TDomainsConfig::TStateStorage *ss);
@@ -46,5 +47,6 @@ namespace NKikimr::NStorage {
4647
ui32 NToSelect = 1;
4748
ui32 OverrideReplicasInRingCount = 0;
4849
ui32 OverrideRingsCount = 0;
50+
ui32 ReplicasSpecificVolume = 200;
4951
};
5052
}

‎ydb/core/blobstorage/nodewarden/distconf_ut.cpp‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,15 @@ namespace NBlobStorageNodeWardenTest{
2323

2424
Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) {
2525

26-
NKikimrConfig::TDomainsConfig::TStateStorage GenerateSimpleStateStorage(ui32 nodes, std::unordered_set<ui32> usedNodes = {}, ui32 overrideReplicasInRingCount = 0, ui32 overrideRingsCount = 0) {
26+
NKikimrConfig::TDomainsConfig::TStateStorage GenerateSimpleStateStorage(ui32 nodes, std::unordered_set<ui32> usedNodes = {}, ui32 overrideReplicasInRingCount = 0, ui32 overrideRingsCount = 0, ui32 replicasSpecificVolume = 200) {
2727
NKikimr::NStorage::TDistributedConfigKeeper keeper(nullptr, nullptr, true);
2828
NKikimrConfig::TDomainsConfig::TStateStorage ss;
2929
NKikimrBlobStorage::TStorageConfig config;
3030
for (ui32 i : xrange(nodes)) {
3131
auto *node = config.AddAllNodes();
3232
node->SetNodeId(i + 1);
3333
}
34-
keeper.GenerateStateStorageConfig(&ss, config, usedNodes, {}, overrideReplicasInRingCount, overrideRingsCount);
34+
keeper.GenerateStateStorageConfig(&ss, config, usedNodes, {}, overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume);
3535
return ss;
3636
}
3737

@@ -45,6 +45,7 @@ Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) {
4545
, ui32 oldNToSelect = 9
4646
, ui32 overrideReplicasInRingCount = 0
4747
, ui32 overrideRingsCount = 0
48+
, ui32 replicasSpecificVolume = 1000
4849
) {
4950
NKikimrBlobStorage::TStorageConfig config;
5051
ui32 nodeId = 1;
@@ -73,7 +74,7 @@ Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) {
7374
for (auto [nodeId, state] : nodesState) {
7475
keeper.SelfHealNodesState[nodeId] = state;
7576
}
76-
keeper.GenerateStateStorageConfig(&ss, config, usedNodes, oldSS, overrideReplicasInRingCount, overrideRingsCount);
77+
keeper.GenerateStateStorageConfig(&ss, config, usedNodes, oldSS, overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume);
7778
return ss;
7879
}
7980

@@ -184,7 +185,7 @@ Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) {
184185
CheckStateStorage(GenerateDCStateStorage(3, 3, 3, { {10, 2}, {11, 4}, {13, 3}, {14, 4}, {15, 2} }, {}, {1, 5, 8, 10, 14, 17, 19, 22, 25}, 9), 9, {1, 4, 7, 10, 14, 17, 19, 22, 25});
185186
}
186187

187-
Y_UNIT_TEST(GenerateConfigReplicasDensity) {
188+
Y_UNIT_TEST(GenerateConfigReplicasOverrides) {
188189
CheckStateStorage(GenerateSimpleStateStorage(100, {}, 0, 0), 5, {1, 2, 3, 4, 5, 6, 7, 8});
189190
CheckStateStorage(GenerateSimpleStateStorage(100, {}, 1, 1), 1, {1});
190191
CheckStateStorage2(GenerateSimpleStateStorage(100, {}, 3, 3),
@@ -201,6 +202,13 @@ Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) {
201202
"Ring { Node: 10 Node: 11 } Ring { Node: 13 Node: 14 } Ring { Node: 16 Node: 17 } "
202203
"Ring { Node: 19 Node: 20 } Ring { Node: 22 Node: 23 } Ring { Node: 25 Node: 26 } } }");
203204
}
205+
206+
Y_UNIT_TEST(GenerateConfigReplicasSpecificVolume) {
207+
CheckStateStorage2(GenerateSimpleStateStorage(100, {}, 0, 3, 35),
208+
"{ RingGroups { NToSelect: 3 Ring { Node: 1 Node: 2 Node: 3 }"
209+
" Ring { Node: 4 Node: 5 Node: 6 } Ring { Node: 7 Node: 8 Node: 9 } } }");
210+
CheckStateStorage2(GenerateSimpleStateStorage(16, {}, 0, 2, 5), "{ RingGroups { NToSelect: 1 Ring { Node: 1 Node: 2 Node: 3 Node: 4 } Ring { Node: 5 Node: 6 Node: 7 Node: 8 } } }");
211+
}
204212
}
205213
}
206214
}

‎ydb/core/cms/config.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ struct TCmsSentinelConfig {
2323
bool PileupReplicas;
2424
ui32 OverrideReplicasInRingCount;
2525
ui32 OverrideRingsCount;
26+
ui32 ReplicasSpecificVolume;
2627

2728
void Serialize(NKikimrCms::TCmsConfig::TSentinelConfig::TStateStorageSelfHealConfig &config) const {
2829
config.SetEnable(Enable);
@@ -34,6 +35,7 @@ struct TCmsSentinelConfig {
3435
config.SetPileupReplicas(PileupReplicas);
3536
config.SetOverrideReplicasInRingCount(OverrideReplicasInRingCount);
3637
config.SetOverrideRingsCount(OverrideRingsCount);
38+
config.SetReplicasSpecificVolume(ReplicasSpecificVolume);
3739
}
3840

3941
void Deserialize(const NKikimrCms::TCmsConfig::TSentinelConfig::TStateStorageSelfHealConfig &config) {
@@ -46,6 +48,7 @@ struct TCmsSentinelConfig {
4648
PileupReplicas = config.GetPileupReplicas();
4749
OverrideReplicasInRingCount = config.GetOverrideReplicasInRingCount();
4850
OverrideRingsCount = config.GetOverrideRingsCount();
51+
ReplicasSpecificVolume = config.GetReplicasSpecificVolume();
4952
}
5053
};
5154

‎ydb/core/cms/sentinel.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1153,6 +1153,7 @@ class TSentinel: public TActorBootstrapped<TSentinel> {
11531153
updateRequest->SetPileupReplicas(Config.StateStorageSelfHealConfig.PileupReplicas);
11541154
updateRequest->SetOverrideReplicasInRingCount(Config.StateStorageSelfHealConfig.OverrideReplicasInRingCount);
11551155
updateRequest->SetOverrideRingsCount(Config.StateStorageSelfHealConfig.OverrideRingsCount);
1156+
updateRequest->SetReplicasSpecificVolume(Config.StateStorageSelfHealConfig.ReplicasSpecificVolume);
11561157
for (auto& [nodeId, node] : SentinelState->Nodes) {
11571158
SentinelState->NeedSelfHealStateStorage |= node.Compute();
11581159
auto* nodeState = updateRequest->AddNodesState();

‎ydb/core/protos/blobstorage_distributed_config.proto‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,7 @@ message TEvNodeConfigInvokeOnRoot {
227227
optional bool NodesState = 3;
228228
optional uint32 OverrideReplicasInRingCount = 4;
229229
optional uint32 OverrideRingsCount = 5;
230+
optional uint32 ReplicasSpecificVolume = 6;
230231
}
231232

232233
message TSelfHealStateStorage {
@@ -235,6 +236,7 @@ message TEvNodeConfigInvokeOnRoot {
235236
optional bool PileupReplicas = 3;
236237
optional uint32 OverrideReplicasInRingCount = 4;
237238
optional uint32 OverrideRingsCount = 5;
239+
optional uint32 ReplicasSpecificVolume = 6;
238240
}
239241

240242
message TSelfHealNodesStateUpdate {
@@ -248,6 +250,7 @@ message TEvNodeConfigInvokeOnRoot {
248250
optional bool PileupReplicas = 4;
249251
optional uint32 OverrideReplicasInRingCount = 5;
250252
optional uint32 OverrideRingsCount = 6;
253+
optional uint32 ReplicasSpecificVolume = 7;
251254
}
252255

253256
message TAdvanceGeneration

0 commit comments

Comments
 (0)