Skip to content

Commit 567b4a1

Browse files
ydbotydbot
authored
stable-25-3-1: Change expected slot count without pdisk restart (#25774)
2 parents dddaa2a + 1eb1585 commit 567b4a1

17 files changed

+405
-9
lines changed

‎ydb/core/base/blobstorage.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -764,6 +764,7 @@ struct TEvBlobStorage {
764764
EvReleaseSyncToken,
765765
EvBSQueueResetConnection, // for test purposes
766766
EvYardResize, // 268 636 340
767+
EvChangeExpectedSlotCount,
767768

768769
EvYardInitResult = EvPut + 9 * 512, /// 268 636 672
769770
EvLogResult,
@@ -820,6 +821,7 @@ struct TEvBlobStorage {
820821
EvYardResizeResult,
821822
EvCommitVDiskMetadata,
822823
EvCommitVDiskMetadataDone,
824+
EvChangeExpectedSlotCountResult,
823825

824826
// internal proxy interface
825827
EvUnusedLocal1 = EvPut + 10 * 512, // Not used. /// 268 637 184

‎ydb/core/blobstorage/nodewarden/node_warden_impl.cpp‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@ STATEFN(TNodeWarden::StateOnline) {
115115
hFunc(NPDisk::TEvSlayResult, Handle);
116116
hFunc(NPDisk::TEvShredPDiskResult, Handle);
117117
hFunc(NPDisk::TEvShredPDisk, Handle);
118+
hFunc(NPDisk::TEvChangeExpectedSlotCountResult, Handle);
118119

119120
hFunc(TEvRegisterPDiskLoadActor, Handle);
120121

@@ -630,6 +631,16 @@ void TNodeWarden::Handle(NPDisk::TEvShredPDiskResult::TPtr ev) {
630631
ev->Get()->Status)));
631632
}
632633

634+
void TNodeWarden::Handle(NPDisk::TEvChangeExpectedSlotCountResult::TPtr ev) {
635+
const NPDisk::TEvChangeExpectedSlotCountResult &msg = *ev->Get();
636+
STLOG(PRI_DEBUG, BS_NODE, NW108, "Handle(NPDisk::TEvChangeExpectedSlotCountResult)", (Msg, msg.ToString()));
637+
638+
// For now, just log the result. In the future, we might want to track this or take action based on the result.
639+
if (msg.Status != NKikimrProto::OK) {
640+
STLOG(PRI_ERROR, BS_NODE, NW109, "ChangeExpectedSlotCount failed", (Status, msg.Status), (ErrorReason, msg.ErrorReason));
641+
}
642+
}
643+
633644
void TNodeWarden::Handle(NPDisk::TEvShredPDisk::TPtr ev) {
634645
// the message has returned to sender -- PDisk was terminated before processing it; normally it must never happen,
635646
// because NodeWarden issues PoisonPill synchronously with removing PDisk from the LocalPDisks set

‎ydb/core/blobstorage/nodewarden/node_warden_impl.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -592,6 +592,7 @@ namespace NKikimr::NStorage {
592592
void Handle(NPDisk::TEvSlayResult::TPtr ev);
593593
void Handle(NPDisk::TEvShredPDiskResult::TPtr ev);
594594
void Handle(NPDisk::TEvShredPDisk::TPtr ev);
595+
void Handle(NPDisk::TEvChangeExpectedSlotCountResult::TPtr ev);
595596
void ProcessShredStatus(ui64 cookie, ui64 generation, std::optional<TString> error);
596597

597598
void PersistConfig(std::optional<TString> mainYaml, ui64 mainYamlVersion, std::optional<TString> storageYaml,

‎ydb/core/blobstorage/nodewarden/node_warden_pdisk.cpp‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -522,7 +522,35 @@ namespace NKikimr::NStorage {
522522

523523
auto processDisk = [&](const TServiceSetPDisk& pdisk) {
524524
const TPDiskKey key(pdisk);
525-
if (!LocalPDisks.contains(key)) {
525+
if (auto it = LocalPDisks.find(key); it != LocalPDisks.end()) {
526+
TPDiskRecord& localPDisk = it->second;
527+
TIntrusivePtr<TPDiskConfig> newPDiskConfig = CreatePDiskConfig(pdisk);
528+
const NKikimrBlobStorage::TPDiskConfig& oldPDiskConfig = localPDisk.Record.GetPDiskConfig();
529+
ui64 newExpectedSlotCount = newPDiskConfig->ExpectedSlotCount;
530+
ui64 oldExpectedSlotCount = oldPDiskConfig.GetExpectedSlotCount();
531+
ui32 newSlotSizeInUnits = newPDiskConfig->SlotSizeInUnits;
532+
ui32 oldSlotSizeInUnits = oldPDiskConfig.GetSlotSizeInUnits();
533+
STLOG(PRI_DEBUG, BS_NODE, NW110, "ApplyServiceSetPDisks",
534+
(PDiskId, key.PDiskId),
535+
(NewExpectedSlotCount, newExpectedSlotCount),
536+
(OldExpectedSlotCount, oldExpectedSlotCount),
537+
(NewSlotSizeInUnits, newSlotSizeInUnits),
538+
(OldSlotSizeInUnits, oldSlotSizeInUnits));
539+
if (newExpectedSlotCount != oldExpectedSlotCount ||
540+
newSlotSizeInUnits != oldSlotSizeInUnits) {
541+
STLOG(PRI_DEBUG, BS_NODE, NW107, "SendChangeExpectedSlotCount",
542+
(PDiskId, key.PDiskId),
543+
(ExpectedSlotCount, newExpectedSlotCount),
544+
(SlotSizeInUnits, newSlotSizeInUnits));
545+
546+
const TActorId pdiskActorId = MakeBlobStoragePDiskID(LocalNodeId, key.PDiskId);
547+
Send(pdiskActorId, new NPDisk::TEvChangeExpectedSlotCount(newExpectedSlotCount, newSlotSizeInUnits));
548+
549+
NKikimrBlobStorage::TPDiskConfig* pdiskConfig = localPDisk.Record.MutablePDiskConfig();
550+
pdiskConfig->SetExpectedSlotCount(newExpectedSlotCount);
551+
pdiskConfig->SetSlotSizeInUnits(newSlotSizeInUnits);
552+
}
553+
} else {
526554
StartLocalPDisk(pdisk, false);
527555
}
528556
pdiskToDelete.erase(key);

‎ydb/core/blobstorage/pdisk/blobstorage_pdisk.h‎

Lines changed: 52 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -308,6 +308,53 @@ struct TEvYardResizeResult : TEventLocal<TEvYardResizeResult, TEvBlobStorage::Ev
308308
}
309309
};
310310

311+
struct TEvChangeExpectedSlotCount : TEventLocal<TEvChangeExpectedSlotCount, TEvBlobStorage::EvChangeExpectedSlotCount> {
312+
ui64 ExpectedSlotCount;
313+
ui32 SlotSizeInUnits;
314+
315+
TEvChangeExpectedSlotCount(ui64 expectedSlotCount, ui32 slotSizeInUnits)
316+
: ExpectedSlotCount(expectedSlotCount)
317+
, SlotSizeInUnits(slotSizeInUnits)
318+
{}
319+
320+
TString ToString() const {
321+
return ToString(*this);
322+
}
323+
324+
static TString ToString(const TEvChangeExpectedSlotCount& record) {
325+
TStringStream str;
326+
str << "{";
327+
str << "EvChangeExpectedSlotCount ";
328+
str << " ExpectedSlotCount# " << record.ExpectedSlotCount;
329+
str << " SlotSizeInUnits# " << record.SlotSizeInUnits;
330+
str << "}";
331+
return str.Str();
332+
}
333+
};
334+
335+
struct TEvChangeExpectedSlotCountResult : TEventLocal<TEvChangeExpectedSlotCountResult, TEvBlobStorage::EvChangeExpectedSlotCountResult> {
336+
NKikimrProto::EReplyStatus Status;
337+
TString ErrorReason;
338+
339+
TEvChangeExpectedSlotCountResult(NKikimrProto::EReplyStatus status, TString errorReason)
340+
: Status(status)
341+
, ErrorReason(std::move(errorReason))
342+
{}
343+
344+
TString ToString() const {
345+
return ToString(*this);
346+
}
347+
348+
static TString ToString(const TEvChangeExpectedSlotCountResult& record) {
349+
TStringStream str;
350+
str << "{";
351+
str << "EvChangeExpectedSlotCountResult Status#" << NKikimrProto::EReplyStatus_Name(record.Status).data();
352+
str << " ErrorReason# \"" << record.ErrorReason << "\"";
353+
str << "}";
354+
return str.Str();
355+
}
356+
};
357+
311358
////////////////////////////////////////////////////////////////////////////
312359
// LOG
313360
////////////////////////////////////////////////////////////////////////////
@@ -1375,11 +1422,11 @@ struct TEvCheckSpace : TEventLocal<TEvCheckSpace, TEvBlobStorage::EvCheckSpace>
13751422
struct TEvCheckSpaceResult : TEventLocal<TEvCheckSpaceResult, TEvBlobStorage::EvCheckSpaceResult> {
13761423
NKikimrProto::EReplyStatus Status;
13771424
TStatusFlags StatusFlags;
1378-
ui32 FreeChunks;
1379-
ui32 TotalChunks; // contains common limit in shared free space mode, Total != Free + Used
1380-
ui32 UsedChunks; // number of chunks allocated by requesting owner
1381-
ui32 NumSlots; // number of VSlots over PDisk
1382-
ui32 NumActiveSlots; // $ \sum_i{ceil(VSlot[i].SlotSizeInUnits / PDisk.SlotSizeInUnits)} $
1425+
ui32 FreeChunks; // contains SharedQuota.Free
1426+
ui32 TotalChunks; // contains OwnerQuota.HardLimit(owner), Total != Free + Used
1427+
ui32 UsedChunks; // equals OwnerQuota.Used(owner) - a number of chunks allocated by requesting owner
1428+
ui32 NumSlots; // number of VDisks over PDisk, not their weight
1429+
ui32 NumActiveSlots; // sum of VDisks weights - $ \sum_i{ceil(VSlot[i].SlotSizeInUnits / PDisk.SlotSizeInUnits)} $
13831430
double Occupancy = 0;
13841431
TString ErrorReason;
13851432
TStatusFlags LogStatusFlags;

‎ydb/core/blobstorage/pdisk/blobstorage_pdisk_actor.cpp‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -704,6 +704,12 @@ class TPDiskActor : public TActorBootstrapped<TPDiskActor> {
704704
PDisk->Mon.YardResize.CountResponse();
705705
}
706706

707+
void InitHandle(NPDisk::TEvChangeExpectedSlotCount::TPtr &ev) {
708+
PDisk->Mon.ChangeExpectedSlotCount.CountRequest();
709+
Send(ev->Sender, new NPDisk::TEvChangeExpectedSlotCountResult(NKikimrProto::CORRUPTED, StateErrorReason));
710+
PDisk->Mon.ChangeExpectedSlotCount.CountResponse();
711+
}
712+
707713
void InitHandle(NPDisk::TEvShredPDisk::TPtr &ev) {
708714
const NPDisk::TEvShredPDisk &evShredPDisk = *ev->Get();
709715
InitQueue.emplace_back(ev->Sender, evShredPDisk.ShredGeneration, ev->Cookie);
@@ -848,6 +854,12 @@ class TPDiskActor : public TActorBootstrapped<TPDiskActor> {
848854
PDisk->Mon.YardResize.CountResponse();
849855
}
850856

857+
void ErrorHandle(NPDisk::TEvChangeExpectedSlotCount::TPtr &ev) {
858+
PDisk->Mon.ChangeExpectedSlotCount.CountRequest();
859+
Send(ev->Sender, new NPDisk::TEvChangeExpectedSlotCountResult(NKikimrProto::CORRUPTED, StateErrorReason));
860+
PDisk->Mon.ChangeExpectedSlotCount.CountResponse();
861+
}
862+
851863
void ErrorHandle(NPDisk::TEvChunkReserve::TPtr &ev) {
852864
PDisk->Mon.ChunkReserve.CountRequest();
853865
Send(ev->Sender, new NPDisk::TEvChunkReserveResult(NKikimrProto::CORRUPTED, 0, StateErrorReason));
@@ -994,6 +1006,12 @@ class TPDiskActor : public TActorBootstrapped<TPDiskActor> {
9941006
PDisk->InputRequest(request);
9951007
}
9961008

1009+
void Handle(NPDisk::TEvChangeExpectedSlotCount::TPtr &ev) {
1010+
PDisk->Mon.YardResize.CountRequest();
1011+
TChangeExpectedSlotCount* request = PDisk->ReqCreator.CreateFromEv<TChangeExpectedSlotCount>(*ev->Get(), ev->Sender);
1012+
PDisk->InputRequest(request);
1013+
}
1014+
9971015
void Handle(NPDisk::TEvChunkReserve::TPtr &ev) {
9981016
auto* request = PDisk->ReqCreator.CreateFromEv<TChunkReserve>(*ev->Get(), ev->Sender);
9991017
PDisk->InputRequest(request);
@@ -1499,6 +1517,7 @@ class TPDiskActor : public TActorBootstrapped<TPDiskActor> {
14991517
hFunc(NPDisk::TEvShredVDiskResult, InitHandle);
15001518
hFunc(NPDisk::TEvContinueShred, InitHandle);
15011519
hFunc(NPDisk::TEvYardResize, InitHandle);
1520+
hFunc(NPDisk::TEvChangeExpectedSlotCount, InitHandle);
15021521

15031522
hFunc(TEvReadMetadata, Handle);
15041523
hFunc(TEvWriteMetadata, Handle);
@@ -1532,6 +1551,7 @@ class TPDiskActor : public TActorBootstrapped<TPDiskActor> {
15321551
hFunc(NPDisk::TEvShredVDiskResult, Handle);
15331552
hFunc(NPDisk::TEvContinueShred, Handle);
15341553
hFunc(NPDisk::TEvYardResize, Handle);
1554+
hFunc(NPDisk::TEvChangeExpectedSlotCount, Handle);
15351555

15361556
cFunc(NActors::TEvents::TSystem::PoisonPill, HandlePoison);
15371557
hFunc(NMon::TEvHttpInfo, Handle);
@@ -1569,6 +1589,7 @@ class TPDiskActor : public TActorBootstrapped<TPDiskActor> {
15691589
hFunc(NPDisk::TEvShredVDiskResult, ErrorHandle);
15701590
hFunc(NPDisk::TEvContinueShred, ErrorHandle);
15711591
hFunc(NPDisk::TEvYardResize, ErrorHandle);
1592+
hFunc(NPDisk::TEvChangeExpectedSlotCount, ErrorHandle);
15721593

15731594
cFunc(NActors::TEvents::TSystem::PoisonPill, HandlePoison);
15741595
hFunc(NMon::TEvHttpInfo, Handle);

‎ydb/core/blobstorage/pdisk/blobstorage_pdisk_chunk_tracker.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -606,6 +606,10 @@ using TColor = NKikimrBlobStorage::TPDiskSpaceColor;
606606
}
607607
}
608608
}
609+
610+
void SetExpectedOwnerCount(size_t newOwnerCount) {
611+
OwnerQuota->SetExpectedOwnerCount(newOwnerCount);
612+
}
609613
};
610614

611615
} // NPDisk

‎ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2103,6 +2103,23 @@ void TPDisk::YardResize(TYardResize &ev) {
21032103
}
21042104
}
21052105

2106+
void TPDisk::ProcessChangeExpectedSlotCount(TChangeExpectedSlotCount& request) {
2107+
TGuard<TMutex> guard(StateMutex);
2108+
ExpectedSlotCount = request.ExpectedSlotCount;
2109+
Cfg->ExpectedSlotCount = request.ExpectedSlotCount;
2110+
Cfg->SlotSizeInUnits = request.SlotSizeInUnits;
2111+
Keeper.SetExpectedOwnerCount(ExpectedSlotCount);
2112+
for (TOwner owner = OwnerBeginUser; owner < OwnerEndUser; ++owner) {
2113+
if (OwnerData[owner].VDiskId != TVDiskID::InvalidId) {
2114+
Keeper.SetOwnerWeight(owner, Cfg->GetOwnerWeight(OwnerData[owner].GroupSizeInUnits));
2115+
}
2116+
}
2117+
2118+
auto result = std::make_unique<NPDisk::TEvChangeExpectedSlotCountResult>(NKikimrProto::OK, TString());
2119+
Mon.ChangeExpectedSlotCount.CountResponse();
2120+
PCtx->ActorSystem->Send(request.Sender, result.release());
2121+
}
2122+
21062123
// Scheduler weight configuration
21072124

21082125
void TPDisk::ConfigureCbs(ui32 ownerId, EGate gate, ui64 weight) {
@@ -2758,6 +2775,9 @@ void TPDisk::ProcessFastOperationsQueue() {
27582775
case ERequestType::RequestYardResize:
27592776
YardResize(static_cast<TYardResize&>(*req));
27602777
break;
2778+
case ERequestType::RequestChangeExpectedSlotCount:
2779+
ProcessChangeExpectedSlotCount(static_cast<TChangeExpectedSlotCount&>(*req));
2780+
break;
27612781
default:
27622782
Y_FAIL_S(PCtx->PDiskLogPrefix << "Unexpected request type# " << TypeName(*req));
27632783
break;
@@ -3387,6 +3407,7 @@ bool TPDisk::PreprocessRequest(TRequestBase *request) {
33873407
case ERequestType::RequestChunkShredResult:
33883408
case ERequestType::RequestContinueShred:
33893409
case ERequestType::RequestYardResize:
3410+
case ERequestType::RequestChangeExpectedSlotCount:
33903411
break;
33913412
case ERequestType::RequestStopDevice:
33923413
BlockDevice->Stop();
@@ -4017,6 +4038,7 @@ bool TPDisk::HandleReadOnlyIfWrite(TRequestBase *request) {
40174038
case ERequestType::RequestPushUnformattedMetadataSector:
40184039
case ERequestType::RequestContinueReadMetadata:
40194040
case ERequestType::RequestYardResize:
4041+
case ERequestType::RequestChangeExpectedSlotCount:
40204042
return false;
40214043

40224044
// Can't be processed in read-only mode.

‎ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -367,6 +367,7 @@ class TPDisk : public IPDisk {
367367
void YardInitFinish(TYardInit &evYardInit);
368368
bool YardInitForKnownVDisk(TYardInit &evYardInit, TOwner owner);
369369
void YardResize(TYardResize &evYardResize);
370+
void ProcessChangeExpectedSlotCount(TChangeExpectedSlotCount& request);
370371

371372
// Scheduler weight configuration
372373
void ConfigureCbs(ui32 ownerId, EGate gate, ui64 weight);

‎ydb/core/blobstorage/pdisk/blobstorage_pdisk_keeper.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,10 @@ class TKeeper {
176176
return ChunkTracker.ColorFlagLimit(owner, color);
177177
}
178178

179+
void SetExpectedOwnerCount(size_t newOwnerCount) {
180+
ChunkTracker.SetExpectedOwnerCount(newOwnerCount);
181+
}
182+
179183
//
180184
// GUI
181185
//

0 commit comments

Comments
 (0)