From f2a87182c28ec5953f1d91f548ee68c5e5a858b2 Mon Sep 17 00:00:00 2001 From: Kirill Pleshivtsev Date: Thu, 6 Nov 2025 16:03:59 +0700 Subject: [PATCH] Unification of naming for index of the device in the partition config --- .../part_nonrepl_actor_base_request.cpp | 2 +- .../part_nonrepl_events_private.h | 7 +++-- .../part_nonrepl_rdma_actor.cpp | 28 +++++++++---------- .../part_nonrepl_rdma_actor.h | 25 ++++++++++------- ...part_nonrepl_rdma_actor_checksumblocks.cpp | 21 ++++++++++---- .../part_nonrepl_rdma_actor_writeblocks.cpp | 26 +++++++++-------- ...epl_rdma_actor_writeblocks_multi_agent.cpp | 10 +++---- .../part_nonrepl_rdma_actor_zeroblocks.cpp | 6 ++-- .../rdma_device_request_handler.h | 11 ++++++-- 9 files changed, 80 insertions(+), 56 deletions(-) diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_actor_base_request.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_actor_base_request.cpp index 831196a212e..45e4d294d66 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_actor_base_request.cpp +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_actor_base_request.cpp @@ -108,7 +108,7 @@ void TDiskAgentBaseRequestActor::Done( for (const auto& dr: DeviceRequests) { completion.Body->RequestResults.push_back( - {.DeviceIndex = dr.DeviceIdx, .Error = {}}); + {.DeviceIdx = dr.DeviceIdx, .Error = {}}); } NCloud::Send(ctx, Part, std::move(completion.Event)); diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_events_private.h b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_events_private.h index b8e20a7eaaa..bab01de4864 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_events_private.h +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_events_private.h @@ -238,9 +238,10 @@ struct TEvNonreplPartitionPrivate struct TDeviceRequestResult { - // Index of device that participated in the request and the - // result of the request for that device. - ui32 DeviceIndex = 0; + // Index of the device in the partition config. + ui32 DeviceIdx = 0; + + // Result of the request for that device. NProto::TError Error; }; diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp index e93063b9a74..e975183eef1 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.cpp @@ -323,13 +323,13 @@ TNonreplicatedPartitionRdmaActor::SendReadRequests( const auto& r = deviceRequests[i]; auto ep = AgentId2Endpoint[r.Device.GetAgentId()]; Y_ABORT_UNLESS(ep); - auto dr = std::make_unique(); + auto dr = std::make_unique( + r.DeviceIdx, + startBlockIndexOffset, + r.DeviceBlockRange.Size(), + i); ui64 sz = r.DeviceBlockRange.Size() * PartConfig->GetBlockSize(); - dr->StartIndexOffset = startBlockIndexOffset; - dr->BlockCount = r.DeviceBlockRange.Size(); - dr->DeviceIdx = r.DeviceIdx; - dr->RequestIndex = i; startBlockIndexOffset += r.DeviceBlockRange.Size(); sentRequestCtx.emplace_back(r.DeviceIdx); @@ -368,7 +368,8 @@ TNonreplicatedPartitionRdmaActor::SendReadRequests( flags, deviceRequest); - requests.push_back({std::move(ep), std::move(req)}); + requests.push_back( + {.Endpoint = std::move(ep), .ClientRequest = std::move(req)}); } for (size_t i = 0; i < requests.size(); ++i) { @@ -726,24 +727,23 @@ void TNonreplicatedPartitionRdmaActor::HandleAgentIsUnavailable( const auto& requestCtx = requestInfo.Value; bool needToCancel = AnyOf( requestCtx, - [&](const auto& ctx) + [&](const TRunningRdmaRequestInfo& item) { - return laggingRows.contains(ctx.DeviceIndex) && + return laggingRows.contains(item.DeviceIdx) && (requestInfo.IsWrite || - devices[ctx.DeviceIndex].GetAgentId() == - laggingAgentId); + devices[item.DeviceIdx].GetAgentId() == laggingAgentId); }); if (!needToCancel) { continue; } - for (auto [deviceIdx, rdmaRequestId]: requestCtx) { - Y_ABORT_UNLESS(deviceIdx < static_cast(devices.size())); - auto agentId = devices[deviceIdx].GetAgentId(); + for (const TRunningRdmaRequestInfo& item: requestCtx) { + Y_ABORT_UNLESS(item.DeviceIdx < static_cast(devices.size())); + const auto& agentId = devices[item.DeviceIdx].GetAgentId(); auto& endpoint = AgentId2Endpoint[agentId]; - endpoint->CancelRequest(rdmaRequestId); + endpoint->CancelRequest(item.SentRequestId); } } } diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.h b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.h index 52a4e781ba8..7ce61d3a36a 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.h +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor.h @@ -31,9 +31,20 @@ namespace NCloud::NBlockStore::NStorage { struct TDeviceReadRequestContext: public TDeviceRequestRdmaContext { - ui64 StartIndexOffset = 0; - ui64 BlockCount = 0; - ui32 RequestIndex = 0; + const ui64 StartIndexOffset = 0; + const ui64 BlockCount = 0; + const ui32 RequestIndex = 0; + + TDeviceReadRequestContext( + ui32 deviceIdx, + ui64 startIndexOffset, + ui64 blockCount, + ui32 requestIndex) + : TDeviceRequestRdmaContext(deviceIdx) + , StartIndexOffset(startIndexOffset) + , BlockCount(blockCount) + , RequestIndex(requestIndex) + {} }; //////////////////////////////////////////////////////////////////////////////// @@ -41,13 +52,7 @@ struct TDeviceReadRequestContext: public TDeviceRequestRdmaContext class TNonreplicatedPartitionRdmaActor final : public NActors::TActorBootstrapped { - struct TDeviceRequestContext - { - ui32 DeviceIndex = 0; - ui64 SentRequestId = 0; - }; - - using TRequestContext = TStackVec; + using TRequestContext = TStackVec; private: const TStorageConfigPtr Config; diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_checksumblocks.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_checksumblocks.cpp index 72381780b6b..3124fe2df3a 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_checksumblocks.cpp +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_checksumblocks.cpp @@ -25,8 +25,17 @@ namespace { struct TDeviceChecksumRequestContext: public TDeviceRequestRdmaContext { - ui64 RangeStartIndex = 0; - ui32 RangeSize = 0; + const ui64 RangeStartIndex = 0; + const ui32 RangeSize = 0; + + TDeviceChecksumRequestContext( + ui32 deviceIdx, + ui64 rangeStartIndex, + ui32 rangeSize) + : TDeviceRequestRdmaContext(deviceIdx) + , RangeStartIndex(rangeStartIndex) + , RangeSize(rangeSize) + {} }; //////////////////////////////////////////////////////////////////////////////// @@ -160,10 +169,10 @@ void TNonreplicatedPartitionRdmaActor::HandleChecksumBlocks( for (auto& r: deviceRequests) { auto ep = AgentId2Endpoint[r.Device.GetAgentId()]; Y_ABORT_UNLESS(ep); - auto dc = std::make_unique(); - dc->RangeStartIndex = r.BlockRange.Start; - dc->RangeSize = r.DeviceBlockRange.Size() * PartConfig->GetBlockSize(); - dc->DeviceIdx = r.DeviceIdx; + auto dc = std::make_unique( + r.DeviceIdx, + r.BlockRange.Start, + r.DeviceBlockRange.Size() * PartConfig->GetBlockSize()); sentRequestCtx.emplace_back(r.DeviceIdx); diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_writeblocks.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_writeblocks.cpp index 349b42e6263..e328a0b6024 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_writeblocks.cpp +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_writeblocks.cpp @@ -245,10 +245,11 @@ void TNonreplicatedPartitionRdmaActor::HandleWriteBlocks( PartConfig, AssignIdToWriteAndZeroRequestsEnabled); - auto context = std::make_unique(); - context->DeviceIdx = deviceRequest.DeviceIdx; + auto context = std::make_unique( + deviceRequest.DeviceIdx); - sentRequestCtx.emplace_back(deviceRequest.DeviceIdx); + sentRequestCtx.emplace_back( + TRunningRdmaRequestInfo{.DeviceIdx = deviceRequest.DeviceIdx}); auto [req, err] = ep->AllocateRequest( requestResponseHandler, @@ -271,12 +272,11 @@ void TNonreplicatedPartitionRdmaActor::HandleWriteBlocks( ctx, deviceRequest.Device.GetDeviceUUID()); - using TResponse = TEvService::TEvWriteBlocksResponse; NCloud::Reply( ctx, *requestInfo, - std::make_unique(std::move(err))); - + std::make_unique( + std::move(err))); return; } @@ -295,7 +295,8 @@ void TNonreplicatedPartitionRdmaActor::HandleWriteBlocks( request, sglist); - requests.push_back({std::move(ep), std::move(req)}); + requests.push_back( + {.Endpoint = std::move(ep), .ClientRequest = std::move(req)}); } for (size_t i = 0; i < requests.size(); ++i) { @@ -419,8 +420,8 @@ void TNonreplicatedPartitionRdmaActor::HandleWriteBlocksLocal( PartConfig, AssignIdToWriteAndZeroRequestsEnabled); - auto context = std::make_unique(); - context->DeviceIdx = deviceRequest.DeviceIdx; + auto context = std::make_unique( + deviceRequest.DeviceIdx); auto [req, err] = ep->AllocateRequest( requestResponseHandler, @@ -443,11 +444,11 @@ void TNonreplicatedPartitionRdmaActor::HandleWriteBlocksLocal( ctx, deviceRequest.Device.GetDeviceUUID()); - using TResponse = TEvService::TEvWriteBlocksLocalResponse; NCloud::Reply( ctx, *requestInfo, - std::make_unique(std::move(err))); + std::make_unique( + std::move(err))); return; } @@ -468,7 +469,8 @@ void TNonreplicatedPartitionRdmaActor::HandleWriteBlocksLocal( blocks += deviceRequest.DeviceBlockRange.Size(); - requests.push_back({std::move(ep), std::move(req)}); + requests.push_back( + {.Endpoint = std::move(ep), .ClientRequest = std::move(req)}); } for (size_t i = 0; i < requests.size(); ++i) { diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_writeblocks_multi_agent.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_writeblocks_multi_agent.cpp index 189dcd1f859..029edc7c913 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_writeblocks_multi_agent.cpp +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_writeblocks_multi_agent.cpp @@ -302,11 +302,11 @@ void TNonreplicatedPartitionRdmaActor::HandleMultiAgentWrite( const auto sentRequestId = ep->SendRequest(std::move(req), requestInfo->CallContext); - RequestsInProgress.AddWriteRequest( - requestId, - TRequestContext{TDeviceRequestContext{ - .DeviceIndex = deviceRequest.DeviceIdx, - .SentRequestId = sentRequestId}}); + TRequestContext sentRequestCtx{ + {.DeviceIdx = deviceRequest.DeviceIdx, + .SentRequestId = sentRequestId}}; + + RequestsInProgress.AddWriteRequest(requestId, sentRequestCtx); } } // namespace NCloud::NBlockStore::NStorage diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_zeroblocks.cpp b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_zeroblocks.cpp index 5534e0dad1d..eae4e88463a 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_zeroblocks.cpp +++ b/cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_zeroblocks.cpp @@ -129,8 +129,7 @@ void TNonreplicatedPartitionRdmaActor::HandleZeroBlocks( msg->Record.GetHeaders().GetVolumeRequestId()); } - auto context = std::make_unique(); - context->DeviceIdx = r.DeviceIdx; + auto context = std::make_unique(r.DeviceIdx); auto [req, err] = ep->AllocateRequest( requestContext, @@ -160,7 +159,8 @@ void TNonreplicatedPartitionRdmaActor::HandleZeroBlocks( 0, // flags deviceRequest); - requests.push_back({std::move(ep), std::move(req)}); + requests.push_back( + {.Endpoint = std::move(ep), .ClientRequest = std::move(req)}); } for (size_t i = 0; i < requests.size(); ++i) { diff --git a/cloud/blockstore/libs/storage/partition_nonrepl/rdma_device_request_handler.h b/cloud/blockstore/libs/storage/partition_nonrepl/rdma_device_request_handler.h index d9b5e213834..9670589f175 100644 --- a/cloud/blockstore/libs/storage/partition_nonrepl/rdma_device_request_handler.h +++ b/cloud/blockstore/libs/storage/partition_nonrepl/rdma_device_request_handler.h @@ -18,14 +18,21 @@ namespace NCloud::NBlockStore::NStorage { struct TDeviceRequestRdmaContext: public NRdma::TNullContext { - ui32 DeviceIdx = 0; + // Index of the device in the partition config. + const ui32 DeviceIdx; - TDeviceRequestRdmaContext() = default; explicit TDeviceRequestRdmaContext(ui32 deviceIdx) : DeviceIdx(deviceIdx) {} }; +struct TRunningRdmaRequestInfo +{ + // Index of the device in the partition config. + ui32 DeviceIdx = 0; + ui64 SentRequestId = 0; +}; + //////////////////////////////////////////////////////////////////////////////// template