Fix possible corruption in RDMA partition during migration; Add loadtests (#4580)

#2988
Fixing a similar problem to the one in
#3241, but this time RDMA-only.
Enabling data integrity feature in local-mirror* loadtests.

Author: komarevtsev-d

cloud/blockstore/libs/rdma/fake/client.cpp

@@ -305,46 +305,46 @@ class TExecuteRequestActor final
 private:
     NProto::TError ExecuteRequest(const TActorContext& ctx)
     {
-        auto [proto, error] =
+        auto [res, error] =
             TBlockStoreProtocol::Serializer()->Parse(Request->RequestBuffer);
         if (HasError(error)) {
             return error;
         }
 
-        switch (proto.MsgId) {
+        switch (res.MsgId) {
             case TBlockStoreProtocol::ReadDeviceBlocksRequest:
-                Y_ABORT_IF(proto.Data);
+                Y_ABORT_IF(res.Data);
                 return SendReadBlocksRequest(
                     ctx,
                     std::move(static_cast<NProto::TReadDeviceBlocksRequest&>(
-                        *proto.Proto)));
+                        *res.Proto)));
 
             case TBlockStoreProtocol::WriteDeviceBlocksRequest:
                 return SendWriteBlocksRequest(
                     ctx,
                     std::move(static_cast<NProto::TWriteDeviceBlocksRequest&>(
-                        *proto.Proto)),
-                    proto.Data);
+                        *res.Proto)),
+                    res.Data);
 
             case TBlockStoreProtocol::ZeroDeviceBlocksRequest:
-                Y_ABORT_IF(proto.Data);
+                Y_ABORT_IF(res.Data);
                 return SendZeroBlocksRequest(
                     ctx,
                     std::move(static_cast<NProto::TZeroDeviceBlocksRequest&>(
-                        *proto.Proto)));
+                        *res.Proto)));
 
             case TBlockStoreProtocol::ChecksumDeviceBlocksRequest:
-                Y_ABORT_IF(proto.Data);
+                Y_ABORT_IF(res.Data);
                 return SendChecksumBlocksRequest(
                     ctx,
                     std::move(
                         static_cast<NProto::TChecksumDeviceBlocksRequest&>(
-                            *proto.Proto)));
+                            *res.Proto)));
 
             default:
                 return MakeError(
                     E_NOT_IMPLEMENTED,
-                    TStringBuilder() << "MsgId: " << proto.MsgId);
+                    TStringBuilder() << "MsgId: " << res.MsgId);
         }
     }
 

cloud/blockstore/libs/storage/partition_nonrepl/part_nonrepl_rdma_actor_writeblocks.cpp

@@ -405,7 +405,8 @@ void TNonreplicatedPartitionRdmaActor::HandleWriteBlocksLocal(
         }
     }
 
-    ui64 blocks = 0;
+    ui64 blocks =
+        deviceRequests[0].BlockRange.Start - msg->Record.GetStartIndex();
     for (const auto& deviceRequest: deviceRequests) {
         auto ep = AgentId2Endpoint[deviceRequest.Device.GetAgentId()];
         Y_ABORT_UNLESS(ep);

cloud/blockstore/tests/loadtest/local-data-integrity/test.py

@@ -8,7 +8,8 @@
 from cloud.blockstore.config.server_pb2 import TServerConfig, TServerAppConfig, \
     TKikimrServiceConfig, TChecksumFlags
 from cloud.blockstore.config.storage_pb2 import TStorageServiceConfig
-from cloud.blockstore.public.sdk.python.protos import STORAGE_MEDIA_SSD, STORAGE_MEDIA_SSD_MIRROR2
+from cloud.blockstore.public.sdk.python.protos import STORAGE_MEDIA_SSD, \
+    STORAGE_MEDIA_SSD_MIRROR2, DIVP_ENABLED_FORCED
 
 from cloud.blockstore.tests.python.lib.disk_agent_runner import LocalDiskAgent
 from cloud.blockstore.tests.python.lib.nbs_runner import LocalNbs
@@ -364,7 +365,7 @@ def __run_test(test_case):
                 devices_per_agent,
                 disk_agent_config_patch=TDiskAgentConfig(
                     DedicatedDiskAgent=True,
-                    EnableDataIntegrityValidationForDrBasedDisks=True,
+                    DataIntegrityValidationPolicyForDrBasedDisks=DIVP_ENABLED_FORCED,
                     ChaosConfig=chaos_config),
                 agent_count=test_case.agent_count,
             )

cloud/blockstore/tests/loadtest/local-mirror-lagging/local-mirror2-basic.txt

@@ -5,6 +5,11 @@ Vertices {
             BlockSize: 4096
             StorageMediaKind: STORAGE_MEDIA_SSD_MIRROR2
         }
+        StartEndpointRequest {
+            IpcType: IPC_GRPC
+            VolumeAccessMode: VOLUME_ACCESS_READ_WRITE
+            VolumeMountMode: VOLUME_MOUNT_LOCAL
+        }
         ArtificialLoadSpec {
             Ranges {
                 Start: 261144

cloud/blockstore/tests/loadtest/local-mirror-lagging/local-mirror2-fresh-device-migration.txt

@@ -36,7 +36,11 @@ Vertices {
 Vertices {
     Test {
         VolumeName: "vol0"
-
+        StartEndpointRequest {
+            IpcType: IPC_GRPC
+            VolumeAccessMode: VOLUME_ACCESS_READ_WRITE
+            VolumeMountMode: VOLUME_MOUNT_LOCAL
+        }
         ArtificialLoadSpec {
             Ranges {
                 Start: 261144

cloud/blockstore/tests/loadtest/local-mirror-lagging/local-mirror2-migration.txt

@@ -35,7 +35,11 @@ Vertices {
 Vertices {
     Test {
         VolumeName: "vol0"
-
+        StartEndpointRequest {
+            IpcType: IPC_GRPC
+            VolumeAccessMode: VOLUME_ACCESS_READ_WRITE
+            VolumeMountMode: VOLUME_MOUNT_LOCAL
+        }
         ArtificialLoadSpec {
             Ranges {
                 Start: 261144

cloud/blockstore/tests/loadtest/local-mirror-lagging/local-mirror2-small-restart-interval.txt

@@ -5,6 +5,11 @@ Vertices {
             BlockSize: 4096
             StorageMediaKind: STORAGE_MEDIA_SSD_MIRROR2
         }
+        StartEndpointRequest {
+            IpcType: IPC_GRPC
+            VolumeAccessMode: VOLUME_ACCESS_READ_WRITE
+            VolumeMountMode: VOLUME_MOUNT_LOCAL
+        }
         ArtificialLoadSpec {
             Ranges {
                 Start: 200000

cloud/blockstore/tests/loadtest/local-mirror-lagging/local-mirror3-basic.txt

@@ -5,6 +5,11 @@ Vertices {
             BlockSize: 4096
             StorageMediaKind: STORAGE_MEDIA_SSD_MIRROR3
         }
+        StartEndpointRequest {
+            IpcType: IPC_GRPC
+            VolumeAccessMode: VOLUME_ACCESS_READ_WRITE
+            VolumeMountMode: VOLUME_MOUNT_LOCAL
+        }
         ArtificialLoadSpec {
             Ranges {
                 Start: 261144

cloud/blockstore/tests/loadtest/local-mirror-lagging/test.py

@@ -6,8 +6,9 @@
 from cloud.blockstore.config.client_pb2 import TClientAppConfig, TClientConfig
 from cloud.blockstore.config.disk_pb2 import DEVICE_ERASE_METHOD_NONE, TDiskAgentConfig
 from cloud.blockstore.config.server_pb2 import TServerConfig, TServerAppConfig, \
-    TKikimrServiceConfig
+    TKikimrServiceConfig, TChecksumFlags
 from cloud.blockstore.config.storage_pb2 import TStorageServiceConfig
+from cloud.blockstore.public.sdk.python.protos import DIVP_ENABLED_FORCED
 
 from cloud.blockstore.tests.python.lib.disk_agent_runner import LocalDiskAgent
 from cloud.blockstore.tests.python.lib.nbs_runner import LocalNbs
@@ -248,6 +249,7 @@ def __run_test(test_case, use_rdma):
             devices_per_agent,
             disk_agent_config_patch=TDiskAgentConfig(
                 DedicatedDiskAgent=True,
+                DataIntegrityValidationPolicyForDrBasedDisks=DIVP_ENABLED_FORCED,
                 # in tests, only one disk is created and it lives until the end,
                 # so we can set DEVICE_ERASE_METHOD_NONE to speed up testing
                 DeviceEraseMethod=DEVICE_ERASE_METHOD_NONE),
@@ -266,6 +268,8 @@ def __run_test(test_case, use_rdma):
         server_app_config.ServerConfig.RdmaClientEnabled = use_rdma
         server_app_config.ServerConfig.EndpointStorageType = EEndpointStorageType.ENDPOINT_STORAGE_FILE
         server_app_config.ServerConfig.EndpointStorageDir = endpoint_storage_dir
+        server_app_config.ServerConfig.ChecksumFlags.CopyFrom(TChecksumFlags())
+        server_app_config.ServerConfig.ChecksumFlags.EnableDataIntegrityClient = True
         server_app_config.KikimrServiceConfig.CopyFrom(TKikimrServiceConfig())
 
         storage = TStorageServiceConfig()

cloud/blockstore/tests/loadtest/local-mirror/local-mirror2-agent-removal.txt

@@ -15,6 +15,11 @@ Vertices {
     Test {
         Name: "shoot1"
         VolumeName: "vol0"
+        StartEndpointRequest {
+            IpcType: IPC_GRPC
+            VolumeAccessMode: VOLUME_ACCESS_READ_WRITE
+            VolumeMountMode: VOLUME_MOUNT_LOCAL
+        }
         ArtificialLoadSpec {
             Ranges {
                 Start: 0
@@ -36,6 +41,11 @@ Vertices {
     Test {
         Name: "shoot2"
         VolumeName: "vol0"
+        StartEndpointRequest {
+            IpcType: IPC_GRPC
+            VolumeAccessMode: VOLUME_ACCESS_READ_WRITE
+            VolumeMountMode: VOLUME_MOUNT_LOCAL
+        }
         ArtificialLoadSpec {
             Ranges {
                 Start: 0
@@ -107,6 +117,11 @@ Vertices {
     Test {
         Name: "shoot3"
         VolumeName: "vol0"
+        StartEndpointRequest {
+            IpcType: IPC_GRPC
+            VolumeAccessMode: VOLUME_ACCESS_READ_WRITE
+            VolumeMountMode: VOLUME_MOUNT_LOCAL
+        }
         ArtificialLoadSpec {
             Ranges {
                 Start: 0