From 7d577f0c53db253b690d9b094318c6a98c9b142c Mon Sep 17 00:00:00 2001 From: Mikhail Montsev Date: Fri, 26 Dec 2025 00:10:27 +0100 Subject: [PATCH 1/6] [Filestore] add benchmark for index tablet DescribeData with 1 MiB sized requests --- .../storage/tablet/bench/tablet_bench.cpp | 93 +++++++++++++++++++ .../libs/storage/tablet/bench/ya.make | 21 +++++ cloud/filestore/libs/storage/tablet/ya.make | 1 + 3 files changed, 115 insertions(+) create mode 100644 cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp create mode 100644 cloud/filestore/libs/storage/tablet/bench/ya.make diff --git a/cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp b/cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp new file mode 100644 index 00000000000..68b2819e684 --- /dev/null +++ b/cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp @@ -0,0 +1,93 @@ +#include + +#include +#include + +#include + +#include + +#include +#include + +using namespace NCloud; +using namespace NCloud::NFileStore; +using namespace NCloud::NFileStore::NStorage; + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +constexpr ui32 BlockSize = 4096; + +//////////////////////////////////////////////////////////////////////////////// + +struct TTabletSetup +{ + TTestEnv Env; + std::unique_ptr TabletClient; + ui64 Handle = 0; + + TTabletSetup() + : Env(TTestEnvConfig{ + .LogPriority_NFS = NActors::NLog::PRI_ALERT, + .LogPriority_KiKiMR = NActors::NLog::PRI_ALERT, + .LogPriority_Others = NActors::NLog::PRI_ALERT}) + { + Env.CreateSubDomain("nfs"); + + ui32 nodeIdx = Env.CreateNode("nfs"); + ui64 tabletId = Env.BootIndexTablet(nodeIdx); + + TabletClient = std::make_unique( + Env.GetRuntime(), + nodeIdx, + tabletId, + TFileSystemConfig{.BlockSize = BlockSize}); + TabletClient->InitSession("client", "session"); + + auto nodeId = CreateNode( + *TabletClient, + TCreateNodeArgs::File(RootNodeId, "test")); + + Handle = CreateHandle(*TabletClient, nodeId); + TabletClient->WriteData(Handle, 0, 1_MB, '1'); + } + + void DescribeData(ui64 offset, ui64 length) + { + auto response = TabletClient->DescribeData(Handle, offset, length); + Y_ABORT_UNLESS(1_MB == response->Record.GetFileSize()); + } +}; + +std::atomic Tablet = nullptr; + +TTabletSetup* GetOrCreateTablet() +{ + if (Tablet.load()) { + return Tablet.load(); + } + + auto tmp = std::make_unique(); + TTabletSetup* expected = nullptr; + if (!Tablet.compare_exchange_strong(expected, tmp.get())) { + return Tablet.load(); + } + + // Memory leak is OK here + return tmp.release(); +} + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +Y_CPU_BENCHMARK(TTablet_DescribeData_1MiBRequestSize, iface) +{ + auto* tablet = GetOrCreateTablet(); + + for (size_t i = 0; i < iface.Iterations(); ++i) { + tablet->DescribeData(0, 1_MB); + } +} diff --git a/cloud/filestore/libs/storage/tablet/bench/ya.make b/cloud/filestore/libs/storage/tablet/bench/ya.make new file mode 100644 index 00000000000..e250cbb6a16 --- /dev/null +++ b/cloud/filestore/libs/storage/tablet/bench/ya.make @@ -0,0 +1,21 @@ +Y_BENCHMARK() + +IF (SANITIZER_TYPE) + TAG(ya:manual) +ENDIF() + +SRCS( + tablet_bench.cpp +) + +PEERDIR( + cloud/filestore/libs/storage/testlib +) + +PEERDIR( + cloud/filestore/libs/storage/tablet +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/cloud/filestore/libs/storage/tablet/ya.make b/cloud/filestore/libs/storage/tablet/ya.make index 4f3b58537b2..5ed6f5ba25e 100644 --- a/cloud/filestore/libs/storage/tablet/ya.make +++ b/cloud/filestore/libs/storage/tablet/ya.make @@ -134,6 +134,7 @@ RECURSE( ) RECURSE_FOR_TESTS( + bench ut ut_cache_stress ut_counters From 6cf7f94ee371bfef395c7ec012f08cfee5788ea9 Mon Sep 17 00:00:00 2001 From: Mikhail Montsev Date: Fri, 26 Dec 2025 19:35:08 +0100 Subject: [PATCH 2/6] Set DispatchedEventsLimit to one billion; enable in-memory-index cache with MixedBlocksOffloadedRangesCapacity set to one million --- .../libs/storage/tablet/bench/tablet_bench.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp b/cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp index 68b2819e684..cf941a79585 100644 --- a/cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp +++ b/cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp @@ -18,10 +18,6 @@ namespace { //////////////////////////////////////////////////////////////////////////////// -constexpr ui32 BlockSize = 4096; - -//////////////////////////////////////////////////////////////////////////////// - struct TTabletSetup { TTestEnv Env; @@ -34,7 +30,14 @@ struct TTabletSetup .LogPriority_KiKiMR = NActors::NLog::PRI_ALERT, .LogPriority_Others = NActors::NLog::PRI_ALERT}) { + NCloud::NFileStore::NProto::TStorageConfig storageConfig; + storageConfig.SetInMemoryIndexCacheEnabled(true); + storageConfig.SetMixedBlocksOffloadedRangesCapacity(1e6); + + Env.UpdateStorageConfig(std::move(storageConfig)); + Env.CreateSubDomain("nfs"); + Env.GetRuntime().SetDispatchedEventsLimit(1e9); ui32 nodeIdx = Env.CreateNode("nfs"); ui64 tabletId = Env.BootIndexTablet(nodeIdx); @@ -43,7 +46,7 @@ struct TTabletSetup Env.GetRuntime(), nodeIdx, tabletId, - TFileSystemConfig{.BlockSize = BlockSize}); + TFileSystemConfig{.BlockSize = 4096}); TabletClient->InitSession("client", "session"); auto nodeId = CreateNode( From e9445b8f0575f4563499b96b40e47744172bacd6 Mon Sep 17 00:00:00 2001 From: Mikhail Montsev Date: Wed, 31 Dec 2025 21:02:33 +0100 Subject: [PATCH 3/6] add comment --- cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp b/cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp index cf941a79585..e34371f1ea1 100644 --- a/cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp +++ b/cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp @@ -64,6 +64,12 @@ struct TTabletSetup } }; +// Using a non-owning static variable because the test Actor System crashes +// during initialization. +// A singleton also doesn’t work because it causes +// other issues during deinitialization. +// A memory leak is the price we pay to prevent the program from crashing. +// This is acceptable because benchmarks are not run under sanitizers std::atomic Tablet = nullptr; TTabletSetup* GetOrCreateTablet() @@ -78,7 +84,6 @@ TTabletSetup* GetOrCreateTablet() return Tablet.load(); } - // Memory leak is OK here return tmp.release(); } From 75e236c6f81fdfe87d3c9f12c1859f1d5b090eb3 Mon Sep 17 00:00:00 2001 From: Mikhail Montsev Date: Wed, 31 Dec 2025 21:08:57 +0100 Subject: [PATCH 4/6] better --- .../libs/storage/tablet/bench/tablet_bench.cpp | 16 +++++++++++----- .../filestore/libs/storage/tablet/bench/ya.make | 4 +--- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp b/cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp index e34371f1ea1..b7b4ae6d816 100644 --- a/cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp +++ b/cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp @@ -18,6 +18,10 @@ namespace { //////////////////////////////////////////////////////////////////////////////// +constexpr ui32 BlockSize = 4096; +constexpr ui64 FileSize = 1_MB; +constexpr ui64 MixedBlocksOffloadedRangesCapacity = 1000000; + struct TTabletSetup { TTestEnv Env; @@ -26,18 +30,20 @@ struct TTabletSetup TTabletSetup() : Env(TTestEnvConfig{ + // Turn off logging in order to reduce performance overhead .LogPriority_NFS = NActors::NLog::PRI_ALERT, .LogPriority_KiKiMR = NActors::NLog::PRI_ALERT, .LogPriority_Others = NActors::NLog::PRI_ALERT}) { NCloud::NFileStore::NProto::TStorageConfig storageConfig; storageConfig.SetInMemoryIndexCacheEnabled(true); - storageConfig.SetMixedBlocksOffloadedRangesCapacity(1e6); + storageConfig.SetMixedBlocksOffloadedRangesCapacity( + MixedBlocksOffloadedRangesCapacity); Env.UpdateStorageConfig(std::move(storageConfig)); Env.CreateSubDomain("nfs"); - Env.GetRuntime().SetDispatchedEventsLimit(1e9); + Env.GetRuntime().SetDispatchedEventsLimit(Max()); ui32 nodeIdx = Env.CreateNode("nfs"); ui64 tabletId = Env.BootIndexTablet(nodeIdx); @@ -46,7 +52,7 @@ struct TTabletSetup Env.GetRuntime(), nodeIdx, tabletId, - TFileSystemConfig{.BlockSize = 4096}); + TFileSystemConfig{.BlockSize = BlockSize}); TabletClient->InitSession("client", "session"); auto nodeId = CreateNode( @@ -54,13 +60,13 @@ struct TTabletSetup TCreateNodeArgs::File(RootNodeId, "test")); Handle = CreateHandle(*TabletClient, nodeId); - TabletClient->WriteData(Handle, 0, 1_MB, '1'); + TabletClient->WriteData(Handle, 0, FileSize, '1'); } void DescribeData(ui64 offset, ui64 length) { auto response = TabletClient->DescribeData(Handle, offset, length); - Y_ABORT_UNLESS(1_MB == response->Record.GetFileSize()); + Y_ABORT_UNLESS(FileSize == response->Record.GetFileSize()); } }; diff --git a/cloud/filestore/libs/storage/tablet/bench/ya.make b/cloud/filestore/libs/storage/tablet/bench/ya.make index e250cbb6a16..6897e2a52a1 100644 --- a/cloud/filestore/libs/storage/tablet/bench/ya.make +++ b/cloud/filestore/libs/storage/tablet/bench/ya.make @@ -1,8 +1,6 @@ Y_BENCHMARK() -IF (SANITIZER_TYPE) - TAG(ya:manual) -ENDIF() +NO_SANITIZE() SRCS( tablet_bench.cpp From 7d3f0a7b3d63f350a506c6461484bd4f53535f44 Mon Sep 17 00:00:00 2001 From: Mikhail Montsev Date: Thu, 1 Jan 2026 21:55:39 +0100 Subject: [PATCH 5/6] get rid of memory leak --- .../storage/tablet/bench/tablet_bench.cpp | 23 +++---------------- .../libs/storage/tablet/bench/ya.make | 4 +++- .../library/actors/util/local_process_key.h | 10 +++++--- 3 files changed, 13 insertions(+), 24 deletions(-) diff --git a/cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp b/cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp index b7b4ae6d816..935e02c4bc0 100644 --- a/cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp +++ b/cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp @@ -5,9 +5,9 @@ #include +#include #include -#include #include using namespace NCloud; @@ -70,27 +70,10 @@ struct TTabletSetup } }; -// Using a non-owning static variable because the test Actor System crashes -// during initialization. -// A singleton also doesn’t work because it causes -// other issues during deinitialization. -// A memory leak is the price we pay to prevent the program from crashing. -// This is acceptable because benchmarks are not run under sanitizers -std::atomic Tablet = nullptr; - TTabletSetup* GetOrCreateTablet() { - if (Tablet.load()) { - return Tablet.load(); - } - - auto tmp = std::make_unique(); - TTabletSetup* expected = nullptr; - if (!Tablet.compare_exchange_strong(expected, tmp.get())) { - return Tablet.load(); - } - - return tmp.release(); + constexpr ui64 Priority = Max(); + return SingletonWithPriority(); } } // namespace diff --git a/cloud/filestore/libs/storage/tablet/bench/ya.make b/cloud/filestore/libs/storage/tablet/bench/ya.make index 6897e2a52a1..e250cbb6a16 100644 --- a/cloud/filestore/libs/storage/tablet/bench/ya.make +++ b/cloud/filestore/libs/storage/tablet/bench/ya.make @@ -1,6 +1,8 @@ Y_BENCHMARK() -NO_SANITIZE() +IF (SANITIZER_TYPE) + TAG(ya:manual) +ENDIF() SRCS( tablet_bench.cpp diff --git a/contrib/ydb/library/actors/util/local_process_key.h b/contrib/ydb/library/actors/util/local_process_key.h index bff8bef81b7..68d2025caa4 100644 --- a/contrib/ydb/library/actors/util/local_process_key.h +++ b/contrib/ydb/library/actors/util/local_process_key.h @@ -131,8 +131,9 @@ class TEnumProcessKey { static size_t GetIndex(const EnumT key) { ui32 index = static_cast(key); - Y_ABORT_UNLESS(index < Enum2Index.size()); - return Enum2Index[index]; + const auto& enum2Index = Singleton()->Enum2Index; + Y_ABORT_UNLESS(index < enum2Index.size()); + return enum2Index[index]; } private: @@ -153,5 +154,8 @@ class TEnumProcessKey { return enum2Index; } - inline static TVector Enum2Index = RegisterAll(); + struct TEnum2Index + { + TVector Enum2Index = RegisterAll(); + }; }; From c48db16a8705bff7d32fee9902aa58b9fb714e76 Mon Sep 17 00:00:00 2001 From: Mikhail Montsev Date: Thu, 1 Jan 2026 22:00:03 +0100 Subject: [PATCH 6/6] add comment --- cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp b/cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp index 935e02c4bc0..407425bcbca 100644 --- a/cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp +++ b/cloud/filestore/libs/storage/tablet/bench/tablet_bench.cpp @@ -72,6 +72,7 @@ struct TTabletSetup TTabletSetup* GetOrCreateTablet() { + // Ensure this singleton is destroyed first to avoid a crash constexpr ui64 Priority = Max(); return SingletonWithPriority(); }