Skip to content

Commit b7e2f3c

Browse files
kungakunga
authored
25-1 EnableLocalDBBtreeIndex tests clean up (#15025)
1 parent 7432fca commit b7e2f3c

File tree

4 files changed

+133
-28
lines changed

4 files changed

+133
-28
lines changed

‎ydb/core/client/flat_ut.cpp‎

Lines changed: 119 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2576,14 +2576,14 @@ Y_UNIT_TEST_SUITE(TFlatTest) {
25762576
annoyingClient.Ls("/dc-1/Dir/TableOld");
25772577
}
25782578

2579-
Y_UNIT_TEST(AutoSplitBySize) {
2579+
Y_UNIT_TEST(AutoSplitBySize_BTreeIndex) {
25802580
TPortManager pm;
25812581
ui16 port = pm.GetPort(2134);
25822582
NKikimrConfig::TFeatureFlags featureFlags;
25832583
featureFlags.SetEnableLocalDBBtreeIndex(true);
25842584
TServerSettings serverSettings(port);
25852585
serverSettings.SetFeatureFlags(featureFlags);
2586-
TServer cleverServer = TServer(serverSettings);
2586+
TServer cleverServer = TServer(serverSettings);
25872587
DisableSplitMergePartCountLimit(cleverServer);
25882588

25892589
cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::OPS_COMPACT, NActors::NLog::PRI_INFO);
@@ -2685,7 +2685,124 @@ Y_UNIT_TEST_SUITE(TFlatTest) {
26852685
fnWriteRow(Sprintf("C-%d", i), bigValue);
26862686
}
26872687

2688+
// Check that another split actually happened
2689+
for (int retry = 0; retry < 30 && partitions.size() == 2; ++retry) {
2690+
partitions = annoyingClient.GetTablePartitions("/dc-1/Dir/T1");
2691+
Sleep(TDuration::Seconds(1));
2692+
}
2693+
UNIT_ASSERT_VALUES_EQUAL(partitions.size(), 3);
2694+
}
2695+
2696+
Y_UNIT_TEST(AutoSplitBySize_FlatIndex) {
2697+
TPortManager pm;
2698+
ui16 port = pm.GetPort(2134);
2699+
NKikimrConfig::TFeatureFlags featureFlags;
2700+
featureFlags.SetEnableLocalDBBtreeIndex(false);
2701+
TServerSettings serverSettings(port);
2702+
serverSettings.SetFeatureFlags(featureFlags);
2703+
TServer cleverServer = TServer(serverSettings);
2704+
DisableSplitMergePartCountLimit(cleverServer);
2705+
2706+
cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::OPS_COMPACT, NActors::NLog::PRI_INFO);
2707+
2708+
TFlatMsgBusClient annoyingClient(port);
2709+
2710+
const char * tableDescr = R"___(
2711+
Name: "T1"
2712+
Columns { Name: "Key" Type: "String"}
2713+
Columns { Name: "Value" Type: "Utf8"}
2714+
KeyColumnNames: ["Key"]
2715+
2716+
PartitionConfig {
2717+
PartitioningPolicy {
2718+
SizeToSplit: 45000000
2719+
}
2720+
CompactionPolicy {
2721+
InMemSizeToSnapshot: 100000
2722+
InMemStepsToSnapshot: 1
2723+
InMemForceStepsToSnapshot: 2
2724+
InMemForceSizeToSnapshot: 200000
2725+
InMemCompactionBrokerQueue: 0
2726+
ReadAheadHiThreshold: 200000
2727+
ReadAheadLoThreshold: 100000
2728+
MinDataPageSize: 7168
2729+
SnapBrokerQueue: 0
2730+
Generation {
2731+
GenerationId: 0
2732+
SizeToCompact: 10000
2733+
CountToCompact: 2
2734+
ForceCountToCompact: 2
2735+
ForceSizeToCompact: 20000
2736+
CompactionBrokerQueue: 1
2737+
KeepInCache: true
2738+
}
2739+
}
2740+
}
2741+
)___";
2742+
2743+
annoyingClient.InitRoot();
2744+
annoyingClient.MkDir("/dc-1", "Dir");
2745+
annoyingClient.CreateTable("/dc-1/Dir", tableDescr);
2746+
2747+
TVector<ui64> partitions = annoyingClient.GetTablePartitions("/dc-1/Dir/T1");
2748+
UNIT_ASSERT_VALUES_EQUAL(partitions.size(), 1);
2749+
2750+
// Force stats reporting without delays
2751+
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
2752+
2753+
cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG);
2754+
// cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NActors::NLog::PRI_DEBUG);
2755+
// cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_DEBUG);
2756+
2757+
// Write rows to trigger split
2758+
auto fnWriteRow = [&](TString key, TString value) {
2759+
Cerr << key << Endl;
2760+
TString insertRowQuery = R"___(
2761+
(
2762+
(let key '('('Key (String '%s))))
2763+
(let value '('('Value (Utf8 '%s))))
2764+
(let ret_ (AsList
2765+
(UpdateRow '/dc-1/Dir/%s key value)
2766+
))
2767+
(return ret_)
2768+
)
2769+
)___";
2770+
2771+
int retryCnt = 20;
2772+
while (retryCnt--) {
2773+
TFlatMsgBusClient::TFlatQueryOptions opts;
2774+
NKikimrClient::TResponse response;
2775+
annoyingClient.FlatQueryRaw(cleverServer.GetRuntime(), Sprintf(insertRowQuery.data(), key.data(), value.data(), "T1"), opts, response);
2776+
ui32 responseStatus = response.GetStatus();
2777+
if (responseStatus == NMsgBusProxy::MSTATUS_REJECTED) {
2778+
Sleep(TDuration::Seconds(1));
2779+
} else {
2780+
UNIT_ASSERT_VALUES_EQUAL(responseStatus, NMsgBusProxy::MSTATUS_OK);
2781+
break;
2782+
}
2783+
}
2784+
};
2785+
2786+
TString bigValue(6*1024*1024, 'a');
2787+
2788+
for (int i = 0; i < 4; ++i) {
2789+
fnWriteRow(Sprintf("A-%d", i), bigValue);
2790+
fnWriteRow(Sprintf("B-%d", i), bigValue);
2791+
}
2792+
26882793
// Check that split actually happened
2794+
for (int retry = 0; retry < 30 && partitions.size() == 1; ++retry) {
2795+
partitions = annoyingClient.GetTablePartitions("/dc-1/Dir/T1");
2796+
Sleep(TDuration::Seconds(1));
2797+
}
2798+
UNIT_ASSERT_VALUES_EQUAL(partitions.size(), 2);
2799+
2800+
// Write some more rows to trigger another split
2801+
for (int i = 0; i < 6; ++i) {
2802+
fnWriteRow(Sprintf("C-%d", i), bigValue);
2803+
}
2804+
2805+
// Check that another split actually happened
26892806
for (int retry = 0; retry < 30 && partitions.size() == 2; ++retry) {
26902807
partitions = annoyingClient.GetTablePartitions("/dc-1/Dir/T1");
26912808
Sleep(TDuration::Seconds(1));

‎ydb/core/tablet_flat/flat_executor_ut.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6754,7 +6754,7 @@ Y_UNIT_TEST_SUITE(TFlatTableExecutor_BTreeIndex) {
67546754
env.SendSync(new TEvents::TEvPoison, false, true);
67556755
env.FireDummyTablet(ui32(NFake::TDummy::EFlg::Comp));
67566756

6757-
// after restart we have no pages in private cache
6757+
// after restart we have no pages in private cache except flat index
67586758
env.SendSync(new NFake::TEvExecute{ new TTxFullScan(readRows, failedAttempts) }, true);
67596759
UNIT_ASSERT_VALUES_EQUAL(readRows, 1000);
67606760
UNIT_ASSERT_VALUES_EQUAL(failedAttempts, 286);

‎ydb/core/tablet_flat/ut/ut_shared_sausagecache.cpp‎

Lines changed: 13 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,6 @@ Y_UNIT_TEST(Limits) {
153153
env->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE);
154154
auto counters = GetSharedPageCounters(env);
155155

156-
env->GetAppData().FeatureFlags.SetEnableLocalDBBtreeIndex(true);
157156
bool bTreeIndex = env->GetAppData().FeatureFlags.GetEnableLocalDBBtreeIndex();
158157
ui32 passiveBytes = bTreeIndex ? 131 : 7772;
159158

@@ -181,30 +180,30 @@ Y_UNIT_TEST(Limits) {
181180
UNIT_ASSERT_VALUES_EQUAL(counters->LoadInFlyBytes->Val(), 0);
182181
UNIT_ASSERT_DOUBLES_EQUAL(counters->ActiveBytes->Val(), static_cast<i64>(8_MB / 3 * 2), static_cast<i64>(1_MB / 3)); // 2 full layers (fresh & staging)
183182
UNIT_ASSERT_VALUES_EQUAL(counters->ActiveLimitBytes->Val(), 8_MB);
184-
UNIT_ASSERT_VALUES_EQUAL(counters->PassiveBytes->Val(), passiveBytes);
183+
UNIT_ASSERT_VALUES_EQUAL(counters->PassiveBytes->Val(), 131);
185184
UNIT_ASSERT_VALUES_EQUAL(counters->MemLimitBytes->Val(), 0);
186185

187186
env->Send(MakeSharedPageCacheId(), TActorId{}, new NMemory::TEvConsumerLimit(100_MB));
188187
WaitEvent(env, NMemory::EvConsumerLimit);
189188
LogCounters(counters);
190189
UNIT_ASSERT_DOUBLES_EQUAL(counters->ActiveBytes->Val(), static_cast<i64>(8_MB / 3 * 2), static_cast<i64>(1_MB / 3));
191190
UNIT_ASSERT_VALUES_EQUAL(counters->ActiveLimitBytes->Val(), 8_MB);
192-
UNIT_ASSERT_VALUES_EQUAL(counters->PassiveBytes->Val(), passiveBytes);
191+
UNIT_ASSERT_VALUES_EQUAL(counters->PassiveBytes->Val(), 131);
193192
UNIT_ASSERT_VALUES_EQUAL(counters->MemLimitBytes->Val(), 100_MB);
194193

195194
env->Send(MakeSharedPageCacheId(), TActorId{}, new NMemory::TEvConsumerLimit(6_MB));
196195
WaitEvent(env, NMemory::EvConsumerLimit);
197196
LogCounters(counters);
198197
UNIT_ASSERT_DOUBLES_EQUAL(counters->ActiveBytes->Val(), static_cast<i64>(8_MB / 3 * 2), static_cast<i64>(1_MB / 3));
199198
UNIT_ASSERT_VALUES_EQUAL(counters->ActiveLimitBytes->Val(), 6_MB);
200-
UNIT_ASSERT_VALUES_EQUAL(counters->PassiveBytes->Val(), passiveBytes);
199+
UNIT_ASSERT_VALUES_EQUAL(counters->PassiveBytes->Val(), 131);
201200
UNIT_ASSERT_VALUES_EQUAL(counters->MemLimitBytes->Val(), 6_MB);
202201

203202
env->Send(MakeSharedPageCacheId(), TActorId{}, new NMemory::TEvConsumerLimit(3_MB));
204203
WaitEvent(env, NMemory::EvConsumerLimit);
205204
UNIT_ASSERT_DOUBLES_EQUAL(counters->ActiveBytes->Val(), static_cast<i64>(3_MB), static_cast<i64>(1_MB / 3));
206205
UNIT_ASSERT_VALUES_EQUAL(counters->ActiveLimitBytes->Val(), 3_MB);
207-
UNIT_ASSERT_VALUES_EQUAL(counters->PassiveBytes->Val(), passiveBytes);
206+
UNIT_ASSERT_VALUES_EQUAL(counters->PassiveBytes->Val(), 131);
208207
UNIT_ASSERT_VALUES_EQUAL(counters->MemLimitBytes->Val(), 3_MB);
209208

210209
env->Send(MakeSharedPageCacheId(), TActorId{}, new NMemory::TEvConsumerLimit(0_MB));
@@ -222,10 +221,6 @@ Y_UNIT_TEST(Limits_Config) {
222221
env->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE);
223222
auto counters = GetSharedPageCounters(env);
224223

225-
env->GetAppData().FeatureFlags.SetEnableLocalDBBtreeIndex(true);
226-
bool bTreeIndex = env->GetAppData().FeatureFlags.GetEnableLocalDBBtreeIndex();
227-
ui32 passiveBytes = bTreeIndex ? 131 : 7772;
228-
229224
env.FireDummyTablet(ui32(NFake::TDummy::EFlg::Comp));
230225
env.SendSync(new NFake::TEvExecute{ new TTxInitSchema() });
231226

@@ -250,36 +245,36 @@ Y_UNIT_TEST(Limits_Config) {
250245
UNIT_ASSERT_VALUES_EQUAL(counters->LoadInFlyBytes->Val(), 0);
251246
UNIT_ASSERT_DOUBLES_EQUAL(counters->ActiveBytes->Val(), static_cast<i64>(8_MB / 3 * 2), static_cast<i64>(1_MB / 3)); // 2 full layers (fresh & staging)
252247
UNIT_ASSERT_VALUES_EQUAL(counters->ActiveLimitBytes->Val(), 8_MB);
253-
UNIT_ASSERT_VALUES_EQUAL(counters->PassiveBytes->Val(), passiveBytes);
248+
UNIT_ASSERT_VALUES_EQUAL(counters->PassiveBytes->Val(), 131);
254249
UNIT_ASSERT_VALUES_EQUAL(counters->ConfigLimitBytes->Val(), 8_MB);
255250

256251
SetupSharedCache(env, NKikimrSharedCache::ThreeLeveledLRU, 100_MB);
257252
LogCounters(counters);
258253
UNIT_ASSERT_DOUBLES_EQUAL(counters->ActiveBytes->Val(), static_cast<i64>(8_MB / 3 * 2), static_cast<i64>(1_MB / 3));
259254
UNIT_ASSERT_VALUES_EQUAL(counters->ActiveLimitBytes->Val(), 8_MB);
260-
UNIT_ASSERT_VALUES_EQUAL(counters->PassiveBytes->Val(), passiveBytes);
255+
UNIT_ASSERT_VALUES_EQUAL(counters->PassiveBytes->Val(), 131);
261256
UNIT_ASSERT_VALUES_EQUAL(counters->ConfigLimitBytes->Val(), 100_MB);
262257

263258
SetupSharedCache(env, NKikimrSharedCache::ThreeLeveledLRU, 2_MB);
264259
LogCounters(counters);
265260
UNIT_ASSERT_DOUBLES_EQUAL(counters->ActiveBytes->Val(), static_cast<i64>(8_MB / 3 * 2), static_cast<i64>(1_MB / 3));
266261
UNIT_ASSERT_VALUES_EQUAL(counters->ActiveLimitBytes->Val(), 2_MB);
267-
UNIT_ASSERT_VALUES_EQUAL(counters->PassiveBytes->Val(), passiveBytes);
262+
UNIT_ASSERT_VALUES_EQUAL(counters->PassiveBytes->Val(), 131);
268263
UNIT_ASSERT_VALUES_EQUAL(counters->ConfigLimitBytes->Val(), 2_MB);
269264

270265
env->Send(MakeSharedPageCacheId(), TActorId{}, new NMemory::TEvConsumerLimit(1_MB));
271266
WaitEvent(env, NMemory::EvConsumerLimit);
272267
UNIT_ASSERT_DOUBLES_EQUAL(counters->ActiveBytes->Val(), static_cast<i64>(1_MB), static_cast<i64>(1_MB / 3));
273268
UNIT_ASSERT_VALUES_EQUAL(counters->ActiveLimitBytes->Val(), 1_MB);
274-
UNIT_ASSERT_VALUES_EQUAL(counters->PassiveBytes->Val(), passiveBytes);
269+
UNIT_ASSERT_VALUES_EQUAL(counters->PassiveBytes->Val(), 131);
275270
UNIT_ASSERT_VALUES_EQUAL(counters->MemLimitBytes->Val(), 1_MB);
276271
UNIT_ASSERT_VALUES_EQUAL(counters->ConfigLimitBytes->Val(), 2_MB);
277272

278273
SetupSharedCache(env, NKikimrSharedCache::ThreeLeveledLRU, 0_MB);
279274
LogCounters(counters);
280275
UNIT_ASSERT_DOUBLES_EQUAL(counters->ActiveBytes->Val(), static_cast<i64>(1_MB), static_cast<i64>(1_MB / 3));
281276
UNIT_ASSERT_VALUES_EQUAL(counters->ActiveLimitBytes->Val(), 0_MB);
282-
UNIT_ASSERT_VALUES_EQUAL(counters->PassiveBytes->Val(), passiveBytes);
277+
UNIT_ASSERT_VALUES_EQUAL(counters->PassiveBytes->Val(), 131);
283278
UNIT_ASSERT_VALUES_EQUAL(counters->ConfigLimitBytes->Val(), 0_MB);
284279
}
285280

@@ -290,11 +285,11 @@ Y_UNIT_TEST(ThreeLeveledLRU) {
290285
env->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE);
291286
auto counters = GetSharedPageCounters(env);
292287

288+
env->GetAppData().FeatureFlags.SetEnableLocalDBBtreeIndex(true);
293289
env.FireDummyTablet(ui32(NFake::TDummy::EFlg::Comp));
294290
env.SendSync(new NFake::TEvExecute{ new TTxInitSchema() });
295291

296292
SetupSharedCache(env);
297-
env->GetAppData().FeatureFlags.SetEnableLocalDBBtreeIndex(true);
298293

299294
// write 100 rows, each ~100KB (~10MB)
300295
for (i64 key = 0; key < 100; ++key) {
@@ -392,11 +387,11 @@ Y_UNIT_TEST(S3FIFO) {
392387
env->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE);
393388
auto counters = GetSharedPageCounters(env);
394389

390+
env->GetAppData().FeatureFlags.SetEnableLocalDBBtreeIndex(true);
395391
env.FireDummyTablet(ui32(NFake::TDummy::EFlg::Comp));
396392
env.SendSync(new NFake::TEvExecute{ new TTxInitSchema() });
397393

398394
SetupSharedCache(env, NKikimrSharedCache::S3FIFO);
399-
env->GetAppData().FeatureFlags.SetEnableLocalDBBtreeIndex(true);
400395

401396
// write 100 rows, each ~100KB (~10MB)
402397
for (i64 key = 0; key < 100; ++key) {
@@ -494,11 +489,11 @@ Y_UNIT_TEST(ClockPro) {
494489
env->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE);
495490
auto counters = GetSharedPageCounters(env);
496491

492+
env->GetAppData().FeatureFlags.SetEnableLocalDBBtreeIndex(true);
497493
env.FireDummyTablet(ui32(NFake::TDummy::EFlg::Comp));
498494
env.SendSync(new NFake::TEvExecute{ new TTxInitSchema() });
499495

500496
SetupSharedCache(env, NKikimrSharedCache::ClockPro);
501-
env->GetAppData().FeatureFlags.SetEnableLocalDBBtreeIndex(true);
502497

503498
// write 100 rows, each ~100KB (~10MB)
504499
for (i64 key = 0; key < 100; ++key) {
@@ -610,11 +605,11 @@ Y_UNIT_TEST(ReplacementPolicySwitch) {
610605
env->SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE);
611606
auto counters = GetSharedPageCounters(env);
612607

608+
env->GetAppData().FeatureFlags.SetEnableLocalDBBtreeIndex(true);
613609
env.FireDummyTablet(ui32(NFake::TDummy::EFlg::Comp));
614610
env.SendSync(new NFake::TEvExecute{ new TTxInitSchema() });
615611

616612
SetupSharedCache(env);
617-
env->GetAppData().FeatureFlags.SetEnableLocalDBBtreeIndex(true);
618613

619614
// write 100 rows, each ~100KB (~10MB)
620615
for (i64 key = 0; key < 100; ++key) {

‎ydb/core/tx/datashard/datashard_ut_trace.cpp‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -284,7 +284,6 @@ Y_UNIT_TEST_SUITE(TDataShardTrace) {
284284
Y_UNIT_TEST(TestTraceDistributedSelect) {
285285
auto [runtime, server, sender] = TestCreateServer();
286286
runtime.GetAppData().FeatureFlags.SetEnableLocalDBBtreeIndex(true);
287-
bool bTreeIndex = runtime.GetAppData().FeatureFlags.GetEnableLocalDBBtreeIndex();
288287

289288
CreateShardedTable(server, sender, "/Root", "table-1", 1, false);
290289

@@ -362,12 +361,6 @@ Y_UNIT_TEST_SUITE(TDataShardTrace) {
362361
ExpectedSpan("Tablet.Transaction",
363362
ExpectedSpan("Tablet.Transaction.Execute",
364363
Repeat("Datashard.Unit", 3)),
365-
// No extra page fault with btree index (root is in meta)
366-
ConditionalSpanVec(!bTreeIndex,
367-
"Tablet.Transaction.Wait",
368-
"Tablet.Transaction.Enqueued",
369-
ExpectedSpan("Tablet.Transaction.Execute",
370-
"Datashard.Unit")),
371364
"Tablet.Transaction.Wait",
372365
"Tablet.Transaction.Enqueued",
373366
ExpectedSpan("Tablet.Transaction.Execute",

0 commit comments

Comments
 (0)