Skip to content

Commit f92dcbc

Browse files
k0ushalk0ushal
committed
Changed ConsolidationTier unit tests to follow the new algorithm
Disabled irrelevant tests
1 parent f913b1b commit f92dcbc

File tree

1 file changed

+152
-44
lines changed

1 file changed

+152
-44
lines changed

‎tests/index/consolidation_policy_tests.cpp‎

Lines changed: 152 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
/// @author Vasiliy Nabatchikov
2222
////////////////////////////////////////////////////////////////////////////////
2323

24-
#include <iostream>
2524
#include "index/composite_reader_impl.hpp"
2625
#include "index/index_meta.hpp"
2726
#include "index/index_writer.hpp"
@@ -74,8 +73,9 @@ namespace {
7473

7574
void AssertCandidates(const irs::IndexReader& reader,
7675
const std::vector<size_t>& expected_candidates,
77-
const irs::Consolidation& actual_candidates) {
78-
ASSERT_EQ(expected_candidates.size(), actual_candidates.size());
76+
const irs::Consolidation& actual_candidates,
77+
const std::string& errMsg = "") {
78+
ASSERT_EQ(expected_candidates.size(), actual_candidates.size()) << errMsg;
7979

8080
for (const size_t expected_candidate_idx : expected_candidates) {
8181
const auto& expected_candidate = reader[expected_candidate_idx];
@@ -189,7 +189,7 @@ void AddSegment(irs::IndexMeta& meta, std::string_view name,
189189

190190
} // namespace
191191

192-
TEST(ConsolidationTierTest, MaxConsolidationSize) {
192+
TEST(ConsolidationTierTest, DISABLED_MaxConsolidationSize) {
193193
irs::IndexMeta meta;
194194
for (size_t i = 0; i < 22; ++i) {
195195
AddSegment(meta, std::to_string(i), 1, 1, 1);
@@ -321,7 +321,7 @@ TEST(ConsolidationTierTest, EmptySegment) {
321321
ASSERT_TRUE(candidates.empty()); // skip empty segments
322322
}
323323

324-
TEST(ConsolidationTierTest, MaxConsolidationCount) {
324+
TEST(ConsolidationTierTest, DISABLED_MaxConsolidationCount) {
325325
// generate meta
326326
irs::IndexMeta meta;
327327
for (size_t i = 0; i < 22; ++i) {
@@ -471,7 +471,7 @@ TEST(ConsolidationTierTest, MaxConsolidationCount) {
471471
}
472472
}
473473

474-
TEST(ConsolidationTierTest, MinConsolidationCount) {
474+
TEST(ConsolidationTierTest, DISABLED_MinConsolidationCount) {
475475
// generate meta
476476
irs::IndexMeta meta;
477477
for (size_t i = 0; i < 22; ++i) {
@@ -633,7 +633,7 @@ TEST(ConsolidationTierTest, MinConsolidationCount) {
633633
}
634634
}
635635

636-
TEST(ConsolidationTierTest, ConsolidationFloor) {
636+
TEST(ConsolidationTierTest, DISABLED_ConsolidationFloor) {
637637
// generate meta
638638
irs::IndexMeta meta;
639639
{
@@ -723,46 +723,60 @@ TEST(ConsolidationTierTest, ConsolidationFloor) {
723723
}
724724
}
725725

726-
TEST(ConsolidationTierTest, PreferSegmentsWithRemovals) {
727-
// generate meta
726+
TEST(ConsolidationTierTest, PreferConsolidationOverCleanupWhenDoesntMeetThreshold) {
728727
irs::IndexMeta meta;
729728
AddSegment(meta, "0", 10, 10, 10);
730729
AddSegment(meta, "1", 10, 10, 10);
731-
AddSegment(meta, "2", 11, 10, 11);
732-
AddSegment(meta, "3", 11, 10, 11);
730+
AddSegment(meta, "2", 10, 10, 10);
731+
AddSegment(meta, "3", 10, 10, 10);
732+
AddSegment(meta, "4", 10, 10, 10);
733+
AddSegment(meta, "5", 11, 8, 81);
734+
AddSegment(meta, "6", 10, 9, 91);
733735
IndexReaderMock reader{meta};
734736

735-
// ensure policy prefers segments with removals
737+
// The conslidation threshold defaults are defined
738+
// in tier::ConsolidationConfig.
739+
736740
irs::index_utils::ConsolidateTier options;
737741
options.floor_segment_bytes = 1;
738742
options.max_segments = 2;
739-
options.min_segments = 1;
743+
options.min_segments = 2;
740744
options.max_segments_bytes = std::numeric_limits<size_t>::max();
741745

742746
irs::ConsolidatingSegments consolidating_segments;
743747
auto policy = irs::index_utils::MakePolicy(options);
744748

745-
const std::vector<std::vector<size_t>> expected_tiers{{2, 3}, {0, 1}};
749+
irs::Consolidation candidates;
750+
policy(candidates, reader, consolidating_segments);
751+
ASSERT_EQ(5, candidates.size());
752+
}
746753

747-
for (auto& expected_tier : expected_tiers) {
748-
irs::Consolidation candidates;
749-
policy(candidates, reader, consolidating_segments);
750-
AssertCandidates(reader, expected_tier, candidates);
751-
candidates.clear();
752-
policy(candidates, reader, consolidating_segments);
753-
AssertCandidates(reader, expected_tier, candidates);
754-
// register candidates for consolidation
755-
for (const auto* candidate : candidates) {
756-
consolidating_segments.emplace(candidate->Meta().name);
757-
}
758-
}
754+
TEST(ConsolidationTierTest, PreferCleanupWhenMeetsThreshold) {
755+
// generate meta
756+
irs::IndexMeta meta;
757+
AddSegment(meta, "0", 10, 10, 10);
758+
AddSegment(meta, "1", 10, 10, 10);
759+
AddSegment(meta, "2", 10, 10, 10);
760+
AddSegment(meta, "3", 10, 10, 10);
761+
AddSegment(meta, "4", 10, 10, 10);
762+
AddSegment(meta, "5", 11, 5, 11);
763+
AddSegment(meta, "6", 10, 5, 11);
764+
IndexReaderMock reader{meta};
759765

760-
// no more segments to consolidate
761-
{
762-
irs::Consolidation candidates;
763-
policy(candidates, reader, consolidating_segments);
764-
ASSERT_TRUE(candidates.empty());
765-
}
766+
// ensure policy prefers segments with removals
767+
irs::index_utils::ConsolidateTier options;
768+
options.floor_segment_bytes = 1;
769+
options.max_segments = 2;
770+
options.min_segments = 2;
771+
options.max_segments_bytes = std::numeric_limits<size_t>::max();
772+
773+
irs::ConsolidatingSegments consolidating_segments;
774+
auto policy = irs::index_utils::MakePolicy(options);
775+
776+
irs::Consolidation candidates;
777+
policy(candidates, reader, consolidating_segments);
778+
ASSERT_EQ(2, candidates.size());
779+
AssertCandidates(reader, {5, 6}, candidates);
766780
}
767781

768782
TEST(ConsolidationTierTest, Singleton) {
@@ -788,31 +802,34 @@ TEST(ConsolidationTierTest, Singleton) {
788802
}
789803
}
790804

791-
// singleton consolidation with removals
805+
// singleton consolidation with < 50% removals
792806
{
793807
irs::ConsolidatingSegments consolidating_segments;
794808
irs::IndexMeta meta;
795-
AddSegment(meta, "0", 100, 99, 150);
809+
AddSegment(meta, "0", 100, 51, 150);
796810
IndexReaderMock reader{meta};
797811

798-
// 1st tier
799812
{
800813
irs::Consolidation candidates;
801814
policy(candidates, reader, consolidating_segments);
802-
AssertCandidates(reader, {0}, candidates);
803-
candidates.clear();
804-
policy(candidates, reader, consolidating_segments);
805-
AssertCandidates(reader, {0}, candidates);
806-
// register candidates for consolidation
807-
for (const auto* candidate : candidates) {
808-
consolidating_segments.emplace(candidate->Meta().name);
809-
}
815+
ASSERT_TRUE(candidates.empty());
810816
}
817+
}
818+
819+
// singleton consolidation with >= 50% removals
820+
{
821+
irs::ConsolidatingSegments consolidating_segments;
822+
irs::IndexMeta meta;
823+
AddSegment(meta, "0", 100, 49, 150);
824+
AddSegment(meta, "1", 100, 59, 150);
825+
AddSegment(meta, "2", 100, 69, 150);
826+
AddSegment(meta, "3", 100, 50, 150);
827+
IndexReaderMock reader{meta};
811828

812829
{
813830
irs::Consolidation candidates;
814831
policy(candidates, reader, consolidating_segments);
815-
ASSERT_TRUE(candidates.empty());
832+
AssertCandidates(reader, {0, 3}, candidates);
816833
}
817834
}
818835
}
@@ -917,6 +934,97 @@ TEST(ConsolidationTierTest, NoCandidates) {
917934
}
918935

919936
TEST(ConsolidationTierTest, SkewedSegments) {
937+
irs::index_utils::ConsolidateTier options;
938+
options.max_segments_bytes = 52500; // max size of the merge
939+
940+
auto policy = irs::index_utils::MakePolicy(options);
941+
942+
// test correct selection of candidates
943+
{
944+
irs::ConsolidatingSegments consolidating_segments;
945+
irs::IndexMeta meta;
946+
AddSegment(meta, "0", 100, 100, 10);
947+
AddSegment(meta, "1", 100, 100, 40);
948+
AddSegment(meta, "2", 100, 100, 60);
949+
AddSegment(meta, "3", 100, 100, 70);
950+
AddSegment(meta, "4", 100, 100, 100);
951+
AddSegment(meta, "5", 100, 100, 150);
952+
AddSegment(meta, "6", 100, 100, 200);
953+
AddSegment(meta, "7", 100, 100, 500);
954+
AddSegment(meta, "8", 100, 100, 750);
955+
AddSegment(meta, "9", 100, 100, 1100);
956+
AddSegment(meta, "10", 100, 100, 90);
957+
AddSegment(meta, "11", 100, 100, 75);
958+
AddSegment(meta, "12", 100, 100, 1500);
959+
AddSegment(meta, "13", 100, 100, 10000);
960+
AddSegment(meta, "14", 100, 100, 5000);
961+
AddSegment(meta, "15", 100, 100, 1750);
962+
AddSegment(meta, "16", 100, 100, 690);
963+
IndexReaderMock reader{meta};
964+
965+
const std::vector<std::vector<size_t>> expected_tiers{
966+
{0, 1, 2, 3, 4, 10, 11},
967+
{7, 8, 9, 12, 15, 16},
968+
{}
969+
};
970+
971+
for (size_t i = 0; i < expected_tiers.size(); i++) {
972+
auto& expected_tier = expected_tiers[i];
973+
irs::Consolidation candidates;
974+
policy(candidates, reader, consolidating_segments);
975+
AssertCandidates(reader, expected_tier, candidates, "Line: " + std::to_string(__LINE__) + ", i = " + std::to_string(i));
976+
candidates.clear();
977+
policy(candidates, reader, consolidating_segments);
978+
AssertCandidates(reader, expected_tier, candidates, "Line: " + std::to_string(__LINE__) + ", i = " + std::to_string(i));
979+
// register candidates for consolidation
980+
for (const auto* candidate : candidates) {
981+
consolidating_segments.emplace(candidate->Meta().name);
982+
}
983+
}
984+
}
985+
986+
{
987+
irs::ConsolidatingSegments consolidating_segments;
988+
irs::IndexMeta meta;
989+
AddSegment(meta, "0", 100, 100, 1);
990+
AddSegment(meta, "1", 100, 100, 1);
991+
AddSegment(meta, "2", 100, 100, 1);
992+
AddSegment(meta, "3", 100, 100, 75);
993+
AddSegment(meta, "4", 100, 100, 90);
994+
AddSegment(meta, "5", 100, 100, 100);
995+
AddSegment(meta, "6", 100, 100, 150);
996+
AddSegment(meta, "7", 100, 100, 200);
997+
AddSegment(meta, "8", 100, 100, 750);
998+
AddSegment(meta, "9", 100, 100, 1100);
999+
AddSegment(meta, "10", 100, 100, 1500);
1000+
AddSegment(meta, "11", 100, 100, 1750);
1001+
AddSegment(meta, "12", 100, 100, 5000);
1002+
AddSegment(meta, "13", 100, 100, 10000);
1003+
IndexReaderMock reader{meta};
1004+
1005+
const std::vector<std::vector<size_t>> expected_tiers{
1006+
{3, 4, 5, 6, 7},
1007+
{0, 1, 2},
1008+
{8, 9, 10, 11}
1009+
};
1010+
1011+
for (size_t i = 0; i < expected_tiers.size(); i++) {
1012+
auto& expected_tier = expected_tiers[i];
1013+
irs::Consolidation candidates;
1014+
policy(candidates, reader, consolidating_segments);
1015+
AssertCandidates(reader, expected_tier, candidates, "Line: " + std::to_string(__LINE__) + ", i = " + std::to_string(i));
1016+
candidates.clear();
1017+
policy(candidates, reader, consolidating_segments);
1018+
AssertCandidates(reader, expected_tier, candidates, "Line: " + std::to_string(__LINE__) + ", i = " + std::to_string(i));
1019+
// register candidates for consolidation
1020+
for (const auto* candidate : candidates) {
1021+
consolidating_segments.emplace(candidate->Meta().name);
1022+
}
1023+
}
1024+
}
1025+
}
1026+
1027+
TEST(ConsolidationTierTest, DISABLED_SkewedSegments) {
9201028
{
9211029
irs::index_utils::ConsolidateTier options;
9221030
// min number of segments per tier to merge at once

0 commit comments

Comments
 (0)