Skip to content

Commit 4eb4b06

Browse files
k0ushalk0ushal
committed
Changed ConsolidationTier unit tests to follow the new algorithm
Disabled irrelevant tests
1 parent c4ce867 commit 4eb4b06

File tree

1 file changed

+156
-48
lines changed

1 file changed

+156
-48
lines changed

‎tests/index/consolidation_policy_tests.cpp‎

Lines changed: 156 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
/// @author Vasiliy Nabatchikov
2222
////////////////////////////////////////////////////////////////////////////////
2323

24-
#include <iostream>
2524
#include "index/composite_reader_impl.hpp"
2625
#include "index/index_meta.hpp"
2726
#include "index/index_writer.hpp"
@@ -74,8 +73,9 @@ namespace {
7473

7574
void AssertCandidates(const irs::IndexReader& reader,
7675
const std::vector<size_t>& expected_candidates,
77-
const irs::Consolidation& actual_candidates) {
78-
ASSERT_EQ(expected_candidates.size(), actual_candidates.size());
76+
const irs::Consolidation& actual_candidates,
77+
const std::string& errMsg = "") {
78+
ASSERT_EQ(expected_candidates.size(), actual_candidates.size()) << errMsg;
7979

8080
for (const size_t expected_candidate_idx : expected_candidates) {
8181
const auto& expected_candidate = reader[expected_candidate_idx];
@@ -189,7 +189,7 @@ void AddSegment(irs::IndexMeta& meta, std::string_view name,
189189

190190
} // namespace
191191

192-
TEST(ConsolidationTierTest, MaxConsolidationSize) {
192+
TEST(ConsolidationTierTest, DISABLED_MaxConsolidationSize) {
193193
irs::IndexMeta meta;
194194
for (size_t i = 0; i < 22; ++i) {
195195
AddSegment(meta, std::to_string(i), 1, 1, 1);
@@ -321,7 +321,7 @@ TEST(ConsolidationTierTest, EmptySegment) {
321321
ASSERT_TRUE(candidates.empty()); // skip empty segments
322322
}
323323

324-
TEST(ConsolidationTierTest, MaxConsolidationCount) {
324+
TEST(ConsolidationTierTest, DISABLED_MaxConsolidationCount) {
325325
// generate meta
326326
irs::IndexMeta meta;
327327
for (size_t i = 0; i < 22; ++i) {
@@ -471,7 +471,7 @@ TEST(ConsolidationTierTest, MaxConsolidationCount) {
471471
}
472472
}
473473

474-
TEST(ConsolidationTierTest, MinConsolidationCount) {
474+
TEST(ConsolidationTierTest, DISABLED_MinConsolidationCount) {
475475
// generate meta
476476
irs::IndexMeta meta;
477477
for (size_t i = 0; i < 22; ++i) {
@@ -633,7 +633,7 @@ TEST(ConsolidationTierTest, MinConsolidationCount) {
633633
}
634634
}
635635

636-
TEST(ConsolidationTierTest, ConsolidationFloor) {
636+
TEST(ConsolidationTierTest, DISABLED_ConsolidationFloor) {
637637
// generate meta
638638
irs::IndexMeta meta;
639639
{
@@ -723,46 +723,60 @@ TEST(ConsolidationTierTest, ConsolidationFloor) {
723723
}
724724
}
725725

726-
TEST(ConsolidationTierTest, PreferSegmentsWithRemovals) {
727-
// generate meta
726+
TEST(ConsolidationTierTest, PreferConsolidationOverCleanupWhenDoesntMeetThreshold) {
728727
irs::IndexMeta meta;
729728
AddSegment(meta, "0", 10, 10, 10);
730729
AddSegment(meta, "1", 10, 10, 10);
731-
AddSegment(meta, "2", 11, 10, 11);
732-
AddSegment(meta, "3", 11, 10, 11);
730+
AddSegment(meta, "2", 10, 10, 10);
731+
AddSegment(meta, "3", 10, 10, 10);
732+
AddSegment(meta, "4", 10, 10, 10);
733+
AddSegment(meta, "5", 11, 8, 81);
734+
AddSegment(meta, "6", 10, 9, 91);
733735
IndexReaderMock reader{meta};
734736

735-
// ensure policy prefers segments with removals
737+
// The conslidation threshold defaults are defined
738+
// in tier::ConsolidationConfig.
739+
736740
irs::index_utils::ConsolidateTier options;
737741
options.floor_segment_bytes = 1;
738742
options.max_segments = 2;
739-
options.min_segments = 1;
743+
options.min_segments = 2;
740744
options.max_segments_bytes = std::numeric_limits<size_t>::max();
741745

742746
irs::ConsolidatingSegments consolidating_segments;
743747
auto policy = irs::index_utils::MakePolicy(options);
744748

745-
const std::vector<std::vector<size_t>> expected_tiers{{2, 3}, {0, 1}};
749+
irs::Consolidation candidates;
750+
policy(candidates, reader, consolidating_segments);
751+
ASSERT_EQ(5, candidates.size());
752+
}
746753

747-
for (auto& expected_tier : expected_tiers) {
748-
irs::Consolidation candidates;
749-
policy(candidates, reader, consolidating_segments);
750-
AssertCandidates(reader, expected_tier, candidates);
751-
candidates.clear();
752-
policy(candidates, reader, consolidating_segments);
753-
AssertCandidates(reader, expected_tier, candidates);
754-
// register candidates for consolidation
755-
for (const auto* candidate : candidates) {
756-
consolidating_segments.emplace(candidate->Meta().name);
757-
}
758-
}
754+
TEST(ConsolidationTierTest, PreferCleanupWhenMeetsThreshold) {
755+
// generate meta
756+
irs::IndexMeta meta;
757+
AddSegment(meta, "0", 10, 10, 10);
758+
AddSegment(meta, "1", 10, 10, 10);
759+
AddSegment(meta, "2", 10, 10, 10);
760+
AddSegment(meta, "3", 10, 10, 10);
761+
AddSegment(meta, "4", 10, 10, 10);
762+
AddSegment(meta, "5", 11, 5, 11);
763+
AddSegment(meta, "6", 10, 5, 11);
764+
IndexReaderMock reader{meta};
759765

760-
// no more segments to consolidate
761-
{
762-
irs::Consolidation candidates;
763-
policy(candidates, reader, consolidating_segments);
764-
ASSERT_TRUE(candidates.empty());
765-
}
766+
// ensure policy prefers segments with removals
767+
irs::index_utils::ConsolidateTier options;
768+
options.floor_segment_bytes = 1;
769+
options.max_segments = 2;
770+
options.min_segments = 2;
771+
options.max_segments_bytes = std::numeric_limits<size_t>::max();
772+
773+
irs::ConsolidatingSegments consolidating_segments;
774+
auto policy = irs::index_utils::MakePolicy(options);
775+
776+
irs::Consolidation candidates;
777+
policy(candidates, reader, consolidating_segments);
778+
ASSERT_EQ(2, candidates.size());
779+
AssertCandidates(reader, {5, 6}, candidates);
766780
}
767781

768782
TEST(ConsolidationTierTest, Singleton) {
@@ -788,31 +802,34 @@ TEST(ConsolidationTierTest, Singleton) {
788802
}
789803
}
790804

791-
// singleton consolidation with removals
805+
// singleton consolidation with < 50% removals
792806
{
793807
irs::ConsolidatingSegments consolidating_segments;
794808
irs::IndexMeta meta;
795-
AddSegment(meta, "0", 100, 99, 150);
809+
AddSegment(meta, "0", 100, 51, 150);
796810
IndexReaderMock reader{meta};
797811

798-
// 1st tier
799812
{
800813
irs::Consolidation candidates;
801814
policy(candidates, reader, consolidating_segments);
802-
AssertCandidates(reader, {0}, candidates);
803-
candidates.clear();
804-
policy(candidates, reader, consolidating_segments);
805-
AssertCandidates(reader, {0}, candidates);
806-
// register candidates for consolidation
807-
for (const auto* candidate : candidates) {
808-
consolidating_segments.emplace(candidate->Meta().name);
809-
}
815+
ASSERT_TRUE(candidates.empty());
810816
}
817+
}
818+
819+
// singleton consolidation with >= 50% removals
820+
{
821+
irs::ConsolidatingSegments consolidating_segments;
822+
irs::IndexMeta meta;
823+
AddSegment(meta, "0", 100, 49, 150);
824+
AddSegment(meta, "1", 100, 59, 150);
825+
AddSegment(meta, "2", 100, 69, 150);
826+
AddSegment(meta, "3", 100, 50, 150);
827+
IndexReaderMock reader{meta};
811828

812829
{
813830
irs::Consolidation candidates;
814831
policy(candidates, reader, consolidating_segments);
815-
ASSERT_TRUE(candidates.empty());
832+
AssertCandidates(reader, {0, 3}, candidates);
816833
}
817834
}
818835
}
@@ -836,10 +853,10 @@ TEST(ConsolidationTierTest, Defaults) {
836853
{
837854
irs::Consolidation candidates;
838855
policy(candidates, reader, consolidating_segments);
839-
AssertCandidates(reader, {0, 1, 2, 3, 4}, candidates);
856+
AssertCandidates(reader, {1, 2, 3, 4}, candidates, "Line: " + std::to_string(__LINE__));
840857
candidates.clear();
841858
policy(candidates, reader, consolidating_segments);
842-
AssertCandidates(reader, {0, 1, 2, 3, 4}, candidates);
859+
AssertCandidates(reader, {1, 2, 3, 4}, candidates, "Line: " + std::to_string(__LINE__));
843860
// register candidates for consolidation
844861
for (const auto* candidate : candidates) {
845862
consolidating_segments.emplace(candidate->Meta().name);
@@ -874,10 +891,10 @@ TEST(ConsolidationTierTest, Defaults) {
874891
{
875892
irs::Consolidation candidates;
876893
policy(candidates, reader, consolidating_segments);
877-
AssertCandidates(reader, {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, candidates);
894+
AssertCandidates(reader, {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, candidates, "Line: " + std::to_string(__LINE__));
878895
candidates.clear();
879896
policy(candidates, reader, consolidating_segments);
880-
AssertCandidates(reader, {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, candidates);
897+
AssertCandidates(reader, {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, candidates, "Line: " + std::to_string(__LINE__));
881898
// register candidates for consolidation
882899
for (const auto* candidate : candidates) {
883900
consolidating_segments.emplace(candidate->Meta().name);
@@ -939,6 +956,97 @@ TEST(ConsolidationTierTest, NoCandidates) {
939956
}
940957

941958
TEST(ConsolidationTierTest, SkewedSegments) {
959+
irs::index_utils::ConsolidateTier options;
960+
options.max_segments_bytes = 52500; // max size of the merge
961+
962+
auto policy = irs::index_utils::MakePolicy(options);
963+
964+
// test correct selection of candidates
965+
{
966+
irs::ConsolidatingSegments consolidating_segments;
967+
irs::IndexMeta meta;
968+
AddSegment(meta, "0", 100, 100, 10);
969+
AddSegment(meta, "1", 100, 100, 40);
970+
AddSegment(meta, "2", 100, 100, 60);
971+
AddSegment(meta, "3", 100, 100, 70);
972+
AddSegment(meta, "4", 100, 100, 100);
973+
AddSegment(meta, "5", 100, 100, 150);
974+
AddSegment(meta, "6", 100, 100, 200);
975+
AddSegment(meta, "7", 100, 100, 500);
976+
AddSegment(meta, "8", 100, 100, 750);
977+
AddSegment(meta, "9", 100, 100, 1100);
978+
AddSegment(meta, "10", 100, 100, 90);
979+
AddSegment(meta, "11", 100, 100, 75);
980+
AddSegment(meta, "12", 100, 100, 1500);
981+
AddSegment(meta, "13", 100, 100, 10000);
982+
AddSegment(meta, "14", 100, 100, 5000);
983+
AddSegment(meta, "15", 100, 100, 1750);
984+
AddSegment(meta, "16", 100, 100, 690);
985+
IndexReaderMock reader{meta};
986+
987+
const std::vector<std::vector<size_t>> expected_tiers{
988+
{0, 1, 2, 3, 4, 10, 11},
989+
{7, 8, 9, 12, 15, 16},
990+
{}
991+
};
992+
993+
for (size_t i = 0; i < expected_tiers.size(); i++) {
994+
auto& expected_tier = expected_tiers[i];
995+
irs::Consolidation candidates;
996+
policy(candidates, reader, consolidating_segments);
997+
AssertCandidates(reader, expected_tier, candidates, "Line: " + std::to_string(__LINE__) + ", i = " + std::to_string(i));
998+
candidates.clear();
999+
policy(candidates, reader, consolidating_segments);
1000+
AssertCandidates(reader, expected_tier, candidates, "Line: " + std::to_string(__LINE__) + ", i = " + std::to_string(i));
1001+
// register candidates for consolidation
1002+
for (const auto* candidate : candidates) {
1003+
consolidating_segments.emplace(candidate->Meta().name);
1004+
}
1005+
}
1006+
}
1007+
1008+
{
1009+
irs::ConsolidatingSegments consolidating_segments;
1010+
irs::IndexMeta meta;
1011+
AddSegment(meta, "0", 100, 100, 1);
1012+
AddSegment(meta, "1", 100, 100, 1);
1013+
AddSegment(meta, "2", 100, 100, 1);
1014+
AddSegment(meta, "3", 100, 100, 75);
1015+
AddSegment(meta, "4", 100, 100, 90);
1016+
AddSegment(meta, "5", 100, 100, 100);
1017+
AddSegment(meta, "6", 100, 100, 150);
1018+
AddSegment(meta, "7", 100, 100, 200);
1019+
AddSegment(meta, "8", 100, 100, 750);
1020+
AddSegment(meta, "9", 100, 100, 1100);
1021+
AddSegment(meta, "10", 100, 100, 1500);
1022+
AddSegment(meta, "11", 100, 100, 1750);
1023+
AddSegment(meta, "12", 100, 100, 5000);
1024+
AddSegment(meta, "13", 100, 100, 10000);
1025+
IndexReaderMock reader{meta};
1026+
1027+
const std::vector<std::vector<size_t>> expected_tiers{
1028+
{3, 4, 5, 6, 7},
1029+
{0, 1, 2},
1030+
{8, 9, 10, 11}
1031+
};
1032+
1033+
for (size_t i = 0; i < expected_tiers.size(); i++) {
1034+
auto& expected_tier = expected_tiers[i];
1035+
irs::Consolidation candidates;
1036+
policy(candidates, reader, consolidating_segments);
1037+
AssertCandidates(reader, expected_tier, candidates, "Line: " + std::to_string(__LINE__) + ", i = " + std::to_string(i));
1038+
candidates.clear();
1039+
policy(candidates, reader, consolidating_segments);
1040+
AssertCandidates(reader, expected_tier, candidates, "Line: " + std::to_string(__LINE__) + ", i = " + std::to_string(i));
1041+
// register candidates for consolidation
1042+
for (const auto* candidate : candidates) {
1043+
consolidating_segments.emplace(candidate->Meta().name);
1044+
}
1045+
}
1046+
}
1047+
}
1048+
1049+
TEST(ConsolidationTierTest, DISABLED_SkewedSegments) {
9421050
{
9431051
irs::index_utils::ConsolidateTier options;
9441052
// min number of segments per tier to merge at once

0 commit comments

Comments
 (0)