Added more unit tests for testing consolidation

Author: k0ushal

core/utils/index_utils.cpp

@@ -41,9 +41,9 @@ namespace tier {
         tier::SegmentAttributes& attrs) {
 
       auto* meta = segment.meta;
-      attrs.byte_size = meta->byte_size;
-      attrs.docs_count = meta->docs_count;
-      attrs.live_docs_count = meta->live_docs_count;
+      attrs.byteSize = meta->byte_size;
+      attrs.docsCount = meta->docs_count;
+      attrs.liveDocsCount = meta->live_docs_count;
     }
 }
 

core/utils/index_utils.hpp

@@ -71,13 +71,13 @@ namespace tier {
   };
 
   struct SegmentAttributes {
-    uint64_t byte_size { 0 };
-    uint64_t docs_count { 0 };
-    uint64_t live_docs_count { 0 };
+    uint64_t byteSize { 0 };
+    uint64_t docsCount { 0 };
+    uint64_t liveDocsCount { 0 };
 
     SegmentAttributes() = default;
     SegmentAttributes(uint64_t b, uint64_t d, uint64_t l) :
-      byte_size(b), docs_count(d), live_docs_count(l) {}
+      byteSize(b), docsCount(d), liveDocsCount(l) {}
   };
 
   //  interface to fetch the required attributes from
@@ -115,11 +115,11 @@ namespace tier {
       do
       {
         accessor_(*itr, attrs);
-        mergeBytes += attrs.byte_size;
+        mergeBytes += attrs.byteSize;
 
       } while (itr++ != end);
 
-      skew = static_cast<double>(attrs.byte_size) / mergeBytes;
+      skew = static_cast<double>(attrs.byteSize) / mergeBytes;
       mergeScore = skew;
     }
 
@@ -136,14 +136,14 @@ namespace tier {
       std::advance(segments.first, 1);
 
       //  Segment to be removed
-      SegmentAttributes rem_seg_attrs;
-      accessor_(*removeSegment, rem_seg_attrs);
+      SegmentAttributes remSegAttrs;
+      accessor_(*removeSegment, remSegAttrs);
 
-      SegmentAttributes last_seg_attrs;
-      accessor_(*lastSegment, last_seg_attrs);
+      SegmentAttributes lastSegAttrs;
+      accessor_(*lastSegment, lastSegAttrs);
 
-      mergeBytes -= rem_seg_attrs.byte_size;
-      skew = static_cast<double>(last_seg_attrs.byte_size) / mergeBytes;
+      mergeBytes -= remSegAttrs.byteSize;
+      skew = static_cast<double>(lastSegAttrs.byteSize) / mergeBytes;
       mergeScore = skew;
 
       return true;
@@ -164,8 +164,8 @@ namespace tier {
       SegmentAttributes attrs;
       accessor_(*addSegment, attrs);
 
-      mergeBytes += attrs.byte_size;
-      skew = static_cast<double>(attrs.byte_size) / mergeBytes;
+      mergeBytes += attrs.byteSize;
+      skew = static_cast<double>(attrs.byteSize) / mergeBytes;
       mergeScore = skew;
 
       return true;
@@ -188,38 +188,38 @@ namespace tier {
     std::vector<Segment>& segments,
     const std::function<
       void(const Segment&,
-            tier::SegmentAttributes& /* byte_size */
+            tier::SegmentAttributes&
           )>& getSegmentAttributes,
     tier::ConsolidationCandidate<Segment>& best) {
 
       auto segmentSortFunc = [&](const Segment& left, const Segment& right) {
 
         tier::SegmentAttributes attrs;
         getSegmentAttributes(left, attrs);
-        auto lLivePerc = static_cast<double>(attrs.live_docs_count) / attrs.docs_count;
+        auto lLivePerc = static_cast<double>(attrs.liveDocsCount) / attrs.docsCount;
 
         getSegmentAttributes(right, attrs);
-        auto rLivePerc = static_cast<double>(attrs.live_docs_count) / attrs.docs_count;
+        auto rLivePerc = static_cast<double>(attrs.liveDocsCount) / attrs.docsCount;
 
         return lLivePerc < rLivePerc;
       };
 
       std::sort(segments.begin(), segments.end(), segmentSortFunc);
 
       auto count = 0;
-      auto total_docs_count = 0;
-      auto total_live_docs_count = 0;
+      auto totalDocsCount = 0;
+      auto totalLiveDocsCount = 0;
       double livePerc;
 
       for (auto itr = segments.begin(); itr != segments.end(); itr++) {
 
         tier::SegmentAttributes attrs;
         getSegmentAttributes(*itr, attrs);
 
-        total_docs_count += attrs.docs_count;
-        total_live_docs_count += attrs.live_docs_count;
+        totalDocsCount += attrs.docsCount;
+        totalLiveDocsCount += attrs.liveDocsCount;
 
-        livePerc = static_cast<double>(total_live_docs_count) / total_docs_count;
+        livePerc = static_cast<double>(totalLiveDocsCount) / totalDocsCount;
         if (livePerc > tier::ConsolidationConfig::maxLivePercentage)
           break;
 
@@ -234,72 +234,85 @@ namespace tier {
     }
 
     //
-    //  This function receives a sorted vector of segments
-    //  and finds the best contiguous subset of segments to
-    //  merge together.
+    //  This function receives a set of segments and finds
+    //  the best subset to merge together.
     //  The best subset is defined as the one with the lowest
-    //  merge cost and the merge cost is computed inside the
-    //  ConslidateCandidate struct.
+    //  merge cost (i.e. skew). The merge cost is computed inside
+    //  the ConslidationCandidate struct upon candidate init,
+    //  push_back() and pop_front() operations.
     //
-    //  findBestConsolidationCandidate merely sets a rolling
-    //  window of size 4 on a subset of segments, it lets
-    //  ConsolidationCandidate compute the cost of merge for that
-    //  subset, repeats this process for all contiguous subsets
-    //  and identifies the best candidate.
+    //  findBestConsolidationCandidate sorts the set of segments
+    //  in the increasing order of the segment sizes and then finds
+    //  the largest possible subset of segments whose consolidated
+    //  size is within the maxSegmentsBytes range and has the
+    //  lowest skew.
+    //
+    //  Currently it is only executed with struct tier::SegmentStats
+    //  as the template argument in ArangoSearch. However we leverage
+    //  this templatized design for writing unit tests.
+    //
+    //  findBestConsolidationCandidate does not use the live %
+    //  to find the best candidate. It only needs the segment
+    //  byte size.
     //
     template<typename Segment>
     bool findBestConsolidationCandidate(
-      std::vector<Segment>& sorted_segments,
-      size_t max_segments_bytes,
+      std::vector<Segment>& segments,
+      size_t maxSegmentsBytes,
       const std::function<
         void(const Segment&,
              SegmentAttributes&
             )>& getSegmentAttributes,
       tier::ConsolidationCandidate<Segment>& best) {
 
+        //  sort segments by segment size
         auto comp = [&](const Segment& lhs, const Segment& rhs) {
 
-          SegmentAttributes l_attrs;
-          SegmentAttributes r_attrs;
+          SegmentAttributes lAttrs;
+          SegmentAttributes rAttrs;
+
+          getSegmentAttributes(lhs, lAttrs);
+          getSegmentAttributes(rhs, rAttrs);
 
-          getSegmentAttributes(lhs, l_attrs);          
-          getSegmentAttributes(rhs, r_attrs);
+          if (lAttrs.byteSize == rAttrs.byteSize) {
 
-          if (l_attrs.byte_size == r_attrs.byte_size) {
-            
-            double lfill_factor = static_cast<double>(l_attrs.live_docs_count) / l_attrs.docs_count;
-            double rfill_factor = static_cast<double>(r_attrs.live_docs_count) / r_attrs.docs_count;
+            double lfill_factor = static_cast<double>(lAttrs.liveDocsCount) / lAttrs.docsCount;
+            double rfill_factor = static_cast<double>(rAttrs.liveDocsCount) / rAttrs.docsCount;
             return lfill_factor > rfill_factor;
           }
 
-          return l_attrs.byte_size < r_attrs.byte_size;
+          return lAttrs.byteSize < rAttrs.byteSize;
         };
 
         //  sort segments in increasing order of the segment byte size
-        std::sort(sorted_segments.begin(), sorted_segments.end(), comp);
+        std::sort(segments.begin(), segments.end(), comp);
 
         //  We start with a min. window size of 2
         //  since a window of size 1 will always
         //  give us a skew of 1.0.
         uint64_t minWindowSize { tier::ConsolidationConfig::candidate_size };
-        auto front = sorted_segments.begin();
+        auto front = segments.begin();
         auto rear = front + minWindowSize - 1;
         tier::ConsolidationCandidate<Segment> candidate(front, rear, getSegmentAttributes);
 
         //  Algorithm:
-        //  We 
+        //  We start by setting the smallest possible window on the list of
+        //  sorted segments. We move the right end ahead to add more segments to
+        //  the window and we incrementally compute the merge cost for each subset.
+        //  We move the left end ahead to remove segments from the window and we
+        //  only do this when we're over the maxSegmentsBytes limit.
         while ((candidate.first() + minWindowSize - 1) <= candidate.last() &&
-                candidate.last() < sorted_segments.end()) {
+                candidate.last() < segments.end()) {
 
-          if (candidate.mergeBytes > max_segments_bytes) {
+          if (candidate.mergeBytes > maxSegmentsBytes) {
             candidate.pop_front();
             continue;
           }
 
           if (!best.initialized || best.mergeScore > candidate.mergeScore)
             best = candidate;
 
-          if (candidate.last() == (sorted_segments.end() - 1))
+          if (candidate.last() == (segments.end() - 1))
             break;
 
           candidate.push_back();
@@ -353,6 +366,8 @@ struct ConsolidateDocsFill {
 
 ConsolidationPolicy MakePolicy(const ConsolidateDocsFill& options);
 
+//  [TODO] Currently unused as the new algorithm uses a different
+//  approach. Only max_segments_bytes is in use.
 struct ConsolidateTier {
   // minimum allowed number of segments to consolidate at once
   size_t min_segments = 1;