MB-15009: Allow stored value age to be changed by cbepctl

Update the flush_param handling so the config value can be
changed.

Update how the defrag ages are configured on the visitor,
rather than wait for a full visit cycle, update the
values for each task run, this means for very large hash-tables
changes to the age values are realised much more quickly.

Change-Id: I89ef8defbb4759a2b0a1a45eab9e26c310f5b27f
Reviewed-on: http://review.couchbase.org/106931
Well-Formed: Build Bot <[email protected]>
Reviewed-by: Dave Rigby <[email protected]>
Tested-by: Build Bot <[email protected]>

Author: jimwwalker

engines/ep/benchmarks/defragmenter_bench.cc

@@ -101,10 +101,12 @@ class DefragmentBench : public benchmark::Fixture {
             std::chrono::milliseconds chunk_duration) {
         // Create and run visitor for the specified number of iterations, with
         // the given age.
-        DefragmentVisitor visitor(age_threshold,
-                                  DefragmenterTask::getMaxValueSize(
-                                          get_mock_server_api()->alloc_hooks),
-                                  age_threshold);
+        DefragmentVisitor visitor(DefragmenterTask::getMaxValueSize(
+                get_mock_server_api()->alloc_hooks));
+
+        visitor.setBlobAgeThreshold(age_threshold);
+        visitor.setStoredValueAgeThreshold(age_threshold);
+
         // Need to run 10 passes; so we allow the deframenter to defrag at
         // least once (given the age_threshold may be up to 10).
         const size_t passes = 10;

engines/ep/management/cbepctl

@@ -188,6 +188,9 @@ Available params for "set":
     defragmenter_age_threshold   - How old (measured in number of defragmenter
                                    passes) must a document be to be considered
                                    for defragmentation.
+    defragmenter_stored_value_age_threshold   - How old (measured in number of defragmenter
+                                   passes) must a StoredValue (key + meta) be to be considered
+                                   for defragmentation.
     defragmenter_chunk_duration  - Maximum time (in ms) defragmentation task
                                    will run for before being paused (and
                                    resumed at the next defragmenter_interval).

engines/ep/src/defragmenter.cc

@@ -40,19 +40,8 @@ bool DefragmenterTask::run(void) {
         // then resume from where we last were, otherwise create a new visitor
         // starting from the beginning.
         if (!prAdapter) {
-            boost::optional<uint8_t> storedValueAge;
-
-            // Only defragment persistent buckets because the HashTable defrag
-            // method doesn't yet know how to maintain the ephemeral seqno
-            // linked-list
-            if (engine->getConfiguration().getBucketType() == "persistent") {
-                storedValueAge = getStoredValueAgeThreshold();
-            }
-
             auto visitor = std::make_unique<DefragmentVisitor>(
-                    getAgeThreshold(),
-                    getMaxValueSize(alloc_hooks),
-                    storedValueAge);
+                    getMaxValueSize(alloc_hooks));
 
             prAdapter =
                     std::make_unique<PauseResumeVBAdapter>(std::move(visitor));
@@ -83,6 +72,13 @@ bool DefragmenterTask::run(void) {
         const auto start = ProcessClock::now();
         const auto deadline = start + getChunkDuration();
         visitor.setDeadline(deadline);
+        visitor.setBlobAgeThreshold(getAgeThreshold());
+        // Only defragment StoredValues of persistent buckets because the
+        // HashTable defrag method doesn't yet know how to maintain the
+        // ephemeral seqno linked-list
+        if (engine->getConfiguration().getBucketType() == "persistent") {
+            visitor.setStoredValueAgeThreshold(getStoredValueAgeThreshold());
+        }
         visitor.clearStats();
 
         // Do it - set off the visitor.

engines/ep/src/defragmenter_visitor.cc

@@ -19,15 +19,11 @@
 
 // DegragmentVisitor implementation ///////////////////////////////////////////
 
-DefragmentVisitor::DefragmentVisitor(uint8_t age_threshold_,
-                                     size_t max_size_class,
-                                     boost::optional<uint8_t> sv_age_threshold)
+DefragmentVisitor::DefragmentVisitor(size_t max_size_class)
     : max_size_class(max_size_class),
-      age_threshold(age_threshold_),
       defrag_count(0),
       visited_count(0),
-      currentVb(nullptr),
-      sv_age_threshold(sv_age_threshold) {
+      currentVb(nullptr) {
 }
 
 DefragmentVisitor::~DefragmentVisitor() {
@@ -37,6 +33,14 @@ void DefragmentVisitor::setDeadline(ProcessClock::time_point deadline) {
     progressTracker.setDeadline(deadline);
 }
 
+void DefragmentVisitor::setBlobAgeThreshold(uint8_t age) {
+    age_threshold = age;
+}
+
+void DefragmentVisitor::setStoredValueAgeThreshold(uint8_t age) {
+    sv_age_threshold = age;
+}
+
 bool DefragmentVisitor::visit(const HashTable::HashBucketLock& lh,
                               StoredValue& v) {
     const size_t value_len = v.valuelen();

engines/ep/src/defragmenter_visitor.h

@@ -31,15 +31,24 @@
  */
 class DefragmentVisitor : public VBucketAwareHTVisitor {
 public:
-    DefragmentVisitor(uint8_t age_threshold_,
-                      size_t max_size_class,
-                      boost::optional<uint8_t> sv_age_threshold);
+    DefragmentVisitor(size_t max_size_class);
 
     ~DefragmentVisitor();
 
     // Set the deadline at which point the visitor will pause visiting.
     void setDeadline(ProcessClock::time_point deadline_);
 
+    /**
+     * Set the age at which Blobs are defragged 0 - 255
+     */
+    void setBlobAgeThreshold(uint8_t age);
+
+    /**
+     * Set the age at which StoredValues are defragged - by default SV
+     * defragging is off until an age is set (0-255)
+     */
+    void setStoredValueAgeThreshold(uint8_t age);
+
     // Implementation of HashTableVisitor interface:
     virtual bool visit(const HashTable::HashBucketLock& lh,
                        StoredValue& v) override;
@@ -68,7 +77,7 @@ class DefragmentVisitor : public VBucketAwareHTVisitor {
     const size_t max_size_class;
 
     // How old a blob must be to consider it for defragmentation.
-    const uint8_t age_threshold;
+    uint8_t age_threshold{0};
 
     /* Runtime state */
 

engines/ep/src/ep_engine.cc

@@ -592,6 +592,10 @@ protocol_binary_response_status EventuallyPersistentEngine::setFlushParam(
                     std::stoull(valz));
         } else if (strcmp(keyz, "defragmenter_age_threshold") == 0) {
             getConfiguration().setDefragmenterAgeThreshold(std::stoull(valz));
+        } else if (strcmp(keyz, "defragmenter_stored_value_age_threshold") ==
+                   0) {
+            getConfiguration().setDefragmenterStoredValueAgeThreshold(
+                    std::stoull(valz));
         } else if (strcmp(keyz, "defragmenter_chunk_duration") == 0) {
             getConfiguration().setDefragmenterChunkDuration(std::stoull(valz));
         } else if (strcmp(keyz, "defragmenter_run") == 0) {

engines/ep/tests/module_tests/defragmenter_test.cc

@@ -252,21 +252,15 @@ TEST_P(DefragmenterTest, DISABLED_MappedMemory) {
     // 3. Enable defragmenter and trigger defragmentation
     AllocHooks::enable_thread_cache(false);
 
-    // Setup the StoredValue de-frag age only if in StoredValue mode
-    boost::optional<uint8_t> storedValueAge;
-    if (isModeStoredValue()) {
-        storedValueAge = 0; // Use zero so all SV's get moved.
-    }
-
     auto defragVisitor = std::make_unique<DefragmentVisitor>(
-            0,
             DefragmenterTask::getMaxValueSize(
-                    get_mock_server_api()->alloc_hooks),
-            storedValueAge);
+                    get_mock_server_api()->alloc_hooks));
 
     if (isModeStoredValue()) {
         // Force visiting of every item by setting a large deadline
         defragVisitor->setDeadline(ProcessClock::now() + std::chrono::hours(5));
+        // And set the age, which enables SV defragging
+        defragVisitor->setStoredValueAgeThreshold(0);
     }
 
     PauseResumeVBAdapter prAdapter(std::move(defragVisitor));
@@ -363,10 +357,8 @@ TEST_P(DefragmenterTest, DISABLED_RefCountMemUsage) {
         AllocHooks::enable_thread_cache(false);
 
         PauseResumeVBAdapter prAdapter(std::make_unique<DefragmentVisitor>(
-                0,
                 DefragmenterTask::getMaxValueSize(
-                        get_mock_server_api()->alloc_hooks),
-                boost::optional<uint8_t>{/*no sv defragging*/}));
+                        get_mock_server_api()->alloc_hooks)));
         prAdapter.visit(*vbucket);
 
         AllocHooks::enable_thread_cache(true);