add range remove crash test

Author: Ravi Nagarjun Akella

conanfile.py

@@ -9,7 +9,7 @@
 
 class HomestoreConan(ConanFile):
     name = "homestore"
-    version = "6.20.24"
+    version = "6.20.25"
 
     homepage = "https://github.com/eBay/Homestore"
     description = "HomeStore Storage Engine"
@@ -53,7 +53,7 @@ def build_requirements(self):
 
     def requirements(self):
         self.requires("iomgr/[^11.3]@oss/master", transitive_headers=True)
-        self.requires("sisl/12.4.11@oss/master", transitive_headers=True)
+        self.requires("sisl/[^12.4]@oss/master", transitive_headers=True)
         self.requires("nuraft_mesg/[~3.8.7]@oss/main", transitive_headers=True)
 
         self.requires("farmhash/cci.20190513@", transitive_headers=True)

src/tests/btree_helpers/btree_test_helper.hpp

@@ -329,6 +329,10 @@ struct BtreeTestHelper {
         do_range_remove(start_k, end_k, false /* removing_all_existing */);
     }
 
+    void range_remove_all(uint32_t start_k, uint32_t end_k) {
+        do_range_remove(start_k, end_k, true /* removing_all_existing */);
+    }
+
     ////////////////////// All query operation variants ///////////////////////////////
     void query_all() { do_query(0u, SISL_OPTIONS["num_entries"].as< uint32_t >() - 1, UINT32_MAX); }
 

src/tests/btree_helpers/shadow_map.hpp

@@ -19,6 +19,7 @@ class ShadowMap {
 public:
     ShadowMap(uint32_t num_keys) : m_range_scheduler(num_keys), m_max_keys{num_keys} {}
 
+    using pick_existing_range_cb_t = std::function< void(const K&) >;
     void put_and_check(const K& key, const V& val, const V& old_val, bool expected_success) {
         std::lock_guard lock{m_mutex};
         auto const [it, happened] = m_map.insert(std::make_pair(key, val));
@@ -69,6 +70,18 @@ class ShadowMap {
         return std::pair(start_it->first, it->first);
     }
 
+    std::pair< K, K > pick_existing_range(const K& start_key, uint32_t max_count, pick_existing_range_cb_t cb) const {
+        std::lock_guard lock{m_mutex};
+        auto const start_it = m_map.lower_bound(start_key);
+        auto it = start_it;
+        uint32_t count = 0;
+        while ((it != m_map.cend()) && (++count < max_count)) {
+            cb(it->first);
+            ++it;
+        }
+        return std::pair(start_it->first, it->first);
+    }
+
     uint32_t max_keys() const { return m_max_keys; }
 
     bool exists(const K& key) const {

src/tests/test_index_crash_recovery.cpp

@@ -236,6 +236,10 @@ class SequenceGenerator {
 
     void reset() { keyStates.clear(); }
 
+    std::map< uint64_t, bool >& getKeyStates() { return keyStates; }
+    std::atomic< uint64_t >& getInUseKeyCount() { return in_use_key_cnt_; }
+    std::uniform_int_distribution<>& getKeyDistribution() { return keyDist_; }
+
 private:
     int putFreq_;
     int removeFreq_;
@@ -264,6 +268,7 @@ struct long_running_crash_options {
     uint32_t num_entries_per_rounds{SISL_OPTIONS["num_entries_per_rounds"].as< uint32_t >()};
     bool load_mode{SISL_OPTIONS.count("load_from_file") > 0};
     bool save_mode{SISL_OPTIONS.count("save_to_file") > 0};
+    bool range_remove_{false};
 };
 
 template < typename TestType >
@@ -545,6 +550,13 @@ struct IndexCrashTest : public test_common::HSTestHelper, BtreeTestHelper< TestT
 
     uint32_t tree_key_count() { return this->m_bt->count_keys(this->m_bt->root_node_id()); }
 
+    std::pair<uint32_t, uint32_t> range_remove_op(SequenceGenerator& generator, uint32_t range_remove_count) {
+        uint32_t key = generator.getKeyDistribution()(g_re); 
+        auto [start_key, end_key] = this->m_shadow_map.pick_existing_range(K{key}, range_remove_count,
+                [&generator](const K& key) { generator.getKeyStates()[key.key()] = false; generator.getInUseKeyCount().fetch_sub(1); });
+        return {start_key.key(), end_key.key()};
+    }
+
     void long_running_crash(long_running_crash_options const& crash_test_options) {
         // set putFreq 100 for the initial load
         SequenceGenerator generator(100 /*putFreq*/, 0 /* removeFreq*/, 0 /*start_range*/,
@@ -719,6 +731,12 @@ struct IndexCrashTest : public test_common::HSTestHelper, BtreeTestHelper< TestT
                     }
                 }
             }
+            if (crash_test_options.range_remove_) {
+                // add one range remove operation
+                auto op = this->range_remove_op(generator, crash_test_options.num_entries_per_rounds);
+                LOGDEBUG("Range removing keys [{}, {})", op.first, op.second);
+                this->range_remove_all(op.first, op.second);
+            }
             if (normal_execution) {
                 if (clean_shutdown) {
                     this->m_shadow_map.save(this->m_shadow_filename);
@@ -881,6 +899,19 @@ TYPED_TEST(IndexCrashTest, long_running_put_remove_crash) {
     this->long_running_crash(crash_test_options);
 }
 
+TYPED_TEST(IndexCrashTest, long_running_put_range_remove_crash) {
+    long_running_crash_options crash_test_options{
+        // put freq should be 100 for range remove test
+        .put_freq = 100,
+        .put_flips = {"crash_flush_on_split_at_parent", "crash_flush_on_split_at_left_child",
+                      "crash_flush_on_split_at_right_child"},
+        .remove_flips = {"crash_flush_on_merge_at_parent", "crash_flush_on_merge_at_left_child"
+                         /*, "crash_flush_on_freed_child"*/},
+        .range_remove_ = true,
+    };
+    this->long_running_crash(crash_test_options);
+}
+
 // Basic reverse and forward order remove with different flip points
 TYPED_TEST(IndexCrashTest, MergeRemoveBasic) {
     vector< std::string > flip_points = {