Add multi threaded UT to trigger cache eviction

Author: Ravi Nagarjun Akella

conanfile.py

@@ -9,7 +9,7 @@
 
 class HomestoreConan(ConanFile):
     name = "homestore"
-    version = "6.20.23"
+    version = "6.20.24"
 
     homepage = "https://github.com/eBay/Homestore"
     description = "HomeStore Storage Engine"
@@ -53,7 +53,7 @@ def build_requirements(self):
 
     def requirements(self):
         self.requires("iomgr/[^11.3]@oss/master", transitive_headers=True)
-        self.requires("sisl/[^12.4.7]@oss/master", transitive_headers=True)
+        self.requires("sisl/12.4.11@oss/master", transitive_headers=True)
         self.requires("nuraft_mesg/[~3.8.7]@oss/main", transitive_headers=True)
 
         self.requires("farmhash/cci.20190513@", transitive_headers=True)

src/lib/index/wb_cache.cpp

@@ -51,7 +51,8 @@ IndexWBCache::IndexWBCache(const std::shared_ptr< VirtualDev >& vdev, std::pair<
                 },
                 [](const sisl::CacheRecord& rec) -> bool {
                     const auto& hnode = (sisl::SingleEntryHashNode< BtreeNodePtr >&)rec;
-                    return static_cast< IndexBtreeNode* >(hnode.m_value.get())->m_idx_buf->is_clean();
+                    auto idx_buf = static_cast< IndexBtreeNode* >(hnode.m_value.get())->m_idx_buf;
+                    return !idx_buf || idx_buf->is_clean();
                 }},
         m_node_size{node_size},
         m_meta_blk{sb.first} {

src/tests/btree_helpers/btree_test_helper.hpp

@@ -50,6 +50,7 @@ struct BtreeTestHelper {
         m_max_range_input = SISL_OPTIONS["num_entries"].as< uint32_t >();
 
         if (m_is_multi_threaded) {
+            m_fibers.clear();
             std::mutex mtx;
             m_run_time = SISL_OPTIONS["run_time"].as< uint32_t >();
             iomanager.run_on_wait(iomgr::reactor_regex::all_worker, [this, &mtx]() {

src/tests/test_common/homestore_test_common.hpp

@@ -31,6 +31,7 @@
 #include <device/hs_super_blk.h>
 #include <iomgr/iomgr_config_generated.h>
 #include <common/homestore_assert.hpp>
+#include <iomgr/http_server.hpp>
 
 #ifdef _PRERELEASE
 #include "common/crash_simulator.hpp"
@@ -70,6 +71,34 @@ using namespace homestore;
 
 namespace test_common {
 
+class test_http_server {
+public:
+    void get_prometheus_metrics(const Pistache::Rest::Request&, Pistache::Http::ResponseWriter response) {
+        response.send(Pistache::Http::Code::Ok,
+                      sisl::MetricsFarm::getInstance().report(sisl::ReportFormat::kTextFormat));
+    }
+
+    void start() {
+        auto http_server_ptr = ioenvironment.get_http_server();
+
+        std::vector< iomgr::http_route > routes = {
+            {Pistache::Http::Method::Get, "/metrics",
+             Pistache::Rest::Routes::bind(&test_http_server::get_prometheus_metrics, this), iomgr::url_t::safe}};
+        try {
+            http_server_ptr->setup_routes(routes);
+            LOGINFO("Started http server ");
+        } catch (std::runtime_error const& e) { LOGERROR("setup routes failed, {}", e.what()) }
+
+        // start the server
+        http_server_ptr->start();
+    }
+
+    void stop() {
+        auto http_server_ptr = ioenvironment.get_http_server();
+        http_server_ptr->stop();
+    }
+};
+
 // Fix a port for http server
 inline static void set_fixed_http_port(uint32_t http_port) {
     SETTINGS_FACTORY(iomgr_config).modifiable_settings([http_port](auto& s) { s.io_env->http_port = http_port; });
@@ -362,6 +391,12 @@ class HSTestHelper {
         }
     }
 
+    test_http_server* get_http_server() {
+        return m_http_server.get();
+    }
+
+    void set_app_mem_size(uint64_t app_mem_size) { m_app_mem_size = app_mem_size; }
+
 private:
     void do_start_homestore(bool fake_restart = false, bool init_device = true, uint32_t shutdown_delay_sec = 5) {
         auto const ndevices = SISL_OPTIONS["num_devs"].as< uint32_t >();
@@ -429,7 +464,7 @@ class HSTestHelper {
             ioenvironment.with_http_server();
         }
 
-        const uint64_t app_mem_size = ((ndevices * dev_size) * 15) / 100;
+        const uint64_t app_mem_size = (m_app_mem_size == 0) ? (((ndevices * dev_size) * 15) / 100) : m_app_mem_size;
         LOGINFO("Initialize and start HomeStore with app_mem_size = {}", homestore::in_bytes(app_mem_size));
 
         using namespace homestore;
@@ -530,6 +565,8 @@ class HSTestHelper {
 protected:
     test_token m_token;
     std::vector< std::string > m_generated_devs;
+    std::unique_ptr< test_http_server > m_http_server;
+    uint64_t m_app_mem_size{0};
 #ifdef _PRERELEASE
     flip::FlipClient m_fc{iomgr_flip::instance()};
     folly::Promise< folly::Unit > m_crash_recovered;

src/tests/test_index_btree.cpp

@@ -51,6 +51,10 @@ SISL_OPTION_GROUP(
      ::cxxopts::value< bool >()->default_value("1"), ""),
     (max_merge_level, "", "max_merge_level", "max merge level", ::cxxopts::value< uint8_t >()->default_value("127"),
      ""),
+    (cache_pct, "", "cache_pct", "cache percentage", ::cxxopts::value< uint32_t >()->default_value("10"),
+     ""),
+    (app_mem_size_mb, "", "app_mem_size_mb", "application memory size", ::cxxopts::value< uint64_t >()->default_value("900"),
+     ""),
     (seed, "", "seed", "random engine seed, use random if not defined",
      ::cxxopts::value< uint64_t >()->default_value("0"), "number"))
 
@@ -108,13 +112,18 @@ struct BtreeTest : public BtreeTestHelper< TestType >, public ::testing::Test {
         this->m_bt = std::make_shared< typename T::BtreeType >(uuid, parent_uuid, 0, this->m_cfg);
         hs()->index_service().add_index_table(this->m_bt);
         LOGINFO("Added index table to index service");
+
+        //start http server
+        m_helper.get_http_server()->start();
+        LOGINFO("Started http server");
     }
 
     void TearDown() override {
         BtreeTestHelper< TestType >::TearDown();
         auto [interior, leaf] = this->m_bt->compute_node_count();
         LOGINFO("Teardown with Root bnode_id {} tree size: {} btree node count (interior = {} leaf= {})",
                 this->m_bt->root_node_id(), this->m_bt->count_keys(this->m_bt->root_node_id()), interior, leaf);
+        m_helper.get_http_server()->stop();
         m_helper.shutdown_homestore(false);
         this->m_bt.reset();
         log_obj_life_counter();
@@ -479,6 +488,11 @@ struct BtreeConcurrentTest : public BtreeTestHelper< TestType >, public ::testin
     void restart_homestore() {
         m_helper.params(HS_SERVICE::INDEX).index_svc_cbs = new TestIndexServiceCallbacks(this);
         m_helper.restart_homestore();
+        BtreeTestHelper< TestType >::SetUp();
+    }
+
+    void set_app_mem_size(uint64_t app_mem_size) {
+        m_helper.set_app_mem_size(app_mem_size);
     }
 
     void SetUp() override {
@@ -558,6 +572,34 @@ TYPED_TEST(BtreeConcurrentTest, ConcurrentAllOps) {
     this->multi_op_execute(ops, !SISL_OPTIONS["init_device"].as< bool >());
 }
 
+TYPED_TEST(BtreeConcurrentTest, ConcurrentAllOpsCacheEviction) {
+    // restart homestore with smaller cache %
+    HS_SETTINGS_FACTORY().modifiable_settings([](auto& s) {
+        s.resource_limits.cache_size_percent = SISL_OPTIONS["cache_pct"].as< uint32_t >();
+        HS_SETTINGS_FACTORY().save();
+    });
+    this->set_app_mem_size(SISL_OPTIONS["app_mem_size_mb"].as< uint64_t >() * 1024 * 1024);
+
+    this->restart_homestore();
+
+    // range put is not supported for non-extent keys
+    std::vector< std::string > input_ops = {"put:80", "remove:10", "query:10"};
+    if (SISL_OPTIONS.count("operation_list")) {
+        input_ops = SISL_OPTIONS["operation_list"].as< std::vector< std::string > >();
+    }
+    auto ops = this->build_op_list(input_ops);
+
+    this->multi_op_execute(ops, !SISL_OPTIONS["init_device"].as< bool >());
+
+    LOGINFO("Metrics dump {}", sisl::MetricsFarm::getInstance().report(sisl::ReportFormat::kTextFormat));
+    // reset cache pct
+    HS_SETTINGS_FACTORY().modifiable_settings([](auto& s) {
+        s.resource_limits.cache_size_percent = 65u;
+        HS_SETTINGS_FACTORY().save();
+    });
+    this->set_app_mem_size(0);
+}
+
 int main(int argc, char* argv[]) {
     int parsed_argc{argc};
     ::testing::InitGoogleTest(&parsed_argc, argv);