issue: 60 solo repl dev recovery (#65)

Author: yamingk

conanfile.py

@@ -9,7 +9,7 @@
 
 class HomeBlocksConan(ConanFile):
     name = "homeblocks"
-    version = "0.0.5"
+    version = "0.0.6"
 
     homepage = "https://github.com/eBay/HomeBlocks"
     description = "Block Store built on HomeStore"

src/lib/homeblks_impl.cpp

@@ -44,7 +44,7 @@ HomeBlocksStats HomeBlocksImpl::get_stats() const {
 }
 
 HomeBlocksImpl::~HomeBlocksImpl() {
-    homestore::HomeStore::instance()->shutdown();
+    homestore::hs()->shutdown();
     homestore::HomeStore::reset_instance();
     iomanager.stop();
 }
@@ -161,7 +161,7 @@ void HomeBlocksImpl::init_homestore() {
     // Note: timeline_consistency doesn't matter as we are using solo repl dev;
     auto repl_app =
         std::make_shared< HBReplApp >(repl_impl_type::solo, false /*timeline_consistency*/, this, _application);
-    bool need_format = HomeStore::instance()
+    bool need_format = homestore::hs()
                            ->with_index_service(std::make_unique< HBIndexSvcCB >(this))
                            .with_repl_data_service(repl_app) // chunk selector defaulted to round_robine
                            .start(hs_input_params{.devices = device_info, .app_mem_size = app_mem_size},
@@ -170,7 +170,7 @@ void HomeBlocksImpl::init_homestore() {
         LOGI("We are starting for the first time. Formatting HomeStore. ");
         if (has_data_dev && has_fast_dev) {
             // NOTE: chunk_size, num_chunks only has to specify one, can be deduced from each other.
-            HomeStore::instance()->format_and_start({
+            homestore::hs()->format_and_start({
                 {HS_SERVICE::META, hs_format_params{.dev_type = HSDevType::Fast, .size_pct = 9.0}},
                 {HS_SERVICE::LOG,
                  hs_format_params{
@@ -186,7 +186,7 @@ void HomeBlocksImpl::init_homestore() {
         } else {
             auto run_on_type = has_fast_dev ? homestore::HSDevType::Fast : homestore::HSDevType::Data;
             LOGD("Running with Single mode, all service on {}", run_on_type);
-            HomeStore::instance()->format_and_start({
+            homestore::hs()->format_and_start({
                 {HS_SERVICE::META, hs_format_params{.dev_type = run_on_type, .size_pct = 5.0}},
                 {HS_SERVICE::LOG,
                  hs_format_params{.dev_type = run_on_type, .size_pct = 10.0, .num_chunks = 0, .chunk_size = 32 * Mi}},
@@ -199,7 +199,7 @@ void HomeBlocksImpl::init_homestore() {
                                   .block_size = DATA_BLK_SIZE}},
             });
         }
-        repl_app->on_repl_devs_init_completed();
+        // repl_app->on_repl_devs_init_completed();
         superblk_init();
     }
 
@@ -241,25 +241,29 @@ void HomeBlocksImpl::register_metablk_cb() {
     using namespace homestore;
 
     // HomeBlks SB
-    HomeStore::instance()->meta_service().register_handler(
+    homestore::hs()->meta_service().register_handler(
         HB_META_NAME,
         [this](homestore::meta_blk* mblk, sisl::byte_view buf, size_t size) {
             on_hb_meta_blk_found(std::move(buf), voidptr_cast(mblk));
         },
         nullptr /*recovery_comp_cb*/, true /* do_crc */);
+}
+
+void HomeBlocksImpl::on_init_complete() {
+    // this is called after HomeStore all recovery completed.
+    // Add anything that needs to be done here.
+    using namespace homestore;
 
     // Volume SB
-    HomeStore::instance()->meta_service().register_handler(
+    homestore::hs()->meta_service().register_handler(
         Volume::VOL_META_NAME,
         [this](homestore::meta_blk* mblk, sisl::byte_view buf, size_t size) {
             on_vol_meta_blk_found(std::move(buf), voidptr_cast(mblk));
         },
-        nullptr /*recovery_comp_cb*/, true /* do_crc */);
-}
+        nullptr /*recovery_comp_cb*/, true /* do_crc */,
+        std::optional< meta_subtype_vec_t >({homestore::hs()->repl_service().get_meta_blk_name()}));
 
-void HomeBlocksImpl::on_init_complete() {
-    // this is called after HomeStore all recovery completed.
-    // Add anything that needs to be done here.
+    homestore::hs()->meta_service().read_sub_sb(Volume::VOL_META_NAME);
 }
 
 void HomeBlocksImpl::init_cp() {}

src/lib/homeblks_impl.hpp

@@ -61,6 +61,9 @@ class HomeBlocksImpl : public HomeBlocks, public VolumeManager, public std::enab
     mutable std::shared_mutex vol_lock_;
     std::map< volume_id_t, VolumePtr > vol_map_;
 
+    mutable std::shared_mutex index_lock_;
+    std::unordered_map< std::string, shared< VolumeIndexTable > > idx_tbl_map_;
+
     bool recovery_done_{false};
     superblk< homeblks_sb_t > sb_;
 

src/lib/volume/tests/test_volume.cpp

@@ -68,7 +68,7 @@ TEST_F(VolumeTest, CreateVolumeThenRecover) {
             ASSERT_TRUE(vinfo_ptr != nullptr);
         }
     }
-#if 0
+
     g_helper->restart(5);
 
     // verify the volumes are still there
@@ -82,7 +82,6 @@ TEST_F(VolumeTest, CreateVolumeThenRecover) {
             ASSERT_TRUE(vinfo_ptr != nullptr);
         }
     }
-#endif
 }
 
 int main(int argc, char* argv[]) {

src/lib/volume/volume.cpp

@@ -19,13 +19,12 @@
 namespace homeblocks {
 
 // this API will be called by volume manager after volume sb is recovered and volume is created;
-shared< VolumeIndexTable > Volume::init_index_table(bool is_recovery,
-                                                    homestore::superblk< homestore::index_table_sb >&& sb) {
-    index_cfg_t cfg(homestore::hs()->index_service().node_size());
-    cfg.m_leaf_node_type = homestore::btree_node_type::PREFIX;
-    cfg.m_int_node_type = homestore::btree_node_type::PREFIX;
-
+shared< VolumeIndexTable > Volume::init_index_table(bool is_recovery, shared< VolumeIndexTable > tbl) {
     if (!is_recovery) {
+        index_cfg_t cfg(homestore::hs()->index_service().node_size());
+        cfg.m_leaf_node_type = homestore::btree_node_type::PREFIX;
+        cfg.m_int_node_type = homestore::btree_node_type::PREFIX;
+
         // create index table;
         auto uuid = hb_utils::gen_random_uuid();
 
@@ -34,15 +33,11 @@ shared< VolumeIndexTable > Volume::init_index_table(bool is_recovery,
         LOGI("Creating index table for volume: {}, index_uuid: {}, parent_uuid: {}", vol_info_->name,
              boost::uuids::to_string(uuid), boost::uuids::to_string(id()));
         indx_tbl_ = std::make_shared< VolumeIndexTable >(uuid, id() /*parent uuid*/, 0 /*user_sb_size*/, cfg);
-        homestore::hs()->index_service().add_index_table(indx_table());
     } else {
-        // recovery path
-        LOGI("Recovering index table for  index_uuid: {}, parent_uuid: {}", boost::uuids::to_string(sb->uuid),
-             boost::uuids::to_string(sb->parent_uuid));
-        indx_tbl_ = std::make_shared< VolumeIndexTable >(std::move(sb), cfg);
-        // HomeStore will add the index table to the index service in recovery case, its done in its constructor;
+        indx_tbl_ = tbl;
     }
 
+    homestore::hs()->index_service().add_index_table(indx_table());
     return indx_table();
 }
 
@@ -60,6 +55,9 @@ bool Volume::init(bool is_recovery) {
         sb_->init(vol_info_->page_size, vol_info_->size_bytes, vol_info_->id, vol_info_->name);
         // write to disk;
         sb_.write();
+
+        init_index_table(false /*is_recovery*/);
+
         // create solo repl dev for volume;
         // members left empty on purpose for solo repl dev
         LOGI("Creating solo repl dev for volume: {}, uuid: {}", vol_info_->name, boost::uuids::to_string(id()));

src/lib/volume/volume.hpp

@@ -30,8 +30,8 @@ using VolumePtr = shared< Volume >;
 using VolIdxTablePtr = shared< VolumeIndexTable >;
 
 using ReplDevPtr = shared< homestore::ReplDev >;
+using index_cfg_t = homestore::BtreeConfig;
 class Volume {
-    using index_cfg_t = homestore::BtreeConfig;
 
 public:
     inline static auto const VOL_META_NAME = std::string("Volume2"); // different from old releae;
@@ -90,16 +90,15 @@ class Volume {
 
     VolIdxTablePtr indx_table() const { return indx_tbl_; }
     volume_id_t id() const { return vol_info_->id; };
+    std::string id_str() const { return boost::uuids::to_string(vol_info_->id); };
     ReplDevPtr rd() const { return rd_; }
 
     VolumeInfoPtr info() const { return vol_info_; }
 
     //
-    // This API is called for both volume creation and volume recovery;
     // Initialize index table for this volume and saves the index handle in the volume object;
     //
-    shared< VolumeIndexTable > init_index_table(bool is_recovery,
-                                                homestore::superblk< homestore::index_table_sb >&& sb = {});
+    shared< VolumeIndexTable > init_index_table(bool is_recovery, shared< VolumeIndexTable > tbl = nullptr);
 
 private:
     //

src/lib/volume_mgr.cpp

@@ -22,11 +22,17 @@ namespace homeblocks {
 std::shared_ptr< VolumeManager > HomeBlocksImpl::volume_manager() { return shared_from_this(); }
 
 void HomeBlocksImpl::on_vol_meta_blk_found(sisl::byte_view const& buf, void* cookie) {
-    // auto sb = homestore::superblk< vol_sb_t >(VOL_META_NAME);
-    // sb.load(buf, cookie);
     auto vol_ptr = Volume::make_volume(buf, cookie);
-    vol_ptr->init_index_table(false);
     auto id = vol_ptr->id();
+
+    {
+        auto lg = std::scoped_lock(index_lock_);
+        auto it = idx_tbl_map_.find(vol_ptr->id_str());
+        DEBUG_ASSERT(it != idx_tbl_map_.end(), "index pid: {} not exists in recovery path, not expected!",
+                     boost::uuids::to_string(vol_ptr->id()));
+        vol_ptr->init_index_table(true /*is_recovery*/, it->second /* table */);
+    }
+
     {
         auto lg = std::scoped_lock(vol_lock_);
         DEBUG_ASSERT(vol_map_.find(id) == vol_map_.end(),
@@ -36,14 +42,17 @@ void HomeBlocksImpl::on_vol_meta_blk_found(sisl::byte_view const& buf, void* coo
 }
 
 shared< VolumeIndexTable > HomeBlocksImpl::recover_index_table(homestore::superblk< homestore::index_table_sb >&& sb) {
-    auto id = sb->parent_uuid; // parent uuid is the volume id;
+    auto pid_str = boost::uuids::to_string(sb->parent_uuid); // parent_uuid is the volume id
     {
-        auto lg = std::scoped_lock(vol_lock_);
-        auto it = vol_map_.find(id);
-        DEBUG_ASSERT(it != vol_map_.end(), "volume id: {} not exists in recovery path, not expected!",
-                     boost::uuids::to_string(id));
+        auto lg = std::scoped_lock(index_lock_);
+        index_cfg_t cfg(homestore::hs()->index_service().node_size());
+        cfg.m_leaf_node_type = homestore::btree_node_type::PREFIX;
+        cfg.m_int_node_type = homestore::btree_node_type::PREFIX;
 
-        return it->second->init_index_table(true, std::move(sb));
+        LOGI("Recovering index table for  index_uuid: {}, parent_uuid: {}", boost::uuids::to_string(sb->uuid), pid_str);
+        auto tbl = std::make_shared< VolumeIndexTable >(std::move(sb), cfg);
+        idx_tbl_map_.emplace(pid_str, tbl);
+        return tbl;
     }
 }