Initial commit for HomeBlks 2.0 (#42)

* Initial commit for HomeBlks 2.0

* update con ver

* add remove_volume and lookup_volume api signature

* add listener and init_homestore

* homestore init

* upgrade sisl ver

* incorperate comment

* add HB and vol meta svc register and recovery cb

Author: yamingk

conanfile.py

@@ -9,7 +9,7 @@
 
 class HomeBlocksConan(ConanFile):
     name = "homeblocks"
-    version = "0.0.1"
+    version = "0.0.2"
 
     homepage = "https://github.com/eBay/HomeBlocks"
     description = "Block Store built on HomeStore"
@@ -44,8 +44,9 @@ def build_requirements(self):
         self.test_requires("gtest/1.14.0")
 
     def requirements(self):
-        self.requires("homestore/[~6.4, include_prerelease=True]@oss/master", transitive_headers=True)
-        self.requires("sisl/[~12.2, include_prerelease=True]@oss/master", transitive_headers=True)
+        self.requires("homestore/[^6.5]@oss/master")
+        self.requires("iomgr/[^11.3]@oss/master")
+        self.requires("sisl/[^12.3, include_prerelease=True]@oss/master", transitive_headers=True)
         self.requires("lz4/1.9.4", override=True)
 
     def validate(self):

src/include/homeblocks/common.hpp src/include/homeblks/common.hppsrc/include/homeblocks/common.hpp renamed to src/include/homeblks/common.hpp

@@ -41,6 +41,8 @@ class HomeBlocksApplication {
     virtual bool spdk_mode() const = 0;
     virtual uint32_t threads() const = 0;
     virtual std::list< device_info_t > devices() const = 0;
+    // in bytes;
+    virtual uint64_t app_mem_size() const = 0;
 
     // Callback made after determining if a SvcId exists or not during initialization, will consume response
     virtual peer_id_t discover_svcid(std::optional< peer_id_t > const& found) const = 0;

src/include/homeblocks/homeblocks.hpp src/include/homeblks/home_blks.hppsrc/include/homeblocks/homeblocks.hpp renamed to src/include/homeblks/home_blks.hpp

@@ -17,13 +17,22 @@ struct VolumeInfo {
 
     volume_id_t id;
     uint64_t size_bytes{0};
+    uint64_t page_size{0};
+    std::string vol_name;
 
     auto operator<=>(VolumeInfo const& rhs) const {
         return boost::uuids::hash_value(id) <=> boost::uuids::hash_value(rhs.id);
     }
+
     auto operator==(VolumeInfo const& rhs) const { return id == rhs.id; }
+
+    std::string to_string() {
+        return fmt::format("VolumeInfo: id={} size_bytes={}, page_size={}, name={}", boost::uuids::to_string(id),
+                           size_bytes, page_size, vol_name);
+    }
 };
 
+using VolumeInfoPtr = std::shared_ptr< VolumeInfo >;
 struct VolumeStats {
     volume_id_t id;
 
@@ -40,6 +49,12 @@ class VolumeManager : public Manager< VolumeError > {
 public:
     virtual NullAsyncResult create_volume(VolumeInfo&& volume_info) = 0;
 
+    virtual NullAsyncResult remove_volume(const volume_id_t& id) = 0;
+
+    virtual VolumeInfoPtr lookup_volume(const volume_id_t& id) = 0;
+
+    // TODO: read/write/unmap APIs
+
     /**
      * Retrieves the statistics for a specific Volume identified by its ID.
      *

src/include/homeblocks/volume_manager.hpp src/include/homeblks/volume_mgr.hppsrc/include/homeblocks/volume_manager.hpp renamed to src/include/homeblks/volume_mgr.hpp

@@ -4,13 +4,14 @@ add_flags("-Wno-unused-parameter")
 
 add_library(${PROJECT_NAME}_core OBJECT)
 target_sources(${PROJECT_NAME}_core PRIVATE
-    homeblocks_impl.cpp
-    volume_manager.cpp
+    homeblks_impl.cpp
+    volume_mgr.cpp
+    listener.cpp
 )
 target_link_libraries(${PROJECT_NAME}_core
     ${COMMON_DEPS}
 )
 
 #add_subdirectory(homestore_backend)
-add_subdirectory(memory_backend)
+#add_subdirectory(memory_backend)
 add_subdirectory(tests)

src/lib/CMakeLists.txt

@@ -0,0 +1,253 @@
+#include <algorithm>
+
+#include <iomgr/io_environment.hpp>
+#include <homestore/homestore.hpp>
+#include <homestore/replication_service.hpp>
+#include <sisl/options/options.h>
+#include "homeblks_impl.hpp"
+#include "listener.hpp"
+
+SISL_OPTION_GROUP(homeblocks,
+                  (executor_type, "", "executor", "Executor to use for Future deferal",
+                   ::cxxopts::value< std::string >()->default_value("immediate"), "immediate|cpu|io"));
+
+SISL_LOGGING_DEF(HOMEBLOCKS_LOG_MODS)
+
+namespace homeblocks {
+
+extern std::shared_ptr< HomeBlocks > init_homeobject(std::weak_ptr< HomeBlocksApplication >&& application) {
+    LOGI("Initializing HomeBlocks");
+    auto inst = std::make_shared< HomeBlocksImpl >(std::move(application));
+    inst->init_homestore();
+    inst->init_cp();
+    return inst;
+}
+
+HomeBlocksStats HomeBlocksImpl::get_stats() const {
+    HomeBlocksStats s;
+    return s;
+}
+
+HomeBlocksImpl::HomeBlocksImpl(std::weak_ptr< HomeBlocksApplication >&& application) :
+        _application(std::move(application)) {
+    auto exe_type = SISL_OPTIONS["executor"].as< std::string >();
+    std::transform(exe_type.begin(), exe_type.end(), exe_type.begin(), ::tolower);
+
+    if ("immediate" == exe_type) [[likely]]
+        executor_ = &folly::QueuedImmediateExecutor::instance();
+    else if ("io" == exe_type)
+        executor_ = folly::getGlobalIOExecutor();
+    else if ("cpu" == exe_type)
+        executor_ = folly::getGlobalCPUExecutor();
+    else
+        RELEASE_ASSERT(false, "Unknown Folly Executor type: [{}]", exe_type);
+    LOGI("initialized with [executor={}]", exe_type);
+}
+
+DevType HomeBlocksImpl::get_device_type(std::string const& devname) {
+    const iomgr::drive_type dtype = iomgr::DriveInterface::get_drive_type(devname);
+    if (dtype == iomgr::drive_type::block_hdd || dtype == iomgr::drive_type::file_on_hdd) { return DevType::HDD; }
+    if (dtype == iomgr::drive_type::file_on_nvme || dtype == iomgr::drive_type::block_nvme) { return DevType::NVME; }
+    return DevType::UNSUPPORTED;
+}
+
+// repl application to init homestore
+class HBReplApp : public homestore::ReplApplication {
+public:
+    HBReplApp(homestore::repl_impl_type impl_type, bool tl_consistency, HomeBlocksImpl* hb,
+              std::weak_ptr< HomeBlocksApplication > ho_app) :
+            impl_type_(impl_type), tl_consistency_(tl_consistency), hb_(hb), ho_app_(ho_app) {}
+
+    // TODO: make this override after the base class in homestore adds a virtual destructor
+    virtual ~HBReplApp() = default;
+
+    // overrides
+    homestore::repl_impl_type get_impl_type() const override { return impl_type_; }
+
+    bool need_timeline_consistency() const override { return tl_consistency_; }
+
+    // this will be called by homestore when create_repl_dev is called;
+    std::shared_ptr< homestore::ReplDevListener > create_repl_dev_listener(homestore::group_id_t group_id) override {
+        return std::make_shared< HBListener >(hb_);
+#if 0
+        std::scoped_lock lock_guard(_repl_sm_map_lock);
+        auto [it, inserted] = _repl_sm_map.emplace(group_id, nullptr);
+        if (inserted) { it->second = std::make_shared< ReplicationStateMachine >(hb_); }
+        return it->second;
+#endif
+    }
+
+    void on_repl_devs_init_completed() override { hb_->on_init_complete(); }
+
+    std::pair< std::string, uint16_t > lookup_peer(homestore::replica_id_t uuid) const override {
+        // r1: should never come here;
+        RELEASE_ASSERT(false, "Unexpected to be called.");
+        return std::make_pair("", 0);
+    }
+
+    homestore::replica_id_t get_my_repl_id() const override { return hb_->our_uuid(); }
+
+private:
+    homestore::repl_impl_type impl_type_;
+    bool tl_consistency_; // indicates whether this application needs timeline consistency;
+    HomeBlocksImpl* hb_;
+    std::weak_ptr< HomeBlocksApplication > ho_app_;
+#if 0
+    std::map< homestore::group_id_t, std::shared_ptr< HBListener> > _repl_sm_map; 
+    std::mutex _repl_sm_map_lock;
+#endif
+};
+
+void HomeBlocksImpl::get_dev_info(shared< HomeBlocksApplication > app, std::vector< homestore::dev_info >& dev_info,
+                                  bool& has_data_dev, bool& has_fast_dev) {
+    for (auto const& dev : app->devices()) {
+        auto input_dev_type = dev.type;
+        auto detected_type = get_device_type(dev.path.string());
+        LOGD("Device {} detected as {}", dev.path.string(), detected_type);
+        auto final_type = (dev.type == DevType::AUTO_DETECT) ? detected_type : input_dev_type;
+        if (final_type == DevType::UNSUPPORTED) {
+            LOGW("Device {} is not supported, skipping", dev.path.string());
+            continue;
+        }
+        if (input_dev_type != DevType::AUTO_DETECT && detected_type != final_type) {
+            LOGW("Device {} detected as {}, but input type is {}, using input type", dev.path.string(), detected_type,
+                 input_dev_type);
+        }
+        auto hs_type = (final_type == DevType::HDD) ? homestore::HSDevType::Data : homestore::HSDevType::Fast;
+        if (hs_type == homestore::HSDevType::Data) { has_data_dev = true; }
+        if (hs_type == homestore::HSDevType::Fast) { has_fast_dev = true; }
+        dev_info.emplace_back(std::filesystem::canonical(dev.path).string(), hs_type);
+    }
+}
+
+void HomeBlocksImpl::init_homestore() {
+    auto app = _application.lock();
+    RELEASE_ASSERT(app, "HomeObjectApplication lifetime unexpected!");
+
+    LOGI("Starting iomgr with {} threads, spdk: {}", app->threads(), false);
+    ioenvironment.with_iomgr(iomgr::iomgr_params{.num_threads = app->threads(), .is_spdk = app->spdk_mode()})
+        .with_http_server();
+
+    const uint64_t app_mem_size = app->app_mem_size();
+    LOGI("Initialize and start HomeStore with app_mem_size = {}", app_mem_size);
+
+    std::vector< homestore::dev_info > device_info;
+    bool has_data_dev{false}, has_fast_dev{false};
+    get_dev_info(app, device_info, has_data_dev, has_fast_dev);
+
+    RELEASE_ASSERT(device_info.size() != 0, "No supported devices found!");
+    using namespace homestore;
+    // Note: timeline_consistency doesn't matter as we are using solo repl dev;
+    auto repl_app =
+        std::make_shared< HBReplApp >(repl_impl_type::solo, false /*timeline_consistency*/, this, _application);
+    bool need_format = HomeStore::instance()
+                           ->with_index_service(std::make_unique< HBIndexSvcCB >(this))
+                           .with_repl_data_service(repl_app) // chunk selector defaulted to round_robine
+                           .start(hs_input_params{.devices = device_info, .app_mem_size = app_mem_size},
+                                  [this]() { register_metablk_cb(); });
+    if (need_format) {
+        LOGI("We are starting for the first time. Formatting HomeStore. ");
+        if (has_data_dev && has_fast_dev) {
+            // NOTE: chunk_size, num_chunks only has to specify one, can be deduced from each other.
+            HomeStore::instance()->format_and_start({
+                {HS_SERVICE::META, hs_format_params{.dev_type = HSDevType::Fast, .size_pct = 9.0}},
+                {HS_SERVICE::LOG,
+                 hs_format_params{
+                     .dev_type = HSDevType::Fast, .size_pct = 45.0, .num_chunks = 0, .chunk_size = 32 * Mi}},
+                {HS_SERVICE::INDEX, hs_format_params{.dev_type = HSDevType::Fast, .size_pct = 45.0}},
+                {HS_SERVICE::REPLICATION,
+                 hs_format_params{.dev_type = HSDevType::Data,
+                                  .size_pct = 95.0,
+                                  .num_chunks = 0, // num_chunks will be deduced from chunk_size
+                                  .chunk_size = HS_CHUNK_SIZE,
+                                  .block_size = DATA_BLK_SIZE}},
+            });
+        } else {
+            auto run_on_type = has_fast_dev ? homestore::HSDevType::Fast : homestore::HSDevType::Data;
+            LOGD("Running with Single mode, all service on {}", run_on_type);
+            HomeStore::instance()->format_and_start({
+                {HS_SERVICE::META, hs_format_params{.dev_type = run_on_type, .size_pct = 5.0}},
+                {HS_SERVICE::LOG,
+                 hs_format_params{.dev_type = run_on_type, .size_pct = 10.0, .num_chunks = 0, .chunk_size = 32 * Mi}},
+                {HS_SERVICE::INDEX, hs_format_params{.dev_type = run_on_type, .size_pct = 5.0}},
+                {HS_SERVICE::REPLICATION,
+                 hs_format_params{.dev_type = run_on_type,
+                                  .size_pct = 75.0,
+                                  .num_chunks = 0, // num_chunks will be deduced from chunk_size;
+                                  .chunk_size = HS_CHUNK_SIZE,
+                                  .block_size = DATA_BLK_SIZE}},
+            });
+        }
+        repl_app->on_repl_devs_init_completed();
+        superblk_init();
+    }
+
+    recovery_done_ = true;
+    LOGI("Initialize and start HomeStore is successfully");
+}
+
+void HomeBlocksImpl::superblk_init() {
+    sb_ = homestore::superblk< homeblks_sb_t >(HB_META_NAME);
+    sb_.create(sizeof(homeblks_sb_t));
+    sb_->magic = HB_SB_MAGIC;
+    sb_->version = HB_SB_VER;
+    sb_->boot_cnt = 0;
+    sb_->init_flag(0);
+    sb_.write();
+}
+
+void HomeBlocksImpl::on_vol_meta_blk_found(sisl::byte_view const& buf, void* cookie) {
+    // auto sb = homestore::superblk< vol_sb_t >(VOL_META_NAME);
+    // sb.load(buf, cookie);
+    // TODO:
+}
+
+void HomeBlocksImpl::on_hb_meta_blk_found(sisl::byte_view const& buf, void* cookie) {
+    sb_ = homestore::superblk< homeblks_sb_t >(HB_META_NAME);
+    sb_.load(buf, cookie);
+    // sb verification
+    RELEASE_ASSERT_EQ(sb_->version, HB_SB_VER);
+    RELEASE_ASSERT_EQ(sb_->magic, HB_SB_MAGIC);
+
+    if (sb_->test_flag(SB_FLAGS_GRACEFUL_SHUTDOWN)) {
+        // if it is a gracefuln shutdown, this flag should be set again in shutdown routine;
+        sb_->clear_flag(SB_FLAGS_GRACEFUL_SHUTDOWN);
+        LOGI("System was shutdown gracefully");
+    } else {
+        LOGI("System experienced sudden crash since last boot");
+    }
+
+    ++sb_->boot_cnt;
+
+    // avoid doing sb meta blk write in callback which will cause deadlock;
+    // the 1st CP should flush all dirty SB before taking traffic;
+}
+
+void HomeBlocksImpl::register_metablk_cb() {
+    // register some callbacks for metadata recovery;
+    using namespace homestore;
+
+    // HomeBlks SB
+    HomeStore::instance()->meta_service().register_handler(
+        HB_META_NAME,
+        [this](homestore::meta_blk* mblk, sisl::byte_view buf, size_t size) {
+            on_hb_meta_blk_found(std::move(buf), voidptr_cast(mblk));
+        },
+        nullptr /*recovery_comp_cb*/, true /* do_crc */);
+
+    // Volume SB
+    HomeStore::instance()->meta_service().register_handler(
+        VOL_META_NAME,
+        [this](homestore::meta_blk* mblk, sisl::byte_view buf, size_t size) {
+            on_vol_meta_blk_found(std::move(buf), voidptr_cast(mblk));
+        },
+        nullptr /*recovery_comp_cb*/, true /* do_crc */);
+}
+
+void HomeBlocksImpl::on_init_complete() {
+    // this is called after HomeStore all recovery completed.
+    // Add anything that needs to be done here.
+}
+
+void HomeBlocksImpl::init_cp() {}
+} // namespace homeblocks