Reserve space in a chunk (eBay#279)

Author: yuwmao

conanfile.py

@@ -9,7 +9,7 @@
 
 class HomeObjectConan(ConanFile):
     name = "homeobject"
-    version = "2.3.7"
+    version = "2.3.8"
 
     homepage = "https://github.com/eBay/HomeObject"
     description = "Blob Store built on HomeReplication"
@@ -49,7 +49,7 @@ def build_requirements(self):
 
     def requirements(self):
         self.requires("sisl/[^12.2]@oss/master", transitive_headers=True)
-        self.requires("homestore/[>=6.8.0]@oss/master")
+        self.requires("homestore/[>=6.9.0]@oss/master")
         self.requires("iomgr/[^11.3]@oss/master")
         self.requires("lz4/1.9.4", override=True)
         self.requires("openssl/3.3.1", override=True)

src/lib/homestore_backend/hs_backend_config.fbs

@@ -15,6 +15,9 @@ table HSBackendSettings {
 
     //Snapshot blob load retry count
     snapshot_blob_load_retry: uint8 = 3 (hotswap);
+
+    //Reserved space in a chunk
+    reserved_bytes_in_chunk: uint64 = 16777216 (hotswap);
 }
 
 root_type HSBackendSettings;

src/lib/homestore_backend/hs_blob_manager.cpp

@@ -1,3 +1,4 @@
+#include "hs_backend_config.hpp"
 #include "hs_homeobject.hpp"
 #include "replication_message.hpp"
 #include "replication_state_machine.hpp"
@@ -408,14 +409,18 @@ HSHomeObject::blob_put_get_blk_alloc_hints(sisl::blob const& header, cintrusive<
     homestore::blk_alloc_hints hints;
 
     auto hs_shard = d_cast< HS_Shard* >((*shard_iter->second).get());
-    BLOGD(tid, msg_header->shard_id, msg_header->blob_id, "Picked p_chunk_id=[{}]", hs_shard->sb_->p_chunk_id);
     hints.chunk_id_hint = hs_shard->sb_->p_chunk_id;
+    if (hs_ctx->is_proposer()) { hints.reserved_blks = get_reserved_blks(); }
+    BLOGD(tid, msg_header->shard_id, msg_header->blob_id, "Picked p_chunk_id={}, reserved_blks={}",
+          hs_shard->sb_->p_chunk_id, get_reserved_blks());
 
     if (msg_header->blob_id != 0) {
         // check if the blob already exists, if yes, return the blk id
         auto r = get_blob_from_index_table(hs_pg->index_table_, msg_header->shard_id, msg_header->blob_id);
         if (r.hasValue()) {
-            BLOGT(tid, msg_header->shard_id, msg_header->blob_id, "Blob has already been persisted");
+            BLOGT(tid, msg_header->shard_id, msg_header->blob_id,
+                  "Blob has already been persisted, blk_num={}, blk_count={}", r.value().blk_num(),
+                  r.value().blk_count());
             hints.committed_blk_id = r.value();
         }
     }

src/lib/homestore_backend/hs_homeobject.cpp

@@ -123,6 +123,10 @@ void HSHomeObject::init_homestore() {
     RELEASE_ASSERT(app_mem_size > 0, "Invalid app_mem_size");
     LOGI("Initialize and start HomeStore with app_mem_size = {}", homestore::in_bytes(app_mem_size));
 
+    if (HS_BACKEND_DYNAMIC_CONFIG(reserved_bytes_in_chunk) > 0) {
+        _hs_reserved_blks = sisl::round_up(HS_BACKEND_DYNAMIC_CONFIG(reserved_bytes_in_chunk) / _data_block_size, 1);
+        LOGI("will reserve {} blks in each chunk", _hs_reserved_blks);
+    }
     std::vector< homestore::dev_info > device_info;
     bool has_data_dev = false;
     bool has_fast_dev = false;

src/lib/homestore_backend/hs_homeobject.hpp

@@ -47,6 +47,7 @@ class HSHomeObject : public HomeObjectImpl {
     static constexpr uint64_t HS_CHUNK_SIZE = 2 * Gi;
     static constexpr uint32_t _data_block_size = 1024;
     static uint64_t _hs_chunk_size;
+    uint32_t _hs_reserved_blks = 0;
     ///
 
     /// Overridable Helpers
@@ -792,6 +793,8 @@ class HSHomeObject : public HomeObjectImpl {
 
     bool pg_exists(pg_id_t pg_id) const;
 
+    uint32_t get_reserved_blks() const { return _hs_reserved_blks; }
+
     void on_create_pg_message_rollback(int64_t lsn, sisl::blob const& header, sisl::blob const& key,
                                        cintrusive< homestore::repl_req_ctx >& hs_ctx);
 

src/lib/homestore_backend/hs_pg_manager.cpp

@@ -1,3 +1,5 @@
+#include "hs_backend_config.hpp"
+
 #include <boost/uuid/random_generator.hpp>
 #include <boost/uuid/string_generator.hpp>
 #include <homestore/replication_service.hpp>
@@ -190,6 +192,7 @@ folly::Expected< HSHomeObject::HS_PG*, PGError > HSHomeObject::local_create_pg(s
     auto index_table = create_index_table();
     auto uuid_str = boost::uuids::to_string(index_table->uuid());
 
+    repl_dev->set_custom_rdev_name(fmt::format("rdev{}", pg_info.id));
     auto hs_pg = std::make_unique< HS_PG >(std::move(pg_info), std::move(repl_dev), index_table, chunk_ids);
     auto ret = hs_pg.get();
     {

src/lib/homestore_backend/replication_state_machine.cpp

@@ -149,7 +149,7 @@ void ReplicationStateMachine::on_error(ReplServiceError error, const sisl::blob&
 }
 
 homestore::ReplResult< homestore::blk_alloc_hints >
-ReplicationStateMachine::get_blk_alloc_hints(sisl::blob const& header, uint32_t data_size) {
+ReplicationStateMachine::get_blk_alloc_hints(sisl::blob const& header, uint32_t data_size, cintrusive< homestore::repl_req_ctx >& hs_ctx) {
     const ReplicationMessageHeader* msg_header = r_cast< const ReplicationMessageHeader* >(header.cbytes());
     switch (msg_header->msg_type) {
     case ReplicationMessageType::CREATE_SHARD_MSG: {
@@ -172,6 +172,7 @@ ReplicationStateMachine::get_blk_alloc_hints(sisl::blob const& header, uint32_t
         static_assert(std::is_same< homestore::chunk_num_t, uint16_t >::value, "chunk_num_t is not uint16_t");
         homestore::chunk_num_t v_chunk_id = v_chunkID.value();
         hints.application_hint = ((uint64_t)pg_id << 16) | v_chunk_id;
+        if (hs_ctx->is_proposer()) { hints.reserved_blks = home_object_->get_reserved_blks(); }
         return hints;
     }
 
@@ -187,7 +188,7 @@ ReplicationStateMachine::get_blk_alloc_hints(sisl::blob const& header, uint32_t
     }
 
     case ReplicationMessageType::PUT_BLOB_MSG:
-        return home_object_->blob_put_get_blk_alloc_hints(header, nullptr);
+        return home_object_->blob_put_get_blk_alloc_hints(header, hs_ctx);
 
     case ReplicationMessageType::DEL_BLOB_MSG:
     default:

src/lib/homestore_backend/replication_state_machine.hpp

@@ -166,7 +166,7 @@ class ReplicationStateMachine : public homestore::ReplDevListener {
     /// @param header Header originally passed with repl_dev::write() api on the leader
     /// @return Expected to return blk_alloc_hints for this write
     homestore::ReplResult< homestore::blk_alloc_hints > get_blk_alloc_hints(sisl::blob const& header,
-                                                                            uint32_t data_size) override;
+                                                                            uint32_t data_size, cintrusive< homestore::repl_req_ctx >& hs_ctx) override;
 
     /// @brief Called when replication module is replacing an existing member with a new member
     void on_replace_member(const homestore::replica_member_info& member_out,