diff --git a/conanfile.py b/conanfile.py index dc340bb5..2741212b 100644 --- a/conanfile.py +++ b/conanfile.py @@ -10,7 +10,7 @@ class HomeObjectConan(ConanFile): name = "homeobject" - version = "4.0.8" + version = "4.0.9" homepage = "https://github.com/eBay/HomeObject" description = "Blob Store built on HomeStore" diff --git a/src/lib/homestore_backend/hs_homeobject.cpp b/src/lib/homestore_backend/hs_homeobject.cpp index 48075dc0..0413c166 100644 --- a/src/lib/homestore_backend/hs_homeobject.cpp +++ b/src/lib/homestore_backend/hs_homeobject.cpp @@ -550,4 +550,15 @@ void HSHomeObject::yield_pg_leadership_to_follower(int32_t pg_id) { } } +void HSHomeObject::trigger_snapshot_creation(int32_t pg_id, int64_t compact_lsn, bool wait_for_commit) { + LOGI("Triggering snapshot creation for pg_id={}, compact_lsn={}, wait_for_commit={}", pg_id, compact_lsn, + wait_for_commit); + auto hs_pg = get_hs_pg(pg_id); + if (!hs_pg) { + LOGE("Failed to trigger snapshot: PG {} not found", pg_id); + return; + } + hs_pg->trigger_snapshot_creation(compact_lsn, wait_for_commit); +} + } // namespace homeobject diff --git a/src/lib/homestore_backend/hs_homeobject.hpp b/src/lib/homestore_backend/hs_homeobject.hpp index d529e7b5..50304a03 100644 --- a/src/lib/homestore_backend/hs_homeobject.hpp +++ b/src/lib/homestore_backend/hs_homeobject.hpp @@ -54,7 +54,8 @@ class HSHomeObject : public HomeObjectImpl { uint32_t _hs_reserved_blks = 0; /// Overridable Helpers - ShardManager::AsyncResult< ShardInfo > _create_shard(pg_id_t, uint64_t size_bytes, std::string meta, trace_id_t tid) override; + ShardManager::AsyncResult< ShardInfo > _create_shard(pg_id_t, uint64_t size_bytes, std::string meta, + trace_id_t tid) override; ShardManager::AsyncResult< ShardInfo > _seal_shard(ShardInfo const&, trace_id_t tid) override; BlobManager::AsyncResult< blob_id_t > _put_blob(ShardInfo const&, Blob&&, trace_id_t tid) override; @@ -196,7 +197,7 @@ class HSHomeObject : public HomeObjectImpl { }; struct shard_info_superblk : DataHeader { - //This version is a common version of DataHeader, each derived struct can have its own version. + // This version is a common version of DataHeader, each derived struct can have its own version. static constexpr uint8_t shard_sb_version = 0x02; uint8_t sb_version{shard_sb_version}; @@ -394,6 +395,8 @@ class HSHomeObject : public HomeObjectImpl { void yield_leadership_to_follower() const; + void trigger_snapshot_creation(int64_t compact_lsn, bool wait_for_commit) const; + /** * Returns all shards */ @@ -437,7 +440,7 @@ class HSHomeObject : public HomeObjectImpl { uint32_t data_offset; // Offset of actual data blob stored after the metadata uint32_t user_key_size; // Actual size of the user key. uint8_t user_key[max_user_key_length + 1]{}; - uint8_t padding[2956]{}; // data_block_size is 4K, so total size of BlobHeader is 4096 bytes + uint8_t padding[2956]{}; // data_block_size is 4K, so total size of BlobHeader is 4096 bytes std::optional< std::string > get_user_key() const { if (user_key_size > max_user_key_length) { return std::nullopt; } @@ -969,6 +972,22 @@ class HSHomeObject : public HomeObjectImpl { */ void yield_pg_leadership_to_follower(int32_t pg_id = 1); + /** + * @brief Manually trigger a snapshot creation. + * @param compact_lsn Expected compact up to LSN. Default is -1, meaning it depends directly on the current HS + * status. + * @param wait_for_commit Wait committed lsn reaches compact_lsn. Default is true, false means the snapshot will be + * triggered based on its latest committed lsn and the log compaction depends on min(snapshot_lsn, compact_lsn) + * + * Recommendation: + * - Please keep wait_for_commit=true to make sure the compact_lsn <= committed_lsn when snapshot is created. If you + * just want to trigger a snapshot manually or want to update the compact_lsn (already less than committed_lsn) for + * log compaction, you can set wait_for_commit=false. + * - wait_for_commit=true might cause the caller blocked for a while until the committed_lsn reaches compact_lsn. + * Please keep in mind and take action if this function costs too long. + */ + void trigger_snapshot_creation(int32_t pg_id, int64_t compact_lsn, bool wait_for_commit); + // Blob manager related. void on_blob_message_rollback(int64_t lsn, sisl::blob const& header, sisl::blob const& key, cintrusive< homestore::repl_req_ctx >& hs_ctx); diff --git a/src/lib/homestore_backend/hs_http_manager.cpp b/src/lib/homestore_backend/hs_http_manager.cpp index 75b15b83..71b007da 100644 --- a/src/lib/homestore_backend/hs_http_manager.cpp +++ b/src/lib/homestore_backend/hs_http_manager.cpp @@ -37,6 +37,8 @@ HttpManager::HttpManager(HSHomeObject& ho) : ho_(ho) { Pistache::Rest::Routes::bind(&HttpManager::reconcile_leader, this)}, {Pistache::Http::Method::Post, "/api/v1/yield_leadership_to_follower", Pistache::Rest::Routes::bind(&HttpManager::yield_leadership_to_follower, this)}, + {Pistache::Http::Method::Post, "/api/v1/trigger_snapshot_creation", + Pistache::Rest::Routes::bind(&HttpManager::trigger_snapshot_creation, this)}, #ifdef _PRERELEASE {Pistache::Http::Method::Post, "/api/v1/crashSystem", Pistache::Rest::Routes::bind(&HttpManager::crash_system, this)}, @@ -91,6 +93,36 @@ void HttpManager::yield_leadership_to_follower(const Pistache::Rest::Request& re response.send(Pistache::Http::Code::Ok, "Yield leadership request submitted"); } +void HttpManager::trigger_snapshot_creation(const Pistache::Rest::Request& request, + Pistache::Http::ResponseWriter response) { + // Extract and validate pg_id parameter (required) + const auto pg_id_param = request.query().get("pg_id"); + if (!pg_id_param) { + response.send(Pistache::Http::Code::Bad_Request, "pg_id is required"); + return; + } + const int32_t pg_id = std::stoi(pg_id_param.value()); + + // Extract compact_lsn parameter (optional, default: -1 means use current HS status) + const auto compact_lsn_param = request.query().get("compact_lsn"); + const int64_t compact_lsn = std::stoll(compact_lsn_param.value_or("-1")); + + // Extract wait_for_commit parameter (optional, default: true) + const auto wait_for_commit_param = request.query().get("wait_for_commit"); + std::string wait_for_commit_mode = wait_for_commit_param.value_or("true"); + if (wait_for_commit_mode != "true" && wait_for_commit_mode != "false") { + response.send(Pistache::Http::Code::Bad_Request, "wait_for_commit must be 'true' or 'false'"); + return; + } + bool wait_for_commit = (wait_for_commit_mode == "true"); + + LOGINFO("Received snapshot creation request for pg_id={}, compact_lsn={}, wait_for_commit={}", pg_id, compact_lsn, + wait_for_commit); + + ho_.trigger_snapshot_creation(pg_id, compact_lsn, wait_for_commit); + response.send(Pistache::Http::Code::Ok, "Snapshot creation request submitted"); +} + void HttpManager::get_pg(const Pistache::Rest::Request& request, Pistache::Http::ResponseWriter response) { auto pg_str = request.query().get("pg_id"); if (!pg_str) { diff --git a/src/lib/homestore_backend/hs_http_manager.hpp b/src/lib/homestore_backend/hs_http_manager.hpp index 62d45f1a..7221850c 100644 --- a/src/lib/homestore_backend/hs_http_manager.hpp +++ b/src/lib/homestore_backend/hs_http_manager.hpp @@ -28,6 +28,7 @@ class HttpManager { void get_malloc_stats(const Pistache::Rest::Request& request, Pistache::Http::ResponseWriter response); void reconcile_leader(const Pistache::Rest::Request& request, Pistache::Http::ResponseWriter response); void yield_leadership_to_follower(const Pistache::Rest::Request& request, Pistache::Http::ResponseWriter response); + void trigger_snapshot_creation(const Pistache::Rest::Request& request, Pistache::Http::ResponseWriter response); void get_pg(const Pistache::Rest::Request& request, Pistache::Http::ResponseWriter response); void get_pg_chunks(const Pistache::Rest::Request& request, Pistache::Http::ResponseWriter response); void dump_chunk(const Pistache::Rest::Request& request, Pistache::Http::ResponseWriter response); diff --git a/src/lib/homestore_backend/hs_pg_manager.cpp b/src/lib/homestore_backend/hs_pg_manager.cpp index 340bf321..61739853 100644 --- a/src/lib/homestore_backend/hs_pg_manager.cpp +++ b/src/lib/homestore_backend/hs_pg_manager.cpp @@ -797,6 +797,10 @@ void HSHomeObject::HS_PG::yield_leadership_to_follower() const { repl_dev_->yield_leadership(false /*immediate_yield*/, candidate_leader_id); } +void HSHomeObject::HS_PG::trigger_snapshot_creation(int64_t compact_lsn, bool wait_for_commit) const { + repl_dev_->trigger_snapshot_creation(compact_lsn, wait_for_commit); +} + std::vector< Shard > HSHomeObject::HS_PG::get_chunk_shards(chunk_num_t v_chunk_id) const { std::vector< Shard > ret; for (auto const& s : shards_) {