update

Author: JacksonYao287

src/lib/homestore_backend/CMakeLists.txt

@@ -169,7 +169,8 @@ add_test(NAME HomestoreTestGC COMMAND homestore_test_gc -csv error --executor im
 add_executable(homestore_test_scrubber)
 target_sources(homestore_test_scrubber PRIVATE $<TARGET_OBJECTS:homestore_tests_scrubber>)
 target_link_libraries(homestore_test_scrubber PUBLIC homeobject_homestore ${COMMON_TEST_DEPS})
-add_test(NAME HomestoreTestScrubber COMMAND homestore_test_scrubber -csv error --executor immediate --config_path ./  
+add_test(NAME HomestoreTestScrubber COMMAND homestore_test_scrubber -csv error --executor immediate --config_path ./
+        --override_config hs_backend_config.enable_scrubber=true
         --override_config nuraft_mesg_config.mesg_factory_config.data_request_deadline_secs:10)
 
 

src/lib/homestore_backend/hs_homeobject.cpp

@@ -259,16 +259,19 @@ void HSHomeObject::init_homestore() {
     } else {
         LOGI("GC is disabled");
     }
-}
 
-void HSHomeObject::on_replica_restart() {
-    std::call_once(replica_restart_flag_, [this]() {
-        // scrubber manager will be created and started only once here. we need make sure scrub manager is created and
-        // started before pg is recovered, so that we a replica becomes leader of a pg, the scrub task can be resumed in
-        // on_become_leader.
+    // start scrubber
+    if (HS_BACKEND_DYNAMIC_CONFIG(enable_scrubber)) {
+        LOGI("Starting scrub manager");
         scrub_mgr_ = std::make_shared< ScrubManager >(this);
         scrub_mgr_->start();
+    } else {
+        LOGI("scrub manager is disabled");
+    }
+}
 
+void HSHomeObject::on_replica_restart() {
+    std::call_once(replica_restart_flag_, [this]() {
         LOGI("Register PG, shard and gc related meta blk handlers");
         using namespace homestore;
         // recover PG
@@ -455,10 +458,10 @@ void HSHomeObject::shutdown() {
     // we need stop gc before shutting down homestore(where metaservice is shutdown), because gc mgr needs metaservice
     // to persist gc task metablk if there is any ongoing gc task. after stopping gc manager, there is no gc task
     // anymore, and thus now new gc task will be written to metaservice during homestore shutdown.
-    gc_mgr_->stop();
+    if (gc_mgr_) gc_mgr_->stop();
 
     LOGI("stopping scrubbing");
-    scrub_mgr_->stop();
+    if (scrub_mgr_) scrub_mgr_->stop();
 
     LOGI("start shutting down HomeStore");
     homestore::HomeStore::instance()->shutdown();

src/lib/homestore_backend/hs_homeobject_fbs/hs_backend_config.fbs

@@ -23,6 +23,10 @@ table HSBackendSettings {
     //TODO: make this hotswap after gc is well tested
     enable_gc: bool = true;
 
+    //Enable scrubber
+    //TODO: make this hotswap after scrubber is well tested
+    enable_scrubber: bool = false;
+
     //Total reserved chunk num (dedicated for gc/egc) per pdev 
     reserved_chunk_num_per_pdev: uint8 = 6;
 

src/lib/homestore_backend/hs_pg_manager.cpp

@@ -1217,7 +1217,10 @@ void HSHomeObject::HS_PG::on_scrub_req_received(boost::intrusive_ptr< sisl::Gene
     }
 
     auto scrub_mgr = home_obj_.scrub_manager();
-    RELEASE_ASSERT(scrub_mgr, "ScrubManager is null in HS_PG::on_scrub_blob_req_received for pg={}", pg_id);
+    if (!scrub_mgr) {
+        LOGW("ScrubManager is not initialized in HS_PG::on_scrub_req_received for pg={}", pg_id);
+        return;
+    }
     scrub_mgr->add_scrub_req(scrub_req);
 }
 
@@ -1321,7 +1324,10 @@ void HSHomeObject::HS_PG::on_scrub_map_received(boost::intrusive_ptr< sisl::Gene
     }
 
     auto scrub_mgr = home_obj_.scrub_manager();
-    RELEASE_ASSERT(scrub_mgr, "ScrubManager is null in HS_PG::on_scrub_map_received for pg={}", pg_id);
+    if (!scrub_mgr) {
+        LOGW("ScrubManager is not initialized in HS_PG::on_scrub_map_received for pg={}", pg_id);
+        return;
+    }
     scrub_mgr->add_scrub_map(pg_id, scrub_map);
 }
 

src/lib/homestore_backend/replication_state_machine.cpp

@@ -291,10 +291,6 @@ void ReplicationStateMachine::on_destroy(const homestore::group_id_t& group_id)
     const auto pg_id = PG_ID.value();
     home_object_->pg_destroy(pg_id);
     LOGI("replica destroyed, cleared pg={} resources with group_id={}", pg_id, boost::uuids::to_string(group_id));
-
-    // we want the scrub_seq_num to keep the same when BR happens. so we only remove it from scrub manager when the
-    // group is destroyed at this replica, so that BR will not delete/change the pg scrub superblk.
-
     // there is a case that after pg is destroyed above and crash happends before scrub_mgr#remove_pg is called, there
     // will be a stale pg_scrub_superblk. we will handle this in metablk replay.
     home_object_->scrub_manager()->remove_pg(pg_id);
@@ -1064,6 +1060,7 @@ void ReplicationStateMachine::on_become_leader(const homestore::group_id_t& grou
     }
     const auto pg_id = pg_id_opt.value();
     RELEASE_ASSERT(home_object_->pg_exists(pg_id), "pg={} should exist, but not! fatal error!", pg_id);
+    // TODO:: add whatever acitons needed to be take.
 }
 
 void ReplicationStateMachine::on_become_follower(const homestore::group_id_t& group_id) {
@@ -1076,7 +1073,7 @@ void ReplicationStateMachine::on_become_follower(const homestore::group_id_t& gr
     RELEASE_ASSERT(home_object_->pg_exists(pg_id), "pg={} should exist, but not! fatal error!", pg_id);
 
     LOGI("become follower of group {}, cancel scrub task for pg={}", group_id, pg_id);
-    // add whatever acitons needed to be take.
+    // TODO:: add whatever acitons needed to be take.
 
     // cancel scrub task if I am not leader again.
     home_object_->scrub_manager()->cancel_scrub_task(pg_id);

src/lib/homestore_backend/scrub_manager.cpp

@@ -271,6 +271,10 @@ void ScrubManager::start() {
 
 void ScrubManager::stop() {
     m_scrub_task_queue.close();
+    for (auto& [_, pg_scrub_ctx] : m_pg_scrub_ctx_map) {
+        pg_scrub_ctx->cancel();
+    }
+
     if (m_scrub_timer_hdl == iomgr::null_timer_handle) {
         LOGINFOMOD(scrubmgr, "scrub scheduler timer is not running, no need to stop it");
         return;