Fix UT: address UAF in TestRaftLogStore lifecycle_test under ASan (eBay#860)

Co-authored-by: yawzhang <yawzhang@ebay.com>

Author: Besroy

conanfile.py

@@ -9,7 +9,7 @@
 
 class HomestoreConan(ConanFile):
     name = "homestore"
-    version = "7.4.2"
+    version = "7.4.3"
 
     homepage = "https://github.com/eBay/Homestore"
     description = "HomeStore Storage Engine"

src/lib/logstore/log_dev.cpp

@@ -158,6 +158,16 @@ void LogDev::stop() {
     // trigger a new flush to make sure all pending writes are flushed
     flush_under_guard();
 
+    {
+        std::unique_lock lg = flush_guard();
+        // wait again for any callbacks issued by new flush
+        while (m_pending_callback.load() > 0) {
+            THIS_LOGDEV_LOG(INFO, "Waiting for new pending callbacks to complete, pending callbacks {}",
+                            m_pending_callback.load());
+            std::this_thread::sleep_for(std::chrono::milliseconds{1000});
+        }
+    }
+
     // after we call stop, we need to do any pending device truncations
     truncate();
     m_id_logstore_map.clear();
@@ -583,10 +593,12 @@ void LogDev::on_flush_completion(LogGroup* lg) {
 
     // since we support out-of-order lsn write, so no need to guarantee the order of logstore write completion
     for (auto const& [idx, req] : req_map) {
+        m_pending_callback++;
         auto callback_lambda = [this, dev_offset, idx, req]() {
             auto ld_key = logdev_key{idx, dev_offset};
             auto comp_cb = req->log_store->get_comp_cb();
             (req->cb) ? req->cb(req, ld_key) : comp_cb(req, ld_key);
+            m_pending_callback--;
         };
 
         // if we do not have repl_service, we run the callback in a random worker for the case of log store UT, where