[TEST] for repl dev ut

Author: yawzhang

src/lib/replication/repl_dev/raft_repl_dev.cpp

@@ -1414,13 +1414,6 @@ void RaftReplDev::handle_commit(repl_req_ptr_t rreq, bool recovery) {
         m_listener->on_commit(rreq->lsn(), rreq->header(), rreq->key(), {rreq->local_blkid()}, rreq);
     }
 
-    if (!recovery) {
-        auto prev_lsn = m_commit_upto_lsn.exchange(rreq->lsn());
-        RD_DBG_ASSERT_GT(rreq->lsn(), prev_lsn,
-                         "Out of order commit of lsns, it is not expected in RaftReplDev. cur_lsns={}, prev_lsns={}",
-                         rreq->lsn(), prev_lsn);
-    }
-
     // Remove the request from repl_key map only after the listener operation is completed.
     // This prevents unnecessary block allocation in the following scenario:
     // 1. The follower processes a commit for LSN 100 and remove rreq from rep_key map before listener commit
@@ -1431,6 +1424,13 @@ void RaftReplDev::handle_commit(repl_req_ptr_t rreq, bool recovery) {
     m_repl_key_req_map.erase(rreq->rkey());
     // Remove the request from lsn map.
     m_state_machine->unlink_lsn_to_req(rreq->lsn(), rreq);
+
+    if (!recovery) {
+        auto prev_lsn = m_commit_upto_lsn.exchange(rreq->lsn());
+        RD_DBG_ASSERT_GT(rreq->lsn(), prev_lsn,
+                         "Out of order commit of lsns, it is not expected in RaftReplDev. cur_lsns={}, prev_lsns={}",
+                         rreq->lsn(), prev_lsn);
+    }
     if (!rreq->is_proposer()) rreq->clear();
 }
 

src/tests/test_common/raft_repl_test_base.hpp

@@ -614,7 +614,19 @@ class RaftReplDevTestBase : public testing::Test {
                         data_size == nullptr ? std::abs(std::lround(num_blks_gen(g_re))) * block_size : *data_size;
                     this->generate_writes(size, block_size, db);
                 });
-                if (wait_for_commit) { g_helper->runner().execute().get(); }
+                if (wait_for_commit) {
+                    g_helper->runner().execute().get();
+                    // wait for last_commit_lsn not changed, so all related rreqs have been removed from map.
+                    // this way to avoid rreqs reused in this case:
+                    // 1. follower committing rreq
+                    // 2. follower received a duplicated append log entries from leader, then get the rreq from map
+                    // 3. follower finished commit, clear rreq, then the append thread hold an empty rreq.
+                    repl_lsn_t current_commit_lsn = 0;
+                    while (current_commit_lsn < repl_dev->get_last_commit_lsn()) {
+                        current_commit_lsn = repl_dev->get_last_commit_lsn();
+                        std::this_thread::sleep_for(std::chrono::seconds{5});
+                    }
+                }
                 break;
             } else {
                 LOGINFO("{} entries were written on the leader_uuid={} my_uuid={}", num_entries,