add yield leader api for test

Author: yawzhang

conanfile.py

@@ -9,7 +9,7 @@
 
 class HomestoreConan(ConanFile):
     name = "homestore"
-    version = "6.20.22"
+    version = "6.20.23"
 
     homepage = "https://github.com/eBay/Homestore"
     description = "HomeStore Storage Engine"

src/include/homestore/replication/repl_dev.h

@@ -531,6 +531,10 @@ class ReplDev {
     /// @brief request leadership if it should be the leader but not, yield leadership if it is leader but should not be
     virtual void reconcile_leader() = 0;
 
+    /// @brief Yield current leadership and becomes a follower.
+    /// this API call nuraft yield_leadership directly
+    virtual void yield_leadership(bool immediate_yield, replica_id_t candidate) = 0;
+
     /// @brief Sets a custom name for the repldev. Users can assign a meaningful name to the repldev for easy debugging.
     virtual void set_custom_rdev_name(std::string const& name) = 0;
 

src/lib/replication/repl_dev/raft_repl_dev.cpp

@@ -1738,6 +1738,10 @@ void RaftReplDev::reconcile_leader() {
     RD_LOGI(NO_TRACE_ID, "Yielded leadership");
 }
 
+void RaftReplDev::yield_leadership(bool immediate_yield, replica_id_t candidate) {
+    raft_server()->yield_leadership(immediate_yield, nuraft_mesg::to_server_id(candidate));
+}
+
 std::set< replica_id_t > RaftReplDev::get_active_peers() const {
     auto repl_status = get_replication_status();
     std::set< replica_id_t > res;

src/lib/replication/repl_dev/raft_repl_dev.h

@@ -298,6 +298,7 @@ class RaftReplDev : public ReplDev,
     std::set< replica_id_t > get_active_peers() const;
     group_id_t group_id() const override { return m_group_id; }
     void reconcile_leader() override;
+    void yield_leadership(bool immediate_yield, replica_id_t candidate) override;
     void set_custom_rdev_name(std::string const& name) override {
         RD_LOGI(NO_TRACE_ID, "Resetting repl dev name from {} to {}", m_rdev_name, name);
         m_rdev_name = name;

src/lib/replication/repl_dev/solo_repl_dev.h

@@ -79,6 +79,7 @@ class SoloReplDev : public ReplDev {
             peer_info{.id_ = m_group_id, .replication_idx_ = 0, .last_succ_resp_us_ = 0, .priority_ = 1}};
     }
     void reconcile_leader() override {}
+    void yield_leadership(bool immediate_yield, replica_id_t candidate) override {}
     bool is_ready_for_traffic() const override { return true; }
     void set_stage(repl_dev_stage_t stage) override {}
     repl_dev_stage_t get_stage() const override { return repl_dev_stage_t::ACTIVE; }

src/tests/test_common/raft_repl_test_base.hpp

@@ -789,6 +789,10 @@ class RaftReplDevTestBase : public testing::Test {
 
     void reconcile_leader(std::shared_ptr< TestReplicatedDB > db) { db->repl_dev()->reconcile_leader(); }
 
+    void yield_leadership(std::shared_ptr< TestReplicatedDB > db, bool immediate_yield, replica_id_t candidate) {
+        db->repl_dev()->yield_leadership(immediate_yield, candidate);
+    }
+
 protected:
     std::vector< std::shared_ptr< TestReplicatedDB > > dbs_;
     uint32_t written_entries_{0};

src/tests/test_raft_repl_dev.cpp

@@ -750,20 +750,24 @@ TEST_F(RaftReplDevTest, ReconcileLeader) {
     this->validate_data();
     g_helper->sync_for_cleanup_start();
 
-    LOGINFO("Restart leader");
-    if (g_helper->replica_num() == 0) { g_helper->restart_homestore(); }
+    LOGINFO("Yield leader");
+    auto expected_leader_num = 1;
+    auto expected_leader = g_helper->replica_id(expected_leader_num);
+    if (g_helper->replica_num() == 0) { this->yield_leadership(dbs_[0], false, expected_leader); }
     g_helper->sync_for_verify_start();
     LOGINFO("Validate leader switched");
     std::this_thread::sleep_for(std::chrono::milliseconds{500});
     auto leader = this->wait_and_get_leader_id();
-    auto leader_replica_num = 0;
-    if (g_helper->replica_num() == 0) {
-        ASSERT_NE(leader, g_helper->my_replica_id());
-        LOGINFO("Leader has changed");
-    } else if (g_helper->my_replica_id() == leader) {
-        leader_replica_num = g_helper->replica_num();
-        LOGINFO("New leader is replica={}, replica_num={}", leader, leader_replica_num);
-    }
+    ASSERT_EQ(leader, expected_leader);
+
+    g_helper->sync_for_test_start();
+    LOGINFO("Trigger reconcile leader on follower, expected no change")
+    if (g_helper->replica_num() == 2) { this->reconcile_leader(dbs_[0]); }
+    g_helper->sync_for_verify_start();
+    LOGINFO("Validate leader unchanged");
+    std::this_thread::sleep_for(std::chrono::milliseconds{500});
+    leader = this->wait_and_get_leader_id();
+    ASSERT_EQ(leader, expected_leader);
 
     g_helper->sync_for_test_start();
     if (g_helper->replica_num() == 0) {
@@ -777,19 +781,13 @@ TEST_F(RaftReplDevTest, ReconcileLeader) {
     if (g_helper->replica_num() == 0) { ASSERT_EQ(leader, g_helper->my_replica_id()); }
     g_helper->sync_for_cleanup_start();
 
-    LOGINFO("Restart leader again");
-    if (g_helper->replica_num() == 0) { g_helper->restart_homestore(); }
+    LOGINFO("Yield leader again");
+    if (g_helper->replica_num() == 0) { this->yield_leadership(dbs_[0], false, expected_leader); }
     g_helper->sync_for_verify_start();
     LOGINFO("Validate leader switched");
     std::this_thread::sleep_for(std::chrono::milliseconds{500});
     leader = this->wait_and_get_leader_id();
-    if (g_helper->replica_num() == 0) {
-        ASSERT_NE(leader, g_helper->my_replica_id());
-        LOGINFO("Leader has changed again");
-    } else if (g_helper->my_replica_id() == leader) {
-        leader_replica_num = g_helper->replica_num();
-        LOGINFO("New leader is replica={}, replica_num={}", leader, leader_replica_num);
-    }
+    ASSERT_EQ(leader, expected_leader);
     g_helper->sync_for_test_start();
 
     if (g_helper->my_replica_id() == leader) {
@@ -800,11 +798,7 @@ TEST_F(RaftReplDevTest, ReconcileLeader) {
     std::this_thread::sleep_for(std::chrono::milliseconds{1000});
     leader = this->wait_and_get_leader_id();
     LOGINFO("Validate leader switched back to initial replica, leader={}", leader);
-    if (g_helper->replica_num() == 0 && leader != g_helper->my_replica_id()) {
-        // yield leadership cannot garantee replica 0 will be leader again
-        // so we just log a warning here if it is not
-        LOGWARN("Leader is not replica 0 after yielding, leader={}", leader);
-    }
+    if (g_helper->replica_num() == 0) { ASSERT_EQ(leader, g_helper->my_replica_id()); }
     g_helper->sync_for_cleanup_start();
 }