feat: add meta join cluster timeout config (#10877)

* feat: add meta join cluster timeout config

* feat: add meta join cluster timeout config

* feat: add meta wait cluster leader timeout config

Author: lichuang

docs/doc/10-deploy/06-metasrv/15-metasrv-config.md

@@ -47,6 +47,7 @@ heartbeat_interval       = 1000 # milli second
 install_snapshot_timeout = 4000 # milli second
 max_applied_log_to_keep  = 1000 # N.O. raft logs
 snapshot_logs_since_last = 1024 # N.O. raft logs
+wait_leader_timeout     = 70000
 #
 # Startup config
 #
@@ -101,6 +102,8 @@ Defines raft behaviors on raft-storage and the state machine.
 
 - `snapshot_logs_since_last` specifies the number of raft-logs since the last snapshot beyond which a snapshot will be generated.
 
+- `wait_leader_timeout` specifies the max time for waiting a cluster leader in milliseconds.
+
 ## 6. Startup config
 
 - `single` tells the node to initialize a single node cluster if it is not

src/binaries/meta/main.rs

@@ -177,7 +177,7 @@ async fn register_node(meta_node: &Arc<MetaNode>, conf: &Config) -> Result<(), a
         "Register node to update raft_api_advertise_host_endpoint and grpc_api_advertise_address"
     );
 
-    let wait_leader_timeout = Duration::from_millis(conf.raft_config.election_timeout().1 * 10);
+    let wait_leader_timeout = Duration::from_millis(conf.raft_config.wait_leader_timeout);
     info!(
         "Wait {:?} for active leader to register node, raft election timeouts: {:?}",
         wait_leader_timeout,

src/meta/raft-store/src/config.rs

@@ -110,6 +110,9 @@ pub struct RaftConfig {
     ///  The node name. If the user specifies a name,
     /// the user-supplied name is used, if not, the default name is used.
     pub cluster_name: String,
+
+    /// Max timeout(in milli seconds) when waiting a cluster leader.
+    pub wait_leader_timeout: u64,
 }
 
 pub fn get_default_raft_advertise_host() -> String {
@@ -142,6 +145,7 @@ impl Default for RaftConfig {
             id: 0,
             sled_tree_prefix: "".to_string(),
             cluster_name: "foo_cluster".to_string(),
+            wait_leader_timeout: 70000,
         }
     }
 }

src/meta/service/src/configs/outer_v0.rs

@@ -277,6 +277,7 @@ pub struct ConfigViaEnv {
     pub kvsrv_snapshot_logs_since_last: u64,
     pub kvsrv_heartbeat_interval: u64,
     pub kvsrv_install_snapshot_timeout: u64,
+    pub kvsrv_wait_leader_timeout: u64,
     pub raft_max_applied_log_to_keep: u64,
     pub kvsrv_single: bool,
     pub metasrv_join: Vec<String>,
@@ -320,6 +321,7 @@ impl From<Config> for ConfigViaEnv {
             kvsrv_snapshot_logs_since_last: cfg.raft_config.snapshot_logs_since_last,
             kvsrv_heartbeat_interval: cfg.raft_config.heartbeat_interval,
             kvsrv_install_snapshot_timeout: cfg.raft_config.install_snapshot_timeout,
+            kvsrv_wait_leader_timeout: cfg.raft_config.wait_leader_timeout,
             raft_max_applied_log_to_keep: cfg.raft_config.max_applied_log_to_keep,
             kvsrv_single: cfg.raft_config.single,
             metasrv_join: cfg.raft_config.join,
@@ -344,6 +346,7 @@ impl Into<Config> for ConfigViaEnv {
             snapshot_logs_since_last: self.kvsrv_snapshot_logs_since_last,
             heartbeat_interval: self.kvsrv_heartbeat_interval,
             install_snapshot_timeout: self.kvsrv_install_snapshot_timeout,
+            wait_leader_timeout: self.kvsrv_wait_leader_timeout,
             max_applied_log_to_keep: self.raft_max_applied_log_to_keep,
             single: self.kvsrv_single,
             join: self.metasrv_join,
@@ -483,6 +486,10 @@ pub struct RaftConfig {
     /// if not, the default name is used
     #[clap(long, default_value = "foo_cluster")]
     pub cluster_name: String,
+
+    /// Max timeout(in milli seconds) when waiting a cluster leader.
+    #[clap(long, default_value = "70000")]
+    pub wait_leader_timeout: u64,
 }
 
 impl Default for RaftConfig {
@@ -511,6 +518,7 @@ impl From<RaftConfig> for InnerRaftConfig {
             id: x.id,
             sled_tree_prefix: x.sled_tree_prefix,
             cluster_name: x.cluster_name,
+            wait_leader_timeout: x.wait_leader_timeout,
         }
     }
 }
@@ -535,6 +543,7 @@ impl From<InnerRaftConfig> for RaftConfig {
             id: inner.id,
             sled_tree_prefix: inner.sled_tree_prefix,
             cluster_name: inner.cluster_name,
+            wait_leader_timeout: inner.wait_leader_timeout,
         }
     }
 }

src/meta/service/src/meta_service/raftmeta.rs

@@ -663,6 +663,7 @@ impl MetaNode {
                 }
             }
         }
+
         Err(MetaManagementError::Join(AnyError::error(format!(
             "fail to join {} cluster via {:?}, caused by errors: {}",
             self.sto.id,

src/meta/service/tests/it/configs.rs

@@ -59,6 +59,7 @@ no_sync = true
 snapshot_logs_since_last = 1000
 heartbeat_interval = 2000
 install_snapshot_timeout = 3000
+wait_leader_timeout = 3000
 single = false
 join = ["j1", "j2"]
 id = 20
@@ -85,6 +86,7 @@ cluster_name = "foo_cluster"
         assert_eq!(cfg.raft_config.snapshot_logs_since_last, 1000);
         assert_eq!(cfg.raft_config.heartbeat_interval, 2000);
         assert_eq!(cfg.raft_config.install_snapshot_timeout, 3000);
+        assert_eq!(cfg.raft_config.wait_leader_timeout, 3000);
         assert!(!cfg.raft_config.single);
         assert_eq!(cfg.raft_config.join, vec!["j1", "j2"]);
         assert_eq!(cfg.raft_config.id, 20);