fix: when saved endpoints is empty, use endpoint config every try_config_endpoints_interval (#10848)

* fix: when saved endpoints is empty, use endpoint config every try_config_endpoints_interval

* fix: when saved endpoints is empty, use endpoint config every try_config_endpoints_interval

* fix: when saved endpoints is empty, use endpoint config every try_config_endpoints_interval

* refactor: refactor by review comment

* refactor: add unhealth_endpoint_evict_time config

* fix: remove unused config

* fix: fix test fail

---------

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Xuanwo <[email protected]>

Author: 3 people

docs/doc/10-deploy/07-query/10-query-config.md

@@ -72,6 +72,11 @@ You can find [sample configuration files](https://github.com/datafuselabs/databe
 * This setting only takes effect when Databend-query works with remote meta service(`endpoints` is not empty). You don't need to configure it for standalone Databend.
 * Default: 60
 
+### unhealth_endpoint_evict_time
+
+* Internal(in seconds) time that not querying an unhealth meta node endpoint.
+* Default: 120
+
 ## 3. Query config
 
 ### admin_api_address

docs/doc/13-sql-reference/20-system-tables/system-configs.md

@@ -66,6 +66,7 @@ mysql> SELECT * FROM system.configs;
 | meta    | auto_sync_interval                     | 0                              |             |
 | meta    | rpc_tls_meta_server_root_ca_cert       |                                |             |
 | meta    | rpc_tls_meta_service_domain_name       | localhost                      |             |
+| meta    | unhealth_endpoint_evict_time           | 120                            |             |
 | cache   | enable_table_meta_cache                | true                           |             |
 | cache   | table_meta_snapshot_count              | 256                            |             |
 | cache   | table_meta_segment_count               | 10240                          |             |

src/binaries/metabench/main.rs

@@ -17,6 +17,7 @@
 use std::fmt::Debug;
 use std::fmt::Display;
 use std::sync::Arc;
+use std::time::Duration;
 use std::time::Instant;
 
 use clap::Parser;
@@ -82,8 +83,15 @@ async fn main() {
         let prefix = config.prefix;
 
         let handle = tokio::spawn(async move {
-            let client =
-                MetaGrpcClient::try_create(vec![addr.to_string()], "root", "xxx", None, None, None);
+            let client = MetaGrpcClient::try_create(
+                vec![addr.to_string()],
+                "root",
+                "xxx",
+                None,
+                None,
+                Duration::from_secs(10),
+                None,
+            );
 
             let client = match client {
                 Ok(client) => client,

src/binaries/metactl/grpc.rs

@@ -14,14 +14,22 @@
 
 use std::fs::File;
 use std::io::Write;
+use std::time::Duration;
 
 use common_meta_client::MetaGrpcClient;
 use common_meta_types::protobuf::Empty;
 use tokio_stream::StreamExt;
 
 pub async fn export_meta(addr: &str, save: String) -> anyhow::Result<()> {
-    let client =
-        MetaGrpcClient::try_create(vec![addr.to_string()], "root", "xxx", None, None, None)?;
+    let client = MetaGrpcClient::try_create(
+        vec![addr.to_string()],
+        "root",
+        "xxx",
+        None,
+        None,
+        Duration::from_secs(10),
+        None,
+    )?;
 
     let mut grpc_client = client.make_client().await?;
 

src/binaries/metactl/main.rs

@@ -19,6 +19,8 @@ use grpc::export_meta;
 
 mod snapshot;
 
+use std::time::Duration;
+
 use clap::Parser;
 use common_base::base::tokio;
 use common_meta_client::MetaGrpcClient;
@@ -263,8 +265,15 @@ async fn bench_client_num_conn(conf: &Config) -> anyhow::Result<()> {
 
     loop {
         i += 1;
-        let client =
-            MetaGrpcClient::try_create(vec![addr.to_string()], "root", "xxx", None, None, None)?;
+        let client = MetaGrpcClient::try_create(
+            vec![addr.to_string()],
+            "root",
+            "xxx",
+            None,
+            None,
+            Duration::from_secs(10),
+            None,
+        )?;
 
         let res = client.get_kv("foo").await;
         println!("{}-th: get_kv(foo): {:?}", i, res);

src/common/grpc/src/client_conf.rs

@@ -38,6 +38,7 @@ pub struct RpcClientConf {
     /// AutoSyncInterval is the interval to update endpoints with its latest members.
     /// None disables auto-sync.
     pub auto_sync_interval: Option<Duration>,
+    pub unhealth_endpoint_evict_time: Duration,
 }
 
 impl RpcClientConf {

src/meta/client/src/grpc_client.rs

@@ -300,6 +300,7 @@ impl MetaGrpcClient {
             &conf.password,
             conf.timeout,
             conf.auto_sync_interval,
+            conf.unhealth_endpoint_evict_time,
             conf.tls_conf.clone(),
         )
     }
@@ -311,6 +312,7 @@ impl MetaGrpcClient {
         password: &str,
         timeout: Option<Duration>,
         auto_sync_interval: Option<Duration>,
+        unhealth_endpoint_evict_time: Duration,
         conf: Option<RpcClientTlsConfig>,
     ) -> Result<Arc<ClientHandle>, MetaClientError> {
         Self::endpoints_non_empty(&endpoints)?;
@@ -339,7 +341,7 @@ impl MetaGrpcClient {
             conn_pool: Pool::new(mgr, Duration::from_millis(50)),
             endpoints: RwLock::new(endpoints),
             current_endpoint: Arc::new(Mutex::new(None)),
-            unhealthy_endpoints: Mutex::new(TtlHashMap::new(Duration::from_secs(120))),
+            unhealthy_endpoints: Mutex::new(TtlHashMap::new(unhealth_endpoint_evict_time)),
             auto_sync_interval,
             username: username.to_string(),
             password: password.to_string(),

src/meta/client/tests/it/grpc_client.rs

@@ -36,8 +36,16 @@ async fn test_grpc_client_action_timeout() {
     // server's handshake impl will sleep 2secs.
     let timeout = Duration::from_secs(3);
 
-    let client =
-        MetaGrpcClient::try_create(vec![srv_addr], "", "", Some(timeout), None, None).unwrap();
+    let client = MetaGrpcClient::try_create(
+        vec![srv_addr],
+        "",
+        "",
+        Some(timeout),
+        None,
+        Duration::from_secs(10),
+        None,
+    )
+    .unwrap();
 
     let res = client
         .get_database(GetDatabaseReq::new("tenant1", "xx"))

src/meta/service/tests/it/grpc/metasrv_grpc_api.rs

@@ -49,6 +49,7 @@ async fn test_restart() -> anyhow::Result<()> {
         "xxx",
         None,
         Some(Duration::from_secs(10)),
+        Duration::from_secs(10),
         None,
     )?;
 
@@ -116,6 +117,7 @@ async fn test_restart() -> anyhow::Result<()> {
         "xxx",
         None,
         Some(Duration::from_secs(10)),
+        Duration::from_secs(10),
         None,
     )?;
 
@@ -211,6 +213,7 @@ async fn test_join() -> anyhow::Result<()> {
         "xxx",
         None,
         Some(Duration::from_secs(10)),
+        Duration::from_secs(10),
         None,
     )?;
     let client1 = MetaGrpcClient::try_create(
@@ -219,6 +222,7 @@ async fn test_join() -> anyhow::Result<()> {
         "xxx",
         None,
         Some(Duration::from_secs(10)),
+        Duration::from_secs(10),
         None,
     )?;
 
@@ -308,6 +312,7 @@ async fn test_auto_sync_addr() -> anyhow::Result<()> {
         "xxx",
         None,
         Some(Duration::from_secs(5)),
+        Duration::from_secs(10),
         None,
     )?;
 

src/meta/service/tests/it/grpc/metasrv_grpc_export.rs

@@ -41,6 +41,7 @@ async fn test_export() -> anyhow::Result<()> {
         "xxx",
         None,
         Some(Duration::from_secs(10)),
+        Duration::from_secs(10),
         None,
     )?;