refactor: init to support external storage for history tables (#18185)

* refactor(storage-config): split StorageParams out from databend-common-meta-app

* chore: add new config for external storage

* refactor: split meta related operations out from `GlobalHistoryLog`

* change default level from WARN to INFO

Author: dqhl76

Cargo.lock

@@ -112,6 +112,7 @@ members = [
     "src/tests/sqlsmith",
     "src/bendsave",
     "src/bendpy",
+    "src/meta/app-storage",
 ]
 
 # Workspace dependencies
@@ -142,6 +143,7 @@ databend-common-license = { path = "src/common/license" }
 databend-common-management = { path = "src/query/management" }
 databend-common-meta-api = { path = "src/meta/api" }
 databend-common-meta-app = { path = "src/meta/app" }
+databend-common-meta-app-storage = { path = "src/meta/app-storage" }
 databend-common-meta-app-types = { path = "src/meta/app-types" }
 databend-common-meta-cache = { path = "src/meta/cache" }
 databend-common-meta-client = { path = "src/meta/client" }

Cargo.toml

@@ -16,6 +16,7 @@ chrono = { workspace = true }
 concurrent-queue = { workspace = true }
 databend-common-base = { workspace = true }
 databend-common-exception = { workspace = true }
+databend-common-meta-app-storage = { workspace = true }
 defer = { workspace = true }
 fastrace = { workspace = true }
 fastrace-opentelemetry = { workspace = true }

src/common/tracing/Cargo.toml

@@ -16,6 +16,7 @@ use std::collections::BTreeMap;
 use std::fmt::Display;
 use std::fmt::Formatter;
 
+use databend_common_meta_app_storage::StorageParams;
 use itertools::Itertools;
 
 // use the uncommon usage of Pascal Case level name
@@ -346,6 +347,7 @@ pub struct HistoryConfig {
     pub level: String,
     pub retention_interval: usize,
     pub tables: Vec<HistoryTableConfig>,
+    pub storage_params: Option<StorageParams>,
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
@@ -358,7 +360,7 @@ impl Display for HistoryConfig {
     fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
         write!(
             f,
-            "enabled={}, log_only={}, interval={}, stage_name={}, level={}, retention_interval={}, tables=[{}]",
+            "enabled={}, log_only={}, interval={}, stage_name={}, level={}, retention_interval={}, tables=[{}], storage_params={}",
             self.on,
             self.log_only,
             self.interval,
@@ -368,7 +370,9 @@ impl Display for HistoryConfig {
             self.tables
                 .iter()
                 .map(|f| format!("{}({} hours)", f.table_name.clone(), f.retention))
-                .join(", ")
+                .join(", "),
+            self.storage_params.as_ref()
+                .map_or("None".to_string(), |p| p.to_string())
         )
     }
 }
@@ -381,10 +385,11 @@ impl Default for HistoryConfig {
             log_only: false,
             // The default value of stage name uses an uuid to avoid conflicts with existing stages
             stage_name: "log_1f93b76af0bd4b1d8e018667865fbc65".to_string(),
-            level: "WARN".to_string(),
+            level: "INFO".to_string(),
             // Trigger the retention task every 24 hours
             retention_interval: 24,
             tables: vec![],
+            storage_params: None,
         }
     }
 }

src/common/tracing/src/config.rs

@@ -0,0 +1,16 @@
+[package]
+name = "databend-common-meta-app-storage"
+version.workspace = true
+authors.workspace = true
+license.workspace = true
+publish.workspace = true
+edition.workspace = true
+
+[dependencies]
+databend-common-base = { workspace = true }
+databend-common-exception = { workspace = true }
+opendal = { workspace = true }
+serde = { workspace = true }
+
+[lints]
+workspace = true

src/meta/app-storage/Cargo.toml

@@ -0,0 +1,18 @@
+// Copyright 2021 Datafuse Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![feature(no_sanitize)]
+mod storage_params;
+
+pub use storage_params::*;

src/meta/app-storage/src/lib.rs

@@ -13,10 +13,10 @@ storage-hdfs = []
 anyerror = { workspace = true }
 chrono = { workspace = true }
 databend-common-ast = { workspace = true }
-databend-common-base = { workspace = true }
 databend-common-exception = { workspace = true }
 databend-common-expression = { workspace = true }
 databend-common-io = { workspace = true }
+databend-common-meta-app-storage = { workspace = true }
 databend-common-meta-app-types = { workspace = true }
 databend-common-meta-kvapi = { workspace = true }
 databend-common-meta-types = { workspace = true }
@@ -28,7 +28,6 @@ itertools = { workspace = true }
 maplit = { workspace = true }
 num-derive = { workspace = true }
 num-traits = { workspace = true }
-opendal = { workspace = true }
 paste = { workspace = true }
 prost = { workspace = true }
 serde = { workspace = true }

…c/meta/app/src/storage/storage_params.rs …c/meta/app-storage/src/storage_params.rssrc/meta/app/src/storage/storage_params.rs renamed to src/meta/app-storage/src/storage_params.rs src/meta/app/src/storage/storage_params.rs renamed to src/meta/app-storage/src/storage_params.rs

@@ -12,6 +12,5 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-mod storage_params;
-
-pub use storage_params::*;
+use databend_common_meta_app_storage;
+pub use databend_common_meta_app_storage::*;

src/meta/app/Cargo.toml

@@ -2769,7 +2769,7 @@ pub struct HistoryLogConfig {
     #[clap(
         long = "log-history-level",
         value_name = "VALUE",
-        default_value = "WARN"
+        default_value = "INFO"
     )]
     #[serde(rename = "level")]
     pub log_history_level: String,
@@ -2788,6 +2788,13 @@ pub struct HistoryLogConfig {
     #[clap(skip)]
     #[serde(rename = "tables")]
     pub log_history_tables: Vec<HistoryLogTableConfig>,
+
+    /// Specifies the storage parameters for the history log
+    /// This is used to configure how the history log data is stored.
+    /// If not specified, the default storage parameters from `[storage]` will be used.
+    #[clap(skip)]
+    #[serde(rename = "storage")]
+    pub log_history_storage_params: Option<StorageConfig>,
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -2837,6 +2844,12 @@ impl TryInto<InnerHistoryConfig> for HistoryLogConfig {
     type Error = ErrorCode;
 
     fn try_into(self) -> Result<InnerHistoryConfig> {
+        let storage_params: Option<InnerStorageConfig> =
+            if self.log_history_storage_params.is_some() {
+                Some(self.log_history_storage_params.unwrap().try_into()?)
+            } else {
+                None
+            };
         Ok(InnerHistoryConfig {
             on: self.log_history_on,
             log_only: self.log_history_log_only,
@@ -2849,12 +2862,18 @@ impl TryInto<InnerHistoryConfig> for HistoryLogConfig {
                 .into_iter()
                 .map(|cfg| cfg.try_into())
                 .collect::<Result<Vec<_>>>()?,
+            storage_params: storage_params.map(|cfg| cfg.params),
         })
     }
 }
 
 impl From<InnerHistoryConfig> for HistoryLogConfig {
     fn from(inner: InnerHistoryConfig) -> Self {
+        let inner_storage_config: Option<InnerStorageConfig> =
+            inner.storage_params.map(|params| InnerStorageConfig {
+                params,
+                ..Default::default()
+            });
         Self {
             log_history_on: inner.on,
             log_history_log_only: inner.log_only,
@@ -2863,6 +2882,7 @@ impl From<InnerHistoryConfig> for HistoryLogConfig {
             log_history_level: inner.level,
             log_history_retention_interval: inner.retention_interval,
             log_history_tables: inner.tables.into_iter().map(Into::into).collect(),
+            log_history_storage_params: inner_storage_config.map(Into::into),
         }
     }
 }