refactor: add query_profile and query_details persistent log tables (#17702)

* feat: allow schema change for persistent log table

* chore: add QueryProfileTable and QueryDetailsTable

* fixup

* test: add logic tests

* fix: pre-list to fix

* fixup

* chore: remove columns that may not be useful for now

* chore: remove columns that may not be useful for now

* apply review suggestions

* apply review suggestion

* add version number support

* fixup

clippy

* temp change docker config for benchmark

* Revert "temp change docker config for benchmark"

This reverts commit b17e25f.

Author: dqhl76

Cargo.lock

@@ -63,7 +63,7 @@ impl GlobalLogger {
     }
 
     // Get the operator for remote log when it is ready.
-    pub(crate) async fn get_operator(&self) -> Option<Operator> {
+    pub async fn get_operator(&self) -> Option<Operator> {
         let operator = self.remote_log_operator.read().await;
         if let Some(operator) = operator.as_ref() {
             return Some(operator.clone());
@@ -383,14 +383,14 @@ pub fn init_logging(
         let mut filter_builder =
             EnvFilterBuilder::new().filter(Some("databend::log::structlog"), LevelFilter::Off);
 
-        if cfg.profile.on && !cfg.profile.dir.is_empty() {
+        if cfg.profile.on {
             filter_builder =
                 filter_builder.filter(Some("databend::log::profile"), LevelFilter::Trace);
         } else {
             filter_builder =
                 filter_builder.filter(Some("databend::log::profile"), LevelFilter::Off);
         }
-        if cfg.query.on && !cfg.query.dir.is_empty() {
+        if cfg.query.on {
             filter_builder =
                 filter_builder.filter(Some("databend::log::query"), LevelFilter::Trace);
         } else {

src/common/tracing/src/init.rs

@@ -53,6 +53,7 @@ pub use crate::remote_log::LogMessage;
 pub use crate::remote_log::RemoteLog;
 pub use crate::remote_log::RemoteLogElement;
 pub use crate::remote_log::RemoteLogGuard;
+pub use crate::remote_log::PERSISTENT_LOG_SCHEMA_VERSION;
 pub use crate::structlog::DummyReporter;
 pub use crate::structlog::StructLogReporter;
 

src/common/tracing/src/lib.rs

@@ -53,6 +53,8 @@ use serde_json::Map;
 use crate::Config;
 use crate::GlobalLogger;
 
+pub const PERSISTENT_LOG_SCHEMA_VERSION: usize = 1;
+
 /// An appender that sends log records to persistent storage
 #[derive(Debug)]
 pub struct RemoteLog {
@@ -178,8 +180,9 @@ impl RemoteLog {
         op.as_ref()?;
 
         let path = format!(
-            "stage/internal/{}/{}.parquet",
+            "stage/internal/{}_v{}/{}.parquet",
             stage_name,
+            PERSISTENT_LOG_SCHEMA_VERSION,
             uuid::Uuid::new_v4()
         );
 

src/common/tracing/src/remote_log.rs

@@ -67,6 +67,7 @@ databend-common-management = { workspace = true }
 databend-common-meta-api = { workspace = true }
 databend-common-meta-app = { workspace = true }
 databend-common-meta-app-types = { workspace = true }
+databend-common-meta-client = { workspace = true }
 databend-common-meta-kvapi = { workspace = true }
 databend-common-meta-semaphore = { workspace = true }
 databend-common-meta-store = { workspace = true }