chore: optimize compilation cache hit rate (#18529)

Author: forsaken628

.config/version-schema.json

@@ -0,0 +1,12 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "dependencies": {
+      "type": "object",
+      "properties": {
+        "databend-common-version": false
+      }
+    }
+  }
+}

Cargo.lock

@@ -219,7 +219,7 @@ databend-storages-common-stage = { path = "src/query/storages/common/stage" }
 databend-storages-common-table-meta = { path = "src/query/storages/common/table_meta" }
 
 # Crates.io dependencies
-ahash = "0.8"
+ahash = { version = "0.8", features = ["no-rng"] }
 aho-corasick = { version = "1.0.1" } #
 anyerror = { version = "=0.1.13" }
 anyhow = { version = "1.0.65" }
@@ -231,11 +231,11 @@ arrow-buffer = { version = "55" }
 arrow-cast = { version = "55", features = ["prettyprint"] }
 arrow-data = { version = "55" }
 arrow-flight = { version = "55", features = ["flight-sql-experimental", "tls"] }
-arrow-ipc = { version = "55" }
+arrow-ipc = { version = "55", features = ["lz4", "zstd"] }
 arrow-ord = { version = "55" }
 arrow-schema = { version = "55", features = ["serde"] }
 arrow-select = { version = "55" }
-arrow-udf-runtime = { version = "0.8.0", default-features = false }
+arrow-udf-runtime = { version = "0.8.0", default-features = false, features = ["javascript", "wasm"] }
 async-backtrace = "0.2"
 async-channel = "1.7.1"
 async-compat = { version = "0.2" }
@@ -246,7 +246,7 @@ async-compression = { git = "https://github.com/datafuse-extras/async-compressio
 async-recursion = "1.1.1"
 async-stream = "0.3.3"
 async-trait = { version = "0.1.88" }
-backoff = "0.4" # FIXME: use backon to replace this.
+backoff = { version = "0.4", features = ["futures", "tokio"] } # FIXME: use backon to replace this.
 backon = "1"
 backtrace = { version = "0.3.73", features = [
     "std",
@@ -260,12 +260,12 @@ bitvec = "1.0.1"
 blake3 = "1.3.1"
 bollard = { version = "0.17" }
 borsh = { version = "1.2.1", features = ["derive"] }
-brotli = "3.3"
+brotli = { version = "3.3", features = ["std"] }
 bstr = "1"
 buf-list = "1.0.3"
 bumpalo = "3.12.0"
 byte-unit = "5.1.6"
-bytemuck = { version = "1", features = ["derive"] }
+bytemuck = { version = "1", features = ["derive", "extern_crate_alloc", "must_cast", "transparentwrapper_extra"] }
 byteorder = "1.4.3"
 bytes = "1.5.0"
 bytesize = "2"
@@ -286,7 +286,7 @@ crossbeam-channel = "0.5.6"
 csv-core = "0.1.11"
 ctor = "0.2"
 ctrlc = { version = "3.2.3", features = ["termination"] }
-dashmap = "6.1.0"
+dashmap = { version = "6.1.0", features = ["serde"] }
 deepsize = { version = "0.2.0" }
 defer = "0.2"
 deltalake = "0.26"
@@ -302,7 +302,7 @@ enquote = "1.1.0"
 enum-as-inner = "0.6"
 enum_dispatch = "0.3.13"
 enumflags2 = { version = "0.7.7", features = ["serde"] }
-ethnum = { version = "1.5.1" }
+ethnum = { version = "1.5.1", features = ["serde", "macros"] }
 faststr = "0.2"
 feature-set = { version = "0.1.1" }
 feistel-permutation-rs = "0.1.1"
@@ -385,7 +385,7 @@ mockall = "0.11.2"
 mysql_async = { version = "0.34", default-features = false, features = ["native-tls-tls"] }
 naive-cityhash = "0.2.0"
 ndarray = "0.15.6"
-num = "0.4.0"
+num = { version = "0.4.0", features = ["std"] }
 num-bigint = "0.4.6"
 num-derive = "0.4.2"
 num-traits = "0.2.19"
@@ -442,7 +442,7 @@ prost = { version = "0.13" }
 prost-build = { version = "0.13" }
 prqlc = "0.11.3"
 raft-log = { version = "0.2.9" }
-rand = { version = "0.8.5", features = ["small_rng"] }
+rand = { version = "0.8.5", features = ["small_rng", "serde1"] }
 rand_distr = "0.4.3"
 rayon = "1.9.0"
 recursive = "0.1.1"
@@ -454,6 +454,7 @@ reqwest = { version = "0.12", default-features = false, features = [
     "http2",
     "native-tls-vendored",
     "native-tls-alpn",
+    "cookies",
 ] }
 reqwest-hickory-resolver = "0.2"
 ringbuffer = "0.14.2"
@@ -512,13 +513,13 @@ tempfile = "3.4.0"
 terminal_size = "0.4.2"
 test-harness = "0.3.0"
 testcontainers = "0.23"
-testcontainers-modules = "0.11.5"
+testcontainers-modules = { version = "0.11.5", features = ["mysql", "redis"] }
 thiserror = { version = "1" }
 thrift = "0.17.0"
 tikv-jemalloc-ctl = { version = "0.6.0", features = ["use_std", "stats"] }
 tikv-jemalloc-sys = "0.6.0"
 tokio = { version = "1.35.0", features = ["full"] }
-tokio-stream = "0.1.11"
+tokio-stream = { version = "0.1.11", features = ["net"] }
 tokio-util = { version = "0.7.13" }
 toml = { version = "0.8", features = ["parse"] }
 tonic = { version = "0.12.3", features = ["transport", "codegen", "prost", "tls-roots", "tls"] }
@@ -539,8 +540,8 @@ watcher = { version = "0.4.2" }
 wiremock = "0.6"
 wkt = "0.11.1"
 xorf = { version = "0.11.0", default-features = false, features = ["binary-fuse"] }
-xorfilter-rs = "0.5"
-zerocopy = "0.8.26"
+xorfilter-rs = { version = "0.5", features = ["cbordata"] }
+zerocopy = { version = "0.8.26", features = ["derive"] }
 zip = "3.0.0"
 zstd = "0.12.3"
 
@@ -564,7 +565,7 @@ vergen = { version = "8.3.1", default-features = false, features = ["build", "ca
 env_logger = "0.11"
 fastrace = { version = "0.7.14", features = ["enable"] }
 fastrace-opentelemetry = "0.10"
-log = { version = "0.4.27", features = ["serde", "kv_unstable_std"] }
+log = { version = "0.4.27", features = ["serde", "kv_serde", "kv_unstable_std"] }
 logcall = "0.1.9"
 opentelemetry = { version = "0.29", features = ["trace", "logs"] }
 opentelemetry-otlp = { version = "0.29", default-features = false, features = [

Cargo.toml

@@ -25,6 +25,7 @@ databend-common-expression = { workspace = true }
 databend-common-license = { workspace = true }
 databend-common-meta-app = { workspace = true }
 databend-common-users = { workspace = true }
+databend-common-version = { workspace = true }
 databend-query = { workspace = true, features = [
     "simd",
     "disable_initial_exec_tls",

src/bendpy/Cargo.toml

@@ -22,6 +22,8 @@ use databend_common_meta_app::principal::UserInfo;
 use databend_common_meta_app::principal::UserPrivilegeSet;
 use databend_common_meta_app::tenant::Tenant;
 use databend_common_users::UserApiProvider;
+use databend_common_version::BUILD_INFO;
+use databend_query::sessions::BuildInfoRef;
 use databend_query::sessions::QueryContext;
 use databend_query::sessions::Session;
 use databend_query::sessions::SessionManager;
@@ -37,6 +39,7 @@ use crate::utils::RUNTIME;
 #[derive(Clone)]
 pub(crate) struct PySessionContext {
     pub(crate) session: Arc<Session>,
+    version: BuildInfoRef,
 }
 
 #[pymethods]
@@ -61,9 +64,12 @@ impl PySessionContext {
             let session = session_manager.register_session(session).unwrap();
 
             let config = GlobalConfig::instance();
-            UserApiProvider::try_create_simple(config.meta.to_meta_grpc_client_conf(), &tenant)
-                .await
-                .unwrap();
+            UserApiProvider::try_create_simple(
+                config.meta.to_meta_grpc_client_conf(&BUILD_INFO),
+                &tenant,
+            )
+            .await
+            .unwrap();
 
             let mut user = UserInfo::new_no_auth("root", "%");
             user.grants.grant_privileges(
@@ -75,15 +81,18 @@ impl PySessionContext {
             Ok::<Arc<Session>, PyErr>(session)
         })?;
 
-        let mut res = Self { session };
+        let mut res = Self {
+            session,
+            version: &BUILD_INFO,
+        };
 
         res.sql("CREATE DATABASE IF NOT EXISTS default", py)
             .and_then(|df| df.collect(py))?;
         Ok(res)
     }
 
     fn sql(&mut self, sql: &str, py: Python) -> PyResult<PyDataFrame> {
-        let ctx = wait_for_future(py, self.session.create_query_context()).unwrap();
+        let ctx = wait_for_future(py, self.session.create_query_context(self.version)).unwrap();
         let res = wait_for_future(py, plan_sql(&ctx, sql));
 
         match res {

src/bendpy/src/context.rs

@@ -29,6 +29,7 @@ use databend_common_config::Config;
 use databend_common_config::InnerConfig;
 use databend_common_license::license_manager::LicenseManager;
 use databend_common_license::license_manager::OssLicenseManager;
+use databend_common_version::BUILD_INFO;
 use databend_query::clusters::ClusterDiscovery;
 use databend_query::GlobalServices;
 use pyo3::prelude::*;
@@ -88,7 +89,7 @@ allow_insecure = true
     INIT.call_once(|| {
         RUNTIME
             .block_on(async {
-                GlobalServices::init(&conf, false).await?;
+                GlobalServices::init(&conf, &BUILD_INFO, false).await?;
                 // init oss license manager
                 OssLicenseManager::init("".to_string()).unwrap();
                 // Cluster register.

src/bendpy/src/lib.rs

@@ -29,9 +29,9 @@ futures = { workspace = true }
 http = { workspace = true }
 log = { workspace = true }
 logforth = { workspace = true }
-opendal = { workspace = true, features = ["services-s3", "services-fs"] }
+opendal = { workspace = true }
 tokio = { workspace = true }
-toml = { workspace = true, features = ["parse"] }
+toml = { workspace = true }
 
 [lints]
 workspace = true

src/bendsave/Cargo.toml

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use anyhow::Result;
+use databend_common_version::BUILD_INFO;
 use futures::SinkExt;
 use futures::TryStreamExt;
 use log::info;
@@ -30,7 +31,7 @@ use crate::utils::DATABEND_META_BACKUP_PATH;
 pub async fn backup(from: &str, to: &str) -> Result<()> {
     let query_cfg = load_query_config(from)?;
     init_query(&query_cfg)?;
-    verify_query_license(&query_cfg).await?;
+    verify_query_license(&query_cfg, &BUILD_INFO).await?;
     let databend_storage = load_query_storage(&query_cfg)?;
 
     let bendsave_storage = load_bendsave_storage(to).await?;

src/bendsave/src/backup.rs

@@ -38,7 +38,9 @@ use databend_common_meta_types::protobuf::ExportRequest;
 use databend_common_storage::init_operator;
 use databend_common_users::builtin::BuiltIn;
 use databend_common_users::UserApiProvider;
+use databend_common_version::BUILD_INFO;
 use databend_enterprise_query::license::RealLicenseManager;
+use databend_query::sessions::BuildInfoRef;
 use databend_query::sessions::SessionManager;
 use futures::TryStream;
 use futures::TryStreamExt;
@@ -71,7 +73,8 @@ pub fn load_query_storage(cfg: &InnerConfig) -> Result<Operator> {
 pub fn load_meta_config(path: &str) -> Result<databend_meta::configs::Config> {
     let content = std::fs::read_to_string(path)?;
     let outer_config: databend_meta::configs::outer_v0::Config = toml::from_str(&content)?;
-    let inner_config: databend_meta::configs::Config = outer_config.into();
+    let inner_config: databend_meta::configs::Config =
+        outer_config.try_into().map_err(|msg| anyhow!("{msg}"))?;
 
     if !inner_config.raft_config.raft_dir.starts_with("/") {
         return Err(anyhow!(
@@ -91,7 +94,7 @@ pub fn load_meta_config(path: &str) -> Result<databend_meta::configs::Config> {
 pub fn init_query(cfg: &InnerConfig) -> Result<()> {
     GlobalInstance::init_production();
 
-    GlobalConfig::init(cfg)?;
+    GlobalConfig::init(cfg, &BUILD_INFO)?;
     GlobalIORuntime::init(cfg.storage.num_cpus as usize)?;
 
     Ok(())
@@ -102,11 +105,11 @@ pub fn init_query(cfg: &InnerConfig) -> Result<()> {
 ///
 /// We only need to call it while backup since we can't access metasrv while
 /// restoring.
-pub async fn verify_query_license(cfg: &InnerConfig) -> Result<()> {
+pub async fn verify_query_license(cfg: &InnerConfig, version: BuildInfoRef) -> Result<()> {
     RealLicenseManager::init(cfg.query.tenant_id.tenant_name().to_string())?;
     SessionManager::init(cfg)?;
     UserApiProvider::init(
-        cfg.meta.to_meta_grpc_client_conf(),
+        cfg.meta.to_meta_grpc_client_conf(version),
         &CacheConfig::default(),
         BuiltIn::default(),
         &cfg.query.tenant_id,
@@ -119,8 +122,10 @@ pub async fn verify_query_license(cfg: &InnerConfig) -> Result<()> {
     let session = session_manager.register_session(session)?;
     let settings = session.get_settings();
 
-    LicenseManagerSwitch::instance()
-        .check_enterprise_enabled(settings.get_enterprise_license(), Feature::SystemManagement)?;
+    LicenseManagerSwitch::instance().check_enterprise_enabled(
+        settings.get_enterprise_license(version),
+        Feature::SystemManagement,
+    )?;
 
     debug!("databend license check passed");
     Ok(())
@@ -141,7 +146,7 @@ pub async fn load_databend_meta() -> Result<(
     impl TryStream<Ok = Bytes, Error = anyhow::Error>,
 )> {
     let cfg = GlobalConfig::instance();
-    let grpc_client_conf = cfg.meta.to_meta_grpc_client_conf();
+    let grpc_client_conf = cfg.meta.to_meta_grpc_client_conf(&BUILD_INFO);
     debug!("connect meta services on {:?}", grpc_client_conf.endpoints);
 
     let meta_client = MetaGrpcClient::try_new(&grpc_client_conf)?;