From 36827af4eb4326a4f4ce17491d28462f65c6b4f8 Mon Sep 17 00:00:00 2001 From: zyy17 Date: Thu, 2 Oct 2025 21:21:48 +0800 Subject: [PATCH 1/2] refactor: add `get_total_cpu_millicores()` / `get_total_cpu_cores()` / `get_total_memory_bytes()` / `get_total_memory_readable()` in common-stat Signed-off-by: zyy17 --- Cargo.lock | 9 ++- .../information_schema/cluster_info.rs | 20 ++++-- src/cmd/src/lib.rs | 8 ++- src/common/config/Cargo.toml | 3 +- src/common/config/src/utils.rs | 47 ++------------ src/common/stat/Cargo.toml | 3 + src/common/stat/src/cgroups.rs | 47 ++++++-------- src/common/stat/src/lib.rs | 61 +++++++++++++++++++ src/flow/Cargo.toml | 1 + src/flow/src/adapter.rs | 5 +- src/mito2/Cargo.toml | 1 + src/mito2/src/config.rs | 21 +++---- src/mito2/src/memtable/partition_tree.rs | 5 +- src/mito2/src/region/options.rs | 5 +- src/standalone/Cargo.toml | 1 + src/standalone/src/information_extension.rs | 6 +- .../information_schema/cluster_info.result | 54 ++++++++-------- .../information_schema/cluster_info.sql | 2 +- .../common/system/information_schema.result | 4 +- .../information_schema/cluster_info.result | 42 ++++++------- .../information_schema/cluster_info.sql | 2 +- 21 files changed, 191 insertions(+), 156 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 42f67c3137f6..947d72755283 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2033,6 +2033,7 @@ dependencies = [ "common-base", "common-error", "common-macro", + "common-stat", "common-telemetry", "common-test-util", "common-wal", @@ -2040,13 +2041,11 @@ dependencies = [ "datanode", "humantime-serde", "meta-client", - "num_cpus", "object-store", "serde", "serde_json", "serde_with", "snafu 0.8.6", - "sysinfo", "temp-env", "tempfile", "toml 0.8.23", @@ -2553,9 +2552,12 @@ dependencies = [ name = "common-stat" version = "0.18.0" dependencies = [ + "common-base", "lazy_static", "nix 0.30.1", + "num_cpus", "prometheus", + "sysinfo", ] [[package]] @@ -4790,6 +4792,7 @@ dependencies = [ "common-query", "common-recordbatch", "common-runtime", + "common-stat", "common-telemetry", "common-time", "common-version", @@ -7611,6 +7614,7 @@ dependencies = [ "common-query", "common-recordbatch", "common-runtime", + "common-stat", "common-telemetry", "common-test-util", "common-time", @@ -12353,6 +12357,7 @@ dependencies = [ "common-options", "common-procedure", "common-query", + "common-stat", "common-telemetry", "common-time", "common-version", diff --git a/src/catalog/src/system_schema/information_schema/cluster_info.rs b/src/catalog/src/system_schema/information_schema/cluster_info.rs index 4082629bd45f..af96a5504d68 100644 --- a/src/catalog/src/system_schema/information_schema/cluster_info.rs +++ b/src/catalog/src/system_schema/information_schema/cluster_info.rs @@ -50,8 +50,8 @@ const PEER_TYPE_METASRV: &str = "METASRV"; const PEER_ID: &str = "peer_id"; const PEER_TYPE: &str = "peer_type"; const PEER_ADDR: &str = "peer_addr"; -const CPUS: &str = "cpus"; -const MEMORY_BYTES: &str = "memory_bytes"; +const TOTAL_CPU_MILLICORES: &str = "total_cpu_millicores"; +const TOTAL_MEMORY_BYTES: &str = "total_memory_bytes"; const VERSION: &str = "version"; const GIT_COMMIT: &str = "git_commit"; const START_TIME: &str = "start_time"; @@ -66,8 +66,8 @@ const INIT_CAPACITY: usize = 42; /// - `peer_id`: the peer server id. /// - `peer_type`: the peer type, such as `datanode`, `frontend`, `metasrv` etc. /// - `peer_addr`: the peer gRPC address. -/// - `cpus`: the number of CPUs of the peer. -/// - `memory_bytes`: the memory bytes of the peer. +/// - `total_cpu_millicores`: the total CPU millicores of the peer. +/// - `total_memory_bytes`: the total memory bytes of the peer. /// - `version`: the build package version of the peer. /// - `git_commit`: the build git commit hash of the peer. /// - `start_time`: the starting time of the peer. @@ -94,8 +94,16 @@ impl InformationSchemaClusterInfo { ColumnSchema::new(PEER_ID, ConcreteDataType::int64_datatype(), false), ColumnSchema::new(PEER_TYPE, ConcreteDataType::string_datatype(), false), ColumnSchema::new(PEER_ADDR, ConcreteDataType::string_datatype(), true), - ColumnSchema::new(CPUS, ConcreteDataType::uint32_datatype(), false), - ColumnSchema::new(MEMORY_BYTES, ConcreteDataType::uint64_datatype(), false), + ColumnSchema::new( + TOTAL_CPU_MILLICORES, + ConcreteDataType::uint32_datatype(), + false, + ), + ColumnSchema::new( + TOTAL_MEMORY_BYTES, + ConcreteDataType::uint64_datatype(), + false, + ), ColumnSchema::new(VERSION, ConcreteDataType::string_datatype(), false), ColumnSchema::new(GIT_COMMIT, ConcreteDataType::string_datatype(), false), ColumnSchema::new( diff --git a/src/cmd/src/lib.rs b/src/cmd/src/lib.rs index b7cf30b8085f..3020a4359082 100644 --- a/src/cmd/src/lib.rs +++ b/src/cmd/src/lib.rs @@ -18,7 +18,7 @@ use async_trait::async_trait; use common_error::ext::ErrorExt; use common_error::status_code::StatusCode; use common_mem_prof::activate_heap_profile; -use common_stat::{get_cpu_limit, get_memory_limit}; +use common_stat::{get_total_cpu_millicores, get_total_memory_bytes}; use common_telemetry::{error, info, warn}; use crate::error::Result; @@ -125,7 +125,8 @@ pub fn log_versions(version: &str, short_version: &str, app: &str) { } pub fn create_resource_limit_metrics(app: &str) { - if let Some(cpu_limit) = get_cpu_limit() { + let cpu_limit = get_total_cpu_millicores(); + if cpu_limit > 0 { info!( "GreptimeDB start with cpu limit in millicores: {}", cpu_limit @@ -133,7 +134,8 @@ pub fn create_resource_limit_metrics(app: &str) { CPU_LIMIT.with_label_values(&[app]).set(cpu_limit); } - if let Some(memory_limit) = get_memory_limit() { + let memory_limit = get_total_memory_bytes(); + if memory_limit > 0 { info!( "GreptimeDB start with memory limit in bytes: {}", memory_limit diff --git a/src/common/config/Cargo.toml b/src/common/config/Cargo.toml index 17279e5eb77a..1d2b21602f48 100644 --- a/src/common/config/Cargo.toml +++ b/src/common/config/Cargo.toml @@ -11,15 +11,14 @@ workspace = true common-base.workspace = true common-error.workspace = true common-macro.workspace = true +common-stat.workspace = true config.workspace = true humantime-serde.workspace = true -num_cpus.workspace = true object-store.workspace = true serde.workspace = true serde_json.workspace = true serde_with.workspace = true snafu.workspace = true -sysinfo.workspace = true toml.workspace = true [dev-dependencies] diff --git a/src/common/config/src/utils.rs b/src/common/config/src/utils.rs index 912fdb0c1335..1bc986b77ef6 100644 --- a/src/common/config/src/utils.rs +++ b/src/common/config/src/utils.rs @@ -13,61 +13,22 @@ // limitations under the License. use common_base::readable_size::ReadableSize; -use sysinfo::System; - -/// Get the CPU core number of system, aware of cgroups. -pub fn get_cpus() -> usize { - // This function will check cgroups - num_cpus::get() -} - -/// Get the total memory of the system. -/// If `cgroup_limits` is enabled, it will also check it. -pub fn get_sys_total_memory() -> Option { - if sysinfo::IS_SUPPORTED_SYSTEM { - let mut sys_info = System::new(); - sys_info.refresh_memory(); - let mut total_memory = sys_info.total_memory(); - // Compare with cgroups memory limit, use smaller values - // This method is only implemented for Linux. It always returns None for all other systems. - if let Some(cgroup_limits) = sys_info.cgroup_limits() { - total_memory = total_memory.min(cgroup_limits.total_memory) - } - Some(ReadableSize(total_memory)) - } else { - None - } -} +use common_stat::{get_total_cpu_millicores, get_total_memory_readable}; /// `ResourceSpec` holds the static resource specifications of a node, /// such as CPU cores and memory capacity. These values are fixed /// at startup and do not change dynamically during runtime. #[derive(Debug, Clone, Copy)] pub struct ResourceSpec { - pub cpus: usize, + pub cpus: i64, pub memory: Option, } impl Default for ResourceSpec { fn default() -> Self { Self { - cpus: get_cpus(), - memory: get_sys_total_memory(), + cpus: get_total_cpu_millicores(), + memory: get_total_memory_readable(), } } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_get_cpus() { - assert!(get_cpus() > 0); - } - - #[test] - fn test_get_sys_total_memory() { - assert!(get_sys_total_memory().unwrap() > ReadableSize::mb(0)); - } -} diff --git a/src/common/stat/Cargo.toml b/src/common/stat/Cargo.toml index 02757e687dbd..3d0198f6a2fc 100644 --- a/src/common/stat/Cargo.toml +++ b/src/common/stat/Cargo.toml @@ -5,9 +5,12 @@ edition.workspace = true license.workspace = true [dependencies] +common-base.workspace = true lazy_static.workspace = true nix.workspace = true +num_cpus.workspace = true prometheus.workspace = true +sysinfo.workspace = true [lints] workspace = true diff --git a/src/common/stat/src/cgroups.rs b/src/common/stat/src/cgroups.rs index 45d66e6f357e..fe26f5ec3607 100644 --- a/src/common/stat/src/cgroups.rs +++ b/src/common/stat/src/cgroups.rs @@ -23,9 +23,6 @@ use prometheus::core::{Collector, Desc}; use prometheus::proto::MetricFamily; use prometheus::{IntGauge, Opts}; -/// `MAX_VALUE` is used to indicate that the resource is unlimited. -pub const MAX_VALUE: i64 = -1; - const CGROUP_UNIFIED_MOUNTPOINT: &str = "/sys/fs/cgroup"; const MEMORY_MAX_FILE_CGROUP_V2: &str = "memory.max"; @@ -43,11 +40,11 @@ const MAX_VALUE_CGROUP_V2: &str = "max"; // For easier comparison, if the memory limit is larger than 1PB we consider it as unlimited. const MAX_MEMORY_IN_BYTES: i64 = 1125899906842624; // 1PB -/// Get the limit of memory in bytes. +/// Get the limit of memory in bytes from cgroups filesystem. /// -/// - If the memory is unlimited, return `-1`. +/// - If the cgroup total memory is unset, return `None`. /// - Return `None` if it fails to read the memory limit or not on linux. -pub fn get_memory_limit() -> Option { +pub fn get_memory_limit_from_cgroups() -> Option { #[cfg(target_os = "linux")] { let memory_max_file = if is_cgroup_v2()? { @@ -58,13 +55,13 @@ pub fn get_memory_limit() -> Option { MEMORY_MAX_FILE_CGROUP_V1 }; - // For cgroup v1, it will return a very large value(different from platform) if the memory is unlimited. + // For cgroup v1, it will return a very large value(different from platform) if the memory is unset. let memory_limit = read_value_from_file(Path::new(CGROUP_UNIFIED_MOUNTPOINT).join(memory_max_file))?; - // If memory limit exceeds 1PB(cgroup v1), consider it as unlimited. + // If memory limit exceeds 1PB(cgroup v1), consider it as unset. if memory_limit > MAX_MEMORY_IN_BYTES { - return Some(MAX_VALUE); + return None; } Some(memory_limit) } @@ -73,10 +70,10 @@ pub fn get_memory_limit() -> Option { None } -/// Get the usage of memory in bytes. +/// Get the usage of memory in bytes from cgroups filesystem. /// /// - Return `None` if it fails to read the memory usage or not on linux or cgroup is v1. -pub fn get_memory_usage() -> Option { +pub fn get_memory_usage_from_cgroups() -> Option { #[cfg(target_os = "linux")] { if is_cgroup_v2()? { @@ -93,11 +90,11 @@ pub fn get_memory_usage() -> Option { None } -/// Get the limit of cpu in millicores. +/// Get the limit of cpu in millicores from cgroups filesystem. /// -/// - If the cpu is unlimited, return `-1`. +/// - If the cpu limit is unset, return `None`. /// - Return `None` if it fails to read the cpu limit or not on linux. -pub fn get_cpu_limit() -> Option { +pub fn get_cpu_limit_from_cgroups() -> Option { #[cfg(target_os = "linux")] if is_cgroup_v2()? { // Read `/sys/fs/cgroup/cpu.max` to get the cpu limit. @@ -108,10 +105,6 @@ pub fn get_cpu_limit() -> Option { Path::new(CGROUP_UNIFIED_MOUNTPOINT).join(CPU_QUOTA_FILE_CGROUP_V1), )?; - if quota == MAX_VALUE { - return Some(MAX_VALUE); - } - let period = read_value_from_file( Path::new(CGROUP_UNIFIED_MOUNTPOINT).join(CPU_PERIOD_FILE_CGROUP_V1), )?; @@ -167,9 +160,9 @@ fn is_cgroup_v2() -> Option { fn read_value_from_file>(path: P) -> Option { let content = read_to_string(&path).ok()?; - // If the content starts with "max", return `MAX_VALUE`. + // If the content starts with "max", return `None`. if content.starts_with(MAX_VALUE_CGROUP_V2) { - return Some(MAX_VALUE); + return None; } content.trim().parse::().ok() @@ -183,10 +176,10 @@ fn get_cgroup_v2_cpu_limit>(path: P) -> Option { return None; } - // If the cpu is unlimited, it will be `-1`. + // If the cgroup cpu limit is unset, return `None`. let quota = fields[0].trim(); if quota == MAX_VALUE_CGROUP_V2 { - return Some(MAX_VALUE); + return None; } let quota = quota.parse::().ok()?; @@ -241,7 +234,7 @@ impl Collector for CgroupsMetricsCollector { self.cpu_usage.set(cpu_usage); } - if let Some(memory_usage) = get_memory_usage() { + if let Some(memory_usage) = get_memory_usage_from_cgroups() { self.memory_usage.set(memory_usage); } @@ -263,8 +256,8 @@ mod tests { 100000 ); assert_eq!( - read_value_from_file(Path::new("testdata").join("memory.max.unlimited")).unwrap(), - MAX_VALUE + read_value_from_file(Path::new("testdata").join("memory.max.unlimited")), + None ); assert_eq!(read_value_from_file(Path::new("non_existent_file")), None); } @@ -276,8 +269,8 @@ mod tests { 1500 ); assert_eq!( - get_cgroup_v2_cpu_limit(Path::new("testdata").join("cpu.max.unlimited")).unwrap(), - MAX_VALUE + get_cgroup_v2_cpu_limit(Path::new("testdata").join("cpu.max.unlimited")), + None ); assert_eq!( get_cgroup_v2_cpu_limit(Path::new("non_existent_file")), diff --git a/src/common/stat/src/lib.rs b/src/common/stat/src/lib.rs index 14d1f90c1df5..2c6cbea3f186 100644 --- a/src/common/stat/src/lib.rs +++ b/src/common/stat/src/lib.rs @@ -15,3 +15,64 @@ mod cgroups; pub use cgroups::*; +use common_base::readable_size::ReadableSize; +use sysinfo::System; + +/// Get the total CPU in millicores. +pub fn get_total_cpu_millicores() -> i64 { + // Get CPU limit from cgroups filesystem. + if let Some(cgroup_cpu_limit) = get_cpu_limit_from_cgroups() { + cgroup_cpu_limit + } else { + // Get total CPU cores from host system. + num_cpus::get() as i64 * 1000 + } +} + +/// Get the total memory in bytes. +pub fn get_total_memory_bytes() -> i64 { + // Get memory limit from cgroups filesystem. + if let Some(cgroup_memory_limit) = get_memory_limit_from_cgroups() { + cgroup_memory_limit + } else { + // Get total memory from host system. + if sysinfo::IS_SUPPORTED_SYSTEM { + let mut sys_info = System::new(); + sys_info.refresh_memory(); + sys_info.total_memory() as i64 + } else { + // If the system is not supported, return -1. + -1 + } + } +} + +/// Get the total CPU cores. The result will be rounded to the nearest integer. +/// For example, if the total CPU is 1.5 cores(1500 millicores), the result will be 2. +pub fn get_total_cpu_cores() -> usize { + ((get_total_cpu_millicores() as f64) / 1000.0).round() as usize +} + +/// Get the total memory in readable size. +pub fn get_total_memory_readable() -> Option { + if get_total_memory_bytes() > 0 { + Some(ReadableSize(get_total_memory_bytes() as u64)) + } else { + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_get_total_cpu_cores() { + assert!(get_total_cpu_cores() > 0); + } + + #[test] + fn test_get_total_memory_readable() { + assert!(get_total_memory_readable().unwrap() > ReadableSize::mb(0)); + } +} diff --git a/src/flow/Cargo.toml b/src/flow/Cargo.toml index 9e0204d91736..07137a9aa564 100644 --- a/src/flow/Cargo.toml +++ b/src/flow/Cargo.toml @@ -32,6 +32,7 @@ common-options.workspace = true common-query.workspace = true common-recordbatch.workspace = true common-runtime.workspace = true +common-stat.workspace = true common-telemetry.workspace = true common-time.workspace = true common-version.workspace = true diff --git a/src/flow/src/adapter.rs b/src/flow/src/adapter.rs index 8750e4f40837..9721d490405f 100644 --- a/src/flow/src/adapter.rs +++ b/src/flow/src/adapter.rs @@ -26,6 +26,7 @@ use common_error::ext::BoxedError; use common_meta::key::TableMetadataManagerRef; use common_options::memory::MemoryOptions; use common_runtime::JoinHandle; +use common_stat::get_total_cpu_cores; use common_telemetry::logging::{LoggingOptions, TracingOptions}; use common_telemetry::{debug, info, trace}; use datatypes::schema::ColumnSchema; @@ -92,7 +93,7 @@ pub struct FlowConfig { impl Default for FlowConfig { fn default() -> Self { Self { - num_workers: (common_config::utils::get_cpus() / 2).max(1), + num_workers: (get_total_cpu_cores() / 2).max(1), batching_mode: BatchingModeOptions::default(), } } @@ -141,7 +142,7 @@ impl Default for FlownodeOptions { impl Configurable for FlownodeOptions { fn validate_sanitize(&mut self) -> common_config::error::Result<()> { if self.flow.num_workers == 0 { - self.flow.num_workers = (common_config::utils::get_cpus() / 2).max(1); + self.flow.num_workers = (get_total_cpu_cores() / 2).max(1); } Ok(()) } diff --git a/src/mito2/Cargo.toml b/src/mito2/Cargo.toml index 380913e25f07..6e575d311029 100644 --- a/src/mito2/Cargo.toml +++ b/src/mito2/Cargo.toml @@ -16,6 +16,7 @@ workspace = true api.workspace = true aquamarine.workspace = true async-channel = "1.9" +common-stat.workspace = true async-stream.workspace = true async-trait.workspace = true bytemuck.workspace = true diff --git a/src/mito2/src/config.rs b/src/mito2/src/config.rs index 966edfdf0012..7a956dffb2b2 100644 --- a/src/mito2/src/config.rs +++ b/src/mito2/src/config.rs @@ -19,6 +19,7 @@ use std::path::Path; use std::time::Duration; use common_base::readable_size::ReadableSize; +use common_stat::{get_total_cpu_cores, get_total_memory_readable}; use common_telemetry::warn; use serde::{Deserialize, Serialize}; use serde_with::serde_as; @@ -159,7 +160,7 @@ impl Default for MitoConfig { compress_manifest: false, max_background_flushes: divide_num_cpus(2), max_background_compactions: divide_num_cpus(4), - max_background_purges: common_config::utils::get_cpus(), + max_background_purges: get_total_cpu_cores(), auto_flush_interval: Duration::from_secs(30 * 60), global_write_buffer_size: ReadableSize::gb(1), global_write_buffer_reject_size: ReadableSize::gb(2), @@ -185,7 +186,7 @@ impl Default for MitoConfig { }; // Adjust buffer and cache size according to system memory if we can. - if let Some(sys_memory) = common_config::utils::get_sys_total_memory() { + if let Some(sys_memory) = get_total_memory_readable() { mito_config.adjust_buffer_and_cache_size(sys_memory); } @@ -224,11 +225,9 @@ impl MitoConfig { self.max_background_compactions = divide_num_cpus(4); } if self.max_background_purges == 0 { - warn!( - "Sanitize max background purges 0 to {}", - common_config::utils::get_cpus() - ); - self.max_background_purges = common_config::utils::get_cpus(); + let cpu_cores = get_total_cpu_cores(); + warn!("Sanitize max background purges 0 to {}", cpu_cores); + self.max_background_purges = cpu_cores; } if self.global_write_buffer_reject_size <= self.global_write_buffer_size { @@ -486,7 +485,7 @@ impl InvertedIndexConfig { pub fn mem_threshold_on_create(&self) -> Option { match self.mem_threshold_on_create { MemoryThreshold::Auto => { - if let Some(sys_memory) = common_config::utils::get_sys_total_memory() { + if let Some(sys_memory) = get_total_memory_readable() { Some((sys_memory / INDEX_CREATE_MEM_THRESHOLD_FACTOR).as_bytes() as usize) } else { Some(ReadableSize::mb(64).as_bytes() as usize) @@ -531,7 +530,7 @@ impl FulltextIndexConfig { pub fn mem_threshold_on_create(&self) -> usize { match self.mem_threshold_on_create { MemoryThreshold::Auto => { - if let Some(sys_memory) = common_config::utils::get_sys_total_memory() { + if let Some(sys_memory) = get_total_memory_readable() { (sys_memory / INDEX_CREATE_MEM_THRESHOLD_FACTOR).as_bytes() as _ } else { ReadableSize::mb(64).as_bytes() as _ @@ -573,7 +572,7 @@ impl BloomFilterConfig { pub fn mem_threshold_on_create(&self) -> Option { match self.mem_threshold_on_create { MemoryThreshold::Auto => { - if let Some(sys_memory) = common_config::utils::get_sys_total_memory() { + if let Some(sys_memory) = get_total_memory_readable() { Some((sys_memory / INDEX_CREATE_MEM_THRESHOLD_FACTOR).as_bytes() as usize) } else { Some(ReadableSize::mb(64).as_bytes() as usize) @@ -588,7 +587,7 @@ impl BloomFilterConfig { /// Divide cpu num by a non-zero `divisor` and returns at least 1. fn divide_num_cpus(divisor: usize) -> usize { debug_assert!(divisor > 0); - let cores = common_config::utils::get_cpus(); + let cores = get_total_cpu_cores(); debug_assert!(cores > 0); cores.div_ceil(divisor) diff --git a/src/mito2/src/memtable/partition_tree.rs b/src/mito2/src/memtable/partition_tree.rs index d20c51a13761..a84e0641f962 100644 --- a/src/mito2/src/memtable/partition_tree.rs +++ b/src/mito2/src/memtable/partition_tree.rs @@ -28,6 +28,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicI64, AtomicU64, AtomicUsize, Ordering}; use common_base::readable_size::ReadableSize; +use common_stat::get_total_memory_readable; use mito_codec::key_values::KeyValue; use mito_codec::row_converter::{PrimaryKeyCodec, build_primary_key_codec}; use serde::{Deserialize, Serialize}; @@ -91,9 +92,9 @@ pub struct PartitionTreeConfig { impl Default for PartitionTreeConfig { fn default() -> Self { let mut fork_dictionary_bytes = ReadableSize::mb(512); - if let Some(sys_memory) = common_config::utils::get_sys_total_memory() { + if let Some(total_memory) = get_total_memory_readable() { let adjust_dictionary_bytes = - std::cmp::min(sys_memory / DICTIONARY_SIZE_FACTOR, fork_dictionary_bytes); + std::cmp::min(total_memory / DICTIONARY_SIZE_FACTOR, fork_dictionary_bytes); if adjust_dictionary_bytes.0 > 0 { fork_dictionary_bytes = adjust_dictionary_bytes; } diff --git a/src/mito2/src/region/options.rs b/src/mito2/src/region/options.rs index 1c075bf6b484..0592af877b9e 100644 --- a/src/mito2/src/region/options.rs +++ b/src/mito2/src/region/options.rs @@ -20,6 +20,7 @@ use std::collections::HashMap; use std::time::Duration; use common_base::readable_size::ReadableSize; +use common_stat::get_total_memory_readable; use common_time::TimeToLive; use common_wal::options::{WAL_OPTIONS_KEY, WalOptions}; use serde::de::Error as _; @@ -347,9 +348,9 @@ pub struct PartitionTreeOptions { impl Default for PartitionTreeOptions { fn default() -> Self { let mut fork_dictionary_bytes = ReadableSize::mb(512); - if let Some(sys_memory) = common_config::utils::get_sys_total_memory() { + if let Some(total_memory) = get_total_memory_readable() { let adjust_dictionary_bytes = std::cmp::min( - sys_memory / crate::memtable::partition_tree::DICTIONARY_SIZE_FACTOR, + total_memory / crate::memtable::partition_tree::DICTIONARY_SIZE_FACTOR, fork_dictionary_bytes, ); if adjust_dictionary_bytes.0 > 0 { diff --git a/src/standalone/Cargo.toml b/src/standalone/Cargo.toml index 035b1dc06a54..4d4722a00200 100644 --- a/src/standalone/Cargo.toml +++ b/src/standalone/Cargo.toml @@ -19,6 +19,7 @@ common-meta.workspace = true common-options.workspace = true common-procedure.workspace = true common-query.workspace = true +common-stat.workspace = true common-telemetry.workspace = true common-time.workspace = true common-version.workspace = true diff --git a/src/standalone/src/information_extension.rs b/src/standalone/src/information_extension.rs index 347955cfab85..cabea19d174b 100644 --- a/src/standalone/src/information_extension.rs +++ b/src/standalone/src/information_extension.rs @@ -75,10 +75,8 @@ impl InformationExtension for StandaloneInformationExtension { // Use `self.start_time_ms` instead. // It's not precise but enough. start_time_ms: self.start_time_ms, - cpus: common_config::utils::get_cpus() as u32, - memory_bytes: common_config::utils::get_sys_total_memory() - .unwrap_or_default() - .as_bytes(), + cpus: common_stat::get_total_cpu_millicores() as u32, + memory_bytes: common_stat::get_total_memory_bytes() as u64, }; Ok(vec![node_info]) } diff --git a/tests/cases/distributed/information_schema/cluster_info.result b/tests/cases/distributed/information_schema/cluster_info.result index 76875e564102..abb29b36a49e 100644 --- a/tests/cases/distributed/information_schema/cluster_info.result +++ b/tests/cases/distributed/information_schema/cluster_info.result @@ -4,21 +4,21 @@ Affected Rows: 0 DESC TABLE CLUSTER_INFO; -+--------------+----------------------+-----+------+---------+---------------+ -| Column | Type | Key | Null | Default | Semantic Type | -+--------------+----------------------+-----+------+---------+---------------+ -| peer_id | Int64 | | NO | | FIELD | -| peer_type | String | | NO | | FIELD | -| peer_addr | String | | YES | | FIELD | -| cpus | UInt32 | | NO | | FIELD | -| memory_bytes | UInt64 | | NO | | FIELD | -| version | String | | NO | | FIELD | -| git_commit | String | | NO | | FIELD | -| start_time | TimestampMillisecond | | YES | | FIELD | -| uptime | String | | YES | | FIELD | -| active_time | String | | YES | | FIELD | -| node_status | String | | YES | | FIELD | -+--------------+----------------------+-----+------+---------+---------------+ ++----------------------+----------------------+------+------+---------+---------------+ +| Column | Type | Key | Null | Default | Semantic Type | ++----------------------+----------------------+------+------+---------+---------------+ +| peer_id | Int64 | | NO | | FIELD | +| peer_type | String | | NO | | FIELD | +| peer_addr | String | | YES | | FIELD | +| total_cpu_millicores | UInt32 | | NO | | FIELD | +| total_memory_bytes | UInt64 | | NO | | FIELD | +| version | String | | NO | | FIELD | +| git_commit | String | | NO | | FIELD | +| start_time | TimestampMillisecond | | YES | | FIELD | +| uptime | String | | YES | | FIELD | +| active_time | String | | YES | | FIELD | +| node_status | String | | YES | | FIELD | ++----------------------+----------------------+------+------+---------+---------------+ -- SQLNESS REPLACE version node_version -- SQLNESS REPLACE (\s\d+\.\d+(?:\.\d+)+\s) Version @@ -86,18 +86,18 @@ SELECT peer_id, node_status FROM CLUSTER_INFO WHERE PEER_TYPE = 'DATANODE' ORDER | 2 | {"workloads"PLACEHOLDER,"leader_regions"PLACEHOLDER,"follower_regions"PLACEHOLDER} | +---------+------------------------------------------------------------------+ -SELECT peer_type, cpus!=0, memory_bytes!=0 FROM CLUSTER_INFO ORDER BY peer_type; - -+-----------+-------------------------------+---------------------------------------+ -| peer_type | cluster_info.cpus != Int64(0) | cluster_info.memory_bytes != Int64(0) | -+-----------+-------------------------------+---------------------------------------+ -| DATANODE | true | true | -| DATANODE | true | true | -| DATANODE | true | true | -| FLOWNODE | true | true | -| FRONTEND | true | true | -| METASRV | true | true | -+-----------+-------------------------------+---------------------------------------+ +SELECT peer_type, total_cpu_millicores!=0, total_memory_bytes!=0 FROM CLUSTER_INFO ORDER BY peer_type; + ++-----------+-----------------------------------------------+---------------------------------------------+ +| peer_type | cluster_info.total_cpu_millicores != Int64(0) | cluster_info.total_memory_bytes != Int64(0) | ++-----------+-----------------------------------------------+---------------------------------------------+ +| DATANODE | true | true | +| DATANODE | true | true | +| DATANODE | true | true | +| FLOWNODE | true | true | +| FRONTEND | true | true | +| METASRV | true | true | ++-----------+-----------------------------------------------+---------------------------------------------+ USE PUBLIC; diff --git a/tests/cases/distributed/information_schema/cluster_info.sql b/tests/cases/distributed/information_schema/cluster_info.sql index 1bd9490bc1f4..61822b94515e 100644 --- a/tests/cases/distributed/information_schema/cluster_info.sql +++ b/tests/cases/distributed/information_schema/cluster_info.sql @@ -50,6 +50,6 @@ SELECT peer_id, peer_type, peer_addr, version, git_commit, start_time, uptime, a -- SQLNESS REPLACE (:\s*(\".*?\"|\[.*?\]|\{.*?\}|[0-9]+|true|false|null)) PLACEHOLDER SELECT peer_id, node_status FROM CLUSTER_INFO WHERE PEER_TYPE = 'DATANODE' ORDER BY peer_id; -SELECT peer_type, cpus!=0, memory_bytes!=0 FROM CLUSTER_INFO ORDER BY peer_type; +SELECT peer_type, total_cpu_millicores!=0, total_memory_bytes!=0 FROM CLUSTER_INFO ORDER BY peer_type; USE PUBLIC; diff --git a/tests/cases/standalone/common/system/information_schema.result b/tests/cases/standalone/common/system/information_schema.result index 8d24c33b77bc..eb90035a782e 100644 --- a/tests/cases/standalone/common/system/information_schema.result +++ b/tests/cases/standalone/common/system/information_schema.result @@ -72,9 +72,9 @@ select * from information_schema.columns order by table_schema, table_name, colu | greptime | information_schema | check_constraints | constraint_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | | greptime | information_schema | check_constraints | constraint_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | | greptime | information_schema | cluster_info | active_time | 10 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | cluster_info | cpus | 4 | | | 10 | 0 | | | | | | select,insert | | UInt32 | int unsigned | FIELD | | No | int unsigned | | | +| greptime | information_schema | cluster_info | total_cpu_millicores | 4 | | | 10 | 0 | | | | | | select,insert | | UInt32 | int unsigned | FIELD | | No | int unsigned | | | | greptime | information_schema | cluster_info | git_commit | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | cluster_info | memory_bytes | 5 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | No | bigint unsigned | | | +| greptime | information_schema | cluster_info | total_memory_bytes | 5 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | No | bigint unsigned | | | | greptime | information_schema | cluster_info | node_status | 11 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | | greptime | information_schema | cluster_info | peer_addr | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | | greptime | information_schema | cluster_info | peer_id | 1 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | diff --git a/tests/cases/standalone/information_schema/cluster_info.result b/tests/cases/standalone/information_schema/cluster_info.result index 674d687cac79..e28aee45adb7 100644 --- a/tests/cases/standalone/information_schema/cluster_info.result +++ b/tests/cases/standalone/information_schema/cluster_info.result @@ -4,21 +4,21 @@ Affected Rows: 0 DESC TABLE CLUSTER_INFO; -+--------------+----------------------+-----+------+---------+---------------+ -| Column | Type | Key | Null | Default | Semantic Type | -+--------------+----------------------+-----+------+---------+---------------+ -| peer_id | Int64 | | NO | | FIELD | -| peer_type | String | | NO | | FIELD | -| peer_addr | String | | YES | | FIELD | -| cpus | UInt32 | | NO | | FIELD | -| memory_bytes | UInt64 | | NO | | FIELD | -| version | String | | NO | | FIELD | -| git_commit | String | | NO | | FIELD | -| start_time | TimestampMillisecond | | YES | | FIELD | -| uptime | String | | YES | | FIELD | -| active_time | String | | YES | | FIELD | -| node_status | String | | YES | | FIELD | -+--------------+----------------------+-----+------+---------+---------------+ ++----------------------+----------------------+------+------+---------+---------------+ +| Column | Type | Key | Null | Default | Semantic Type | ++----------------------+----------------------+------+------+---------+---------------+ +| peer_id | Int64 | | NO | | FIELD | +| peer_type | String | | NO | | FIELD | +| peer_addr | String | | YES | | FIELD | +| total_cpu_millicores | UInt32 | | NO | | FIELD | +| total_memory_bytes | UInt64 | | NO | | FIELD | +| version | String | | NO | | FIELD | +| git_commit | String | | NO | | FIELD | +| start_time | TimestampMillisecond | | YES | | FIELD | +| uptime | String | | YES | | FIELD | +| active_time | String | | YES | | FIELD | +| node_status | String | | YES | | FIELD | ++----------------------+----------------------+------+------+---------+---------------+ -- SQLNESS REPLACE version node_version -- SQLNESS REPLACE (\d+\.\d+(?:\.\d+)+) Version @@ -60,13 +60,13 @@ SELECT peer_id, peer_type, peer_addr, version, git_commit, start_time, uptime, a ++ ++ -SELECT peer_type, cpus!=0, memory_bytes!=0 FROM CLUSTER_INFO ORDER BY peer_type; +SELECT peer_type, total_cpu_millicores!=0, total_memory_bytes!=0 FROM CLUSTER_INFO ORDER BY peer_type; -+------------+-------------------------------+---------------------------------------+ -| peer_type | cluster_info.cpus != Int64(0) | cluster_info.memory_bytes != Int64(0) | -+------------+-------------------------------+---------------------------------------+ -| STANDALONE | true | true | -+------------+-------------------------------+---------------------------------------+ ++------------+-----------------------------------------------+---------------------------------------------+ +| peer_type | cluster_info.total_cpu_millicores != Int64(0) | cluster_info.total_memory_bytes != Int64(0) | ++------------+-----------------------------------------------+---------------------------------------------+ +| STANDALONE | true | true | ++------------+-----------------------------------------------+---------------------------------------------+ USE PUBLIC; diff --git a/tests/cases/standalone/information_schema/cluster_info.sql b/tests/cases/standalone/information_schema/cluster_info.sql index 70fc16dd2050..7b134f118aee 100644 --- a/tests/cases/standalone/information_schema/cluster_info.sql +++ b/tests/cases/standalone/information_schema/cluster_info.sql @@ -30,6 +30,6 @@ SELECT peer_id, peer_type, peer_addr, version, git_commit, start_time, uptime, a SELECT peer_id, peer_type, peer_addr, version, git_commit, start_time, uptime, active_time FROM CLUSTER_INFO WHERE PEER_ID > 0; -SELECT peer_type, cpus!=0, memory_bytes!=0 FROM CLUSTER_INFO ORDER BY peer_type; +SELECT peer_type, total_cpu_millicores!=0, total_memory_bytes!=0 FROM CLUSTER_INFO ORDER BY peer_type; USE PUBLIC; From e064f6f92eeaf1d6676d49046975de7bdee09892 Mon Sep 17 00:00:00 2001 From: zyy17 Date: Fri, 3 Oct 2025 11:55:48 +0800 Subject: [PATCH 2/2] tests: update sqlness test cases Signed-off-by: zyy17 --- .../information_schema/cluster_info.result | 30 +++++++++---------- .../common/system/information_schema.result | 4 +-- .../information_schema/cluster_info.result | 30 +++++++++---------- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/tests/cases/distributed/information_schema/cluster_info.result b/tests/cases/distributed/information_schema/cluster_info.result index abb29b36a49e..3552a870e545 100644 --- a/tests/cases/distributed/information_schema/cluster_info.result +++ b/tests/cases/distributed/information_schema/cluster_info.result @@ -4,21 +4,21 @@ Affected Rows: 0 DESC TABLE CLUSTER_INFO; -+----------------------+----------------------+------+------+---------+---------------+ -| Column | Type | Key | Null | Default | Semantic Type | -+----------------------+----------------------+------+------+---------+---------------+ -| peer_id | Int64 | | NO | | FIELD | -| peer_type | String | | NO | | FIELD | -| peer_addr | String | | YES | | FIELD | -| total_cpu_millicores | UInt32 | | NO | | FIELD | -| total_memory_bytes | UInt64 | | NO | | FIELD | -| version | String | | NO | | FIELD | -| git_commit | String | | NO | | FIELD | -| start_time | TimestampMillisecond | | YES | | FIELD | -| uptime | String | | YES | | FIELD | -| active_time | String | | YES | | FIELD | -| node_status | String | | YES | | FIELD | -+----------------------+----------------------+------+------+---------+---------------+ ++----------------------+----------------------+-----+------+---------+---------------+ +| Column | Type | Key | Null | Default | Semantic Type | ++----------------------+----------------------+-----+------+---------+---------------+ +| peer_id | Int64 | | NO | | FIELD | +| peer_type | String | | NO | | FIELD | +| peer_addr | String | | YES | | FIELD | +| total_cpu_millicores | UInt32 | | NO | | FIELD | +| total_memory_bytes | UInt64 | | NO | | FIELD | +| version | String | | NO | | FIELD | +| git_commit | String | | NO | | FIELD | +| start_time | TimestampMillisecond | | YES | | FIELD | +| uptime | String | | YES | | FIELD | +| active_time | String | | YES | | FIELD | +| node_status | String | | YES | | FIELD | ++----------------------+----------------------+-----+------+---------+---------------+ -- SQLNESS REPLACE version node_version -- SQLNESS REPLACE (\s\d+\.\d+(?:\.\d+)+\s) Version diff --git a/tests/cases/standalone/common/system/information_schema.result b/tests/cases/standalone/common/system/information_schema.result index eb90035a782e..7db1845c89ad 100644 --- a/tests/cases/standalone/common/system/information_schema.result +++ b/tests/cases/standalone/common/system/information_schema.result @@ -72,14 +72,14 @@ select * from information_schema.columns order by table_schema, table_name, colu | greptime | information_schema | check_constraints | constraint_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | | greptime | information_schema | check_constraints | constraint_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | | greptime | information_schema | cluster_info | active_time | 10 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | -| greptime | information_schema | cluster_info | total_cpu_millicores | 4 | | | 10 | 0 | | | | | | select,insert | | UInt32 | int unsigned | FIELD | | No | int unsigned | | | | greptime | information_schema | cluster_info | git_commit | 7 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | -| greptime | information_schema | cluster_info | total_memory_bytes | 5 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | No | bigint unsigned | | | | greptime | information_schema | cluster_info | node_status | 11 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | | greptime | information_schema | cluster_info | peer_addr | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | | greptime | information_schema | cluster_info | peer_id | 1 | | | 19 | 0 | | | | | | select,insert | | Int64 | bigint | FIELD | | No | bigint | | | | greptime | information_schema | cluster_info | peer_type | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | | greptime | information_schema | cluster_info | start_time | 8 | | | | | 3 | | | | | select,insert | | TimestampMillisecond | timestamp(3) | FIELD | | Yes | timestamp(3) | | | +| greptime | information_schema | cluster_info | total_cpu_millicores | 4 | | | 10 | 0 | | | | | | select,insert | | UInt32 | int unsigned | FIELD | | No | int unsigned | | | +| greptime | information_schema | cluster_info | total_memory_bytes | 5 | | | 20 | 0 | | | | | | select,insert | | UInt64 | bigint unsigned | FIELD | | No | bigint unsigned | | | | greptime | information_schema | cluster_info | uptime | 9 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | | greptime | information_schema | cluster_info | version | 6 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | | greptime | information_schema | collation_character_set_applicability | character_set_name | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | diff --git a/tests/cases/standalone/information_schema/cluster_info.result b/tests/cases/standalone/information_schema/cluster_info.result index e28aee45adb7..fd13ab6b69f4 100644 --- a/tests/cases/standalone/information_schema/cluster_info.result +++ b/tests/cases/standalone/information_schema/cluster_info.result @@ -4,21 +4,21 @@ Affected Rows: 0 DESC TABLE CLUSTER_INFO; -+----------------------+----------------------+------+------+---------+---------------+ -| Column | Type | Key | Null | Default | Semantic Type | -+----------------------+----------------------+------+------+---------+---------------+ -| peer_id | Int64 | | NO | | FIELD | -| peer_type | String | | NO | | FIELD | -| peer_addr | String | | YES | | FIELD | -| total_cpu_millicores | UInt32 | | NO | | FIELD | -| total_memory_bytes | UInt64 | | NO | | FIELD | -| version | String | | NO | | FIELD | -| git_commit | String | | NO | | FIELD | -| start_time | TimestampMillisecond | | YES | | FIELD | -| uptime | String | | YES | | FIELD | -| active_time | String | | YES | | FIELD | -| node_status | String | | YES | | FIELD | -+----------------------+----------------------+------+------+---------+---------------+ ++----------------------+----------------------+-----+------+---------+---------------+ +| Column | Type | Key | Null | Default | Semantic Type | ++----------------------+----------------------+-----+------+---------+---------------+ +| peer_id | Int64 | | NO | | FIELD | +| peer_type | String | | NO | | FIELD | +| peer_addr | String | | YES | | FIELD | +| total_cpu_millicores | UInt32 | | NO | | FIELD | +| total_memory_bytes | UInt64 | | NO | | FIELD | +| version | String | | NO | | FIELD | +| git_commit | String | | NO | | FIELD | +| start_time | TimestampMillisecond | | YES | | FIELD | +| uptime | String | | YES | | FIELD | +| active_time | String | | YES | | FIELD | +| node_status | String | | YES | | FIELD | ++----------------------+----------------------+-----+------+---------+---------------+ -- SQLNESS REPLACE version node_version -- SQLNESS REPLACE (\d+\.\d+(?:\.\d+)+) Version