diff --git a/Cargo.lock b/Cargo.lock index 45d242094d5c..b3c168ebd46f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2323,7 +2323,7 @@ dependencies = [ "futures-util", "serde", "snafu 0.8.5", - "sqlparser 0.52.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=71dd86058d2af97b9925093d40c4e03360403170)", + "sqlparser 0.52.0 (git+https://github.com/NiwakaDev/sqlparser-rs?rev=daf8c33234f1b4568824b32efcc5d611091c4834)", "sqlparser_derive 0.1.1", "statrs", "store-api", @@ -3463,7 +3463,7 @@ dependencies = [ "serde", "serde_json", "snafu 0.8.5", - "sqlparser 0.52.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=71dd86058d2af97b9925093d40c4e03360403170)", + "sqlparser 0.52.0 (git+https://github.com/NiwakaDev/sqlparser-rs?rev=daf8c33234f1b4568824b32efcc5d611091c4834)", "sqlparser_derive 0.1.1", ] @@ -4359,7 +4359,7 @@ dependencies = [ "session", "snafu 0.8.5", "sql", - "sqlparser 0.52.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=71dd86058d2af97b9925093d40c4e03360403170)", + "sqlparser 0.52.0 (git+https://github.com/NiwakaDev/sqlparser-rs?rev=daf8c33234f1b4568824b32efcc5d611091c4834)", "store-api", "strfmt", "table", @@ -7864,7 +7864,7 @@ dependencies = [ "session", "snafu 0.8.5", "sql", - "sqlparser 0.52.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=71dd86058d2af97b9925093d40c4e03360403170)", + "sqlparser 0.52.0 (git+https://github.com/NiwakaDev/sqlparser-rs?rev=daf8c33234f1b4568824b32efcc5d611091c4834)", "store-api", "substrait 0.12.0", "table", @@ -8119,7 +8119,7 @@ dependencies = [ "serde_json", "snafu 0.8.5", "sql", - "sqlparser 0.52.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=71dd86058d2af97b9925093d40c4e03360403170)", + "sqlparser 0.52.0 (git+https://github.com/NiwakaDev/sqlparser-rs?rev=daf8c33234f1b4568824b32efcc5d611091c4834)", "store-api", "table", ] @@ -9122,7 +9122,7 @@ dependencies = [ "session", "snafu 0.8.5", "sql", - "sqlparser 0.52.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=71dd86058d2af97b9925093d40c4e03360403170)", + "sqlparser 0.52.0 (git+https://github.com/NiwakaDev/sqlparser-rs?rev=daf8c33234f1b4568824b32efcc5d611091c4834)", "statrs", "store-api", "substrait 0.12.0", @@ -10921,7 +10921,7 @@ dependencies = [ "serde", "serde_json", "snafu 0.8.5", - "sqlparser 0.52.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=71dd86058d2af97b9925093d40c4e03360403170)", + "sqlparser 0.52.0 (git+https://github.com/NiwakaDev/sqlparser-rs?rev=daf8c33234f1b4568824b32efcc5d611091c4834)", "sqlparser_derive 0.1.1", "store-api", "table", @@ -10987,14 +10987,14 @@ dependencies = [ [[package]] name = "sqlparser" version = "0.52.0" -source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=71dd86058d2af97b9925093d40c4e03360403170#71dd86058d2af97b9925093d40c4e03360403170" +source = "git+https://github.com/NiwakaDev/sqlparser-rs?rev=daf8c33234f1b4568824b32efcc5d611091c4834#daf8c33234f1b4568824b32efcc5d611091c4834" dependencies = [ "lazy_static", "log", "regex", "serde", "sqlparser 0.52.0 (registry+https://github.com/rust-lang/crates.io-index)", - "sqlparser_derive 0.2.2 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=71dd86058d2af97b9925093d40c4e03360403170)", + "sqlparser_derive 0.2.2 (git+https://github.com/NiwakaDev/sqlparser-rs?rev=daf8c33234f1b4568824b32efcc5d611091c4834)", ] [[package]] @@ -11022,7 +11022,7 @@ dependencies = [ [[package]] name = "sqlparser_derive" version = "0.2.2" -source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=71dd86058d2af97b9925093d40c4e03360403170#71dd86058d2af97b9925093d40c4e03360403170" +source = "git+https://github.com/NiwakaDev/sqlparser-rs?rev=daf8c33234f1b4568824b32efcc5d611091c4834#daf8c33234f1b4568824b32efcc5d611091c4834" dependencies = [ "proc-macro2", "quote", @@ -11859,7 +11859,7 @@ dependencies = [ "serde_yaml", "snafu 0.8.5", "sql", - "sqlparser 0.52.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=71dd86058d2af97b9925093d40c4e03360403170)", + "sqlparser 0.52.0 (git+https://github.com/NiwakaDev/sqlparser-rs?rev=daf8c33234f1b4568824b32efcc5d611091c4834)", "sqlx", "store-api", "strum 0.25.0", diff --git a/Cargo.toml b/Cargo.toml index b53f827e3ec8..db1aec809692 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -188,7 +188,7 @@ smallvec = { version = "1", features = ["serde"] } snafu = "0.8" sysinfo = "0.30" # on branch v0.52.x -sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "71dd86058d2af97b9925093d40c4e03360403170", features = [ +sqlparser = { git = "https://github.com/NiwakaDev/sqlparser-rs", rev = "daf8c33234f1b4568824b32efcc5d611091c4834", features = [ "visitor", "serde", ] } # on branch v0.44.x diff --git a/src/common/function/src/scalars/math.rs b/src/common/function/src/scalars/math.rs index 6635e70b171f..ba0fd7b66c15 100644 --- a/src/common/function/src/scalars/math.rs +++ b/src/common/function/src/scalars/math.rs @@ -44,6 +44,7 @@ impl MathFunction { registry.register(Arc::new(RateFunction)); registry.register(Arc::new(RangeFunction)); registry.register(Arc::new(ClampFunction)); + registry.register(Arc::new(WithinFilterFunction)); } } @@ -87,3 +88,39 @@ impl Function for RangeFunction { .context(GeneralDataFusionSnafu) } } + +#[derive(Clone, Debug, Default)] +struct WithinFilterFunction; + +impl fmt::Display for WithinFilterFunction { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "WithinFilterFunction") + } +} + +pub const WITHIN_FILTER_NAME: &str = "within_filter"; + +impl Function for WithinFilterFunction { + fn name(&self) -> &str { + WITHIN_FILTER_NAME + } + + fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result { + Ok(ConcreteDataType::boolean_datatype()) + } + + fn signature(&self) -> Signature { + Signature::uniform( + 2, + vec![ConcreteDataType::string_datatype()], + Volatility::Immutable, + ) + } + + fn eval(&self, _func_ctx: FunctionContext, _columns: &[VectorRef]) -> Result { + Err(DataFusionError::Internal( + "within_filter function just a empty function, it should not be eval!".into(), + )) + .context(GeneralDataFusionSnafu) + } +} diff --git a/src/query/src/dist_plan/merge_scan.rs b/src/query/src/dist_plan/merge_scan.rs index 47a951258312..040158aa4d6c 100644 --- a/src/query/src/dist_plan/merge_scan.rs +++ b/src/query/src/dist_plan/merge_scan.rs @@ -198,7 +198,6 @@ impl MergeScanExec { let dbname = context.task_id().unwrap_or_default(); let tracing_context = TracingContext::from_json(context.session_id().as_str()); let current_channel = self.query_ctx.channel(); - let stream = Box::pin(stream!({ // only report metrics once for each MergeScan if partition == 0 { diff --git a/src/query/src/error.rs b/src/query/src/error.rs index e696008cf546..281edd3e859d 100644 --- a/src/query/src/error.rs +++ b/src/query/src/error.rs @@ -323,6 +323,13 @@ pub enum Error { #[snafu(implicit)] location: Location, }, + + #[snafu(display("Within filter interval error: {}", message))] + WithinFilterInternal { + message: String, + #[snafu(implicit)] + location: Location, + }, } impl ErrorExt for Error { @@ -370,7 +377,9 @@ impl ErrorExt for Error { RegionQuery { source, .. } => source.status_code(), TableMutation { source, .. } => source.status_code(), - MissingTableMutationHandler { .. } => StatusCode::Unexpected, + WithinFilterInternal { .. } | MissingTableMutationHandler { .. } => { + StatusCode::Unexpected + } GetRegionMetadata { .. } => StatusCode::RegionNotReady, TableReadOnly { .. } => StatusCode::Unsupported, GetFulltextOptions { source, .. } | GetSkippingIndexOptions { source, .. } => { diff --git a/src/query/src/lib.rs b/src/query/src/lib.rs index 6e1fbfae0af8..d69b1baf5dd7 100644 --- a/src/query/src/lib.rs +++ b/src/query/src/lib.rs @@ -41,6 +41,7 @@ pub mod region_query; pub mod sql; pub mod stats; pub(crate) mod window_sort; +mod within_filter; #[cfg(test)] pub(crate) mod test_util; diff --git a/src/query/src/optimizer/windowed_sort.rs b/src/query/src/optimizer/windowed_sort.rs index 49cde9783b3a..2231b8ca066c 100644 --- a/src/query/src/optimizer/windowed_sort.rs +++ b/src/query/src/optimizer/windowed_sort.rs @@ -174,7 +174,6 @@ fn fetch_partition_range(input: Arc) -> DataFusionResult> + let mut extension_rules: Vec< + Arc<(dyn ExtensionAnalyzerRule + std::marker::Send + Sync + 'static)>, + > = Vec::new(); // The [`TypeConversionRule`] must be at first extension_rules.insert(0, Arc::new(TypeConversionRule) as _); + extension_rules.push(Arc::new(WithinFilterRule::new())); // Apply the datafusion rules let mut analyzer = Analyzer::new(); diff --git a/src/query/src/within_filter.rs b/src/query/src/within_filter.rs new file mode 100644 index 000000000000..9bf749b9b5be --- /dev/null +++ b/src/query/src/within_filter.rs @@ -0,0 +1,359 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use chrono::format::{Parsed, StrftimeItems}; +use chrono::{Datelike, Duration, NaiveDate, NaiveDateTime}; +use common_function::scalars::math::WITHIN_FILTER_NAME; +use common_time::timestamp::{TimeUnit, Timestamp}; +use datafusion::config::ConfigOptions; +use datafusion_common::tree_node::{Transformed, TreeNode}; +use datafusion_common::{DataFusionError, Result as DFResult, ScalarValue}; +use datafusion_expr::{BinaryExpr, Expr, Filter, LogicalPlan, Operator}; +use snafu::ensure; + +use crate::error::WithinFilterInternalSnafu; +use crate::optimizer::ExtensionAnalyzerRule; +use crate::QueryEngineContext; + +/// `WithinFilterRule` is an analyzer rule for DataFusion that converts the `within(ts, timestamp)` +/// function to a range +/// expression like `ts >= start AND ts < end`. +/// +/// # Purpose +/// This rule simplifies time-based queries for users by letting them specify +/// a single timestamp like `'2025-04-19'` while the filter +/// is converted to: +/// ```sql +/// ts >= '2025-04-19 00:00:00' AND ts < '2025-04-20 00:00:00' +/// ``` +/// Instead of writing these conditions manually, users can simply use the +/// `WITHIN` syntax. +/// +/// # How It Works +/// 1. The rule analyzer detects `within(ts, timestamp)` functions in a `LogicalPlan`. +/// 2. It infers the precision of the given timestamp (year, month, day, second, etc.). +/// 3. Based on that precision, the rule calculates the appropriate start and end timestamps (e.g., `start = 2025-04-19 23:50:00`, `end = 2025-04-19 23:51:00`). +/// 4. The `within` function is converted to a range expression like `ts >= start AND ts < end`. +/// +/// # Examples +/// - convert `ts WITHIN '2025'` +/// to: +/// ```sql +/// ts >= '2025-01-01 00:00:00' AND ts < '2026-01-01 00:00:00' +/// ``` +/// - convert `ts WITHIN '2025-04-19'` +/// to: +/// ```sql +/// ts >= '2025-04-19 00:00:00' AND ts < '2025-04-20 00:00:00' +/// ``` +/// - convert WITHIN '2025-04-19 23:50'` +/// to: +/// ```sql +/// ts >= '2025-04-19 23:50:00' AND ts < '2025-04-19 23:51:00' +/// ``` +pub struct WithinFilterRule {} + +impl WithinFilterRule { + pub fn new() -> Self { + WithinFilterRule {} + } +} + +impl ExtensionAnalyzerRule for WithinFilterRule { + fn analyze( + &self, + plan: LogicalPlan, + _ctx: &QueryEngineContext, + _config: &ConfigOptions, + ) -> DFResult { + plan.transform(|plan| match plan.clone() { + LogicalPlan::Filter(filter) => { + if let Expr::ScalarFunction(func) = &filter.predicate + && func.func.name() == WITHIN_FILTER_NAME + { + ensure!( + func.args.len() == 2, + WithinFilterInternalSnafu { + message: "expected 2 arguments", + } + ); + let column_name = func.args[0].clone(); + let time_arg = func.args[1].clone(); + if let Expr::Literal(literal) = time_arg + && let ScalarValue::Utf8(Some(s)) = literal + { + if let Some((start, end)) = try_to_infer_time_range(&s) { + return Ok(Transformed::yes(convert_plan( + filter.input, + &column_name, + start, + end, + )?)); + } + } + Err(DataFusionError::Plan( + "Failed to convert within filter to normal filter.".to_string(), + )) + } else { + Ok(Transformed::no(plan)) + } + } + _ => Ok(Transformed::no(plan)), + }) + .map(|t| t.data) + } +} + +/// Infers the time range from a given timestamp string. +fn try_to_infer_time_range(timestamp: &str) -> Option<(Timestamp, Timestamp)> { + fn try_parse_year(s: &str) -> Option { + let mut parsed = Parsed::new(); + if chrono::format::parse(&mut parsed, s, StrftimeItems::new("%Y")).is_err() { + return None; + } + parsed.set_month(1).unwrap(); + parsed.set_day(1).unwrap(); + Some(parsed.to_naive_date().unwrap()) + } + fn try_parse_month(s: &str) -> Option { + let mut parsed = Parsed::new(); + if chrono::format::parse(&mut parsed, s, StrftimeItems::new("%Y-%m")).is_err() { + return None; + } + parsed.set_day(1).unwrap(); + Some(parsed.to_naive_date().unwrap()) + } + fn try_parse_hour(s: &str) -> Option { + let mut parsed = Parsed::new(); + if chrono::format::parse(&mut parsed, s, StrftimeItems::new("%Y-%m-%dT%H")).is_err() { + return None; + } + parsed.set_minute(0).unwrap(); + Some(parsed.to_naive_datetime_with_offset(0).unwrap()) + } + if let Some(naive_date) = try_parse_year(timestamp) { + let start = Timestamp::from_chrono_date(naive_date).unwrap(); + let end = NaiveDate::from_ymd_opt(naive_date.year() + 1, 1, 1).unwrap(); + let end = Timestamp::from_chrono_date(end).unwrap(); + return Some((start, end)); + } + if let Ok(naive_date) = NaiveDate::parse_from_str(timestamp, "%Y-%m-%d") { + let start = Timestamp::from_chrono_date(naive_date).unwrap(); + let end = naive_date + Duration::days(1); + let end = Timestamp::from_chrono_date(end).unwrap(); + return Some((start, end)); + } + if let Some(naive_date) = try_parse_month(timestamp) { + let start = Timestamp::from_chrono_date(naive_date).unwrap(); + let end = if naive_date.month() == 12 { + NaiveDate::from_ymd_opt(naive_date.year() + 1, 1, 1).unwrap() + } else { + NaiveDate::from_ymd_opt(naive_date.year(), naive_date.month() + 1, 1).unwrap() + }; + let end = Timestamp::from_chrono_date(end).unwrap(); + return Some((start, end)); + } + if let Some(naive_date) = try_parse_hour(timestamp) { + let start = Timestamp::from_chrono_datetime(naive_date).unwrap(); + let end = naive_date + Duration::hours(1); + let end = Timestamp::from_chrono_datetime(end).unwrap(); + return Some((start, end)); + } + if let Ok(naive_date) = NaiveDateTime::parse_from_str(timestamp, "%Y-%m-%dT%H:%M") { + let end = naive_date + Duration::minutes(1); + let end = Timestamp::from_chrono_datetime(end).unwrap(); + let start = Timestamp::from_chrono_datetime(naive_date).unwrap(); + return Some((start, end)); + } + if let Ok(naive_date) = NaiveDateTime::parse_from_str(timestamp, "%Y-%m-%dT%H:%M:%S") { + let end = naive_date + Duration::seconds(1); + let end = Timestamp::from_chrono_datetime(end).unwrap(); + let start = Timestamp::from_chrono_datetime(naive_date).unwrap(); + return Some((start, end)); + } + None +} + +fn convert_plan( + input_plan: Arc, + column_name: &Expr, + start: Timestamp, + end: Timestamp, +) -> DFResult { + let value = Some(start.value()); + let start = match start.unit() { + TimeUnit::Second => ScalarValue::TimestampSecond(value, None), + TimeUnit::Millisecond => ScalarValue::TimestampMillisecond(value, None), + TimeUnit::Microsecond => ScalarValue::TimestampMicrosecond(value, None), + TimeUnit::Nanosecond => ScalarValue::TimestampNanosecond(value, None), + }; + let value = Some(end.value()); + let end = match end.unit() { + TimeUnit::Second => ScalarValue::TimestampSecond(value, None), + TimeUnit::Millisecond => ScalarValue::TimestampMillisecond(value, None), + TimeUnit::Microsecond => ScalarValue::TimestampMicrosecond(value, None), + TimeUnit::Nanosecond => ScalarValue::TimestampNanosecond(value, None), + }; + let left = Expr::BinaryExpr(BinaryExpr { + left: Box::new(column_name.clone()), + op: Operator::GtEq, + right: Box::new(Expr::Literal(start)), + }); + let right = Expr::BinaryExpr(BinaryExpr { + left: Box::new(column_name.clone()), + op: Operator::Lt, + right: Box::new(Expr::Literal(end)), + }); + let new_expr = Expr::BinaryExpr(BinaryExpr::new( + Box::new(left), + Operator::And, + Box::new(right), + )); + Ok(LogicalPlan::Filter(Filter::try_new(new_expr, input_plan)?)) +} + +#[cfg(test)] +mod test { + use std::sync::Arc; + + use catalog::memory::MemoryCatalogManager; + use catalog::RegisterTableRequest; + use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; + use datatypes::prelude::ConcreteDataType; + use datatypes::schema::{ColumnSchema, Schema}; + use session::context::QueryContext; + use table::metadata::{TableInfoBuilder, TableMetaBuilder}; + use table::test_util::EmptyTable; + + use super::*; + use crate::error::Result; + use crate::parser::QueryLanguageParser; + use crate::{QueryEngineFactory, QueryEngineRef}; + + async fn create_test_engine() -> QueryEngineRef { + let table_name = "test".to_string(); + let columns = vec![ + ColumnSchema::new( + "tag_1".to_string(), + ConcreteDataType::string_datatype(), + false, + ), + ColumnSchema::new( + "ts".to_string(), + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ) + .with_time_index(true), + ColumnSchema::new( + "field_1".to_string(), + ConcreteDataType::float64_datatype(), + true, + ), + ]; + let schema = Arc::new(Schema::new(columns)); + let table_meta = TableMetaBuilder::default() + .schema(schema) + .primary_key_indices(vec![0]) + .value_indices(vec![6]) + .next_column_id(1024) + .build() + .unwrap(); + let table_info = TableInfoBuilder::default() + .name(&table_name) + .meta(table_meta) + .build() + .unwrap(); + let table = EmptyTable::from_table_info(&table_info); + let catalog_list = MemoryCatalogManager::with_default_setup(); + assert!(catalog_list + .register_table_sync(RegisterTableRequest { + catalog: DEFAULT_CATALOG_NAME.to_string(), + schema: DEFAULT_SCHEMA_NAME.to_string(), + table_name, + table_id: 1024, + table, + }) + .is_ok()); + QueryEngineFactory::new(catalog_list, None, None, None, None, false).query_engine() + } + + async fn do_query(sql: &str) -> Result { + let stmt = QueryLanguageParser::parse_sql(sql, &QueryContext::arc()).unwrap(); + let engine = create_test_engine().await; + engine.planner().plan(&stmt, QueryContext::arc()).await + } + + #[tokio::test] + async fn test_within_filter() { + // TODO: test within filter with time zone + + // 2015-01-01T00:00:00 <= timestamp < 2016-01-01T00:00:00 + let sql = "SELECT * FROM test WHERE ts WITHIN '2015'"; + let plan = do_query(sql).await.unwrap(); + let expected = "Projection: *\ + \n Filter: test.ts >= TimestampSecond(1420070400, None) AND test.ts < TimestampSecond(1451606400, None)\ + \n TableScan: test"; + assert_eq!(expected, plan.to_string()); + + // 2025-03-01T00:00:00 <= timestamp < 2025-04-01T00:00:00 + let sql = "SELECT * FROM test WHERE ts WITHIN '2025-3'"; + let plan = do_query(sql).await.unwrap(); + let expected = "Projection: *\ + \n Filter: test.ts >= TimestampSecond(1740787200, None) AND test.ts < TimestampSecond(1743465600, None)\ + \n TableScan: test"; + assert_eq!(expected, plan.to_string()); + + // 2025-12-1T00:00:00 <= timestamp < 2026-01-01T00:00:00 + let sql = "SELECT * FROM test WHERE ts WITHIN '2025-12'"; + let plan = do_query(sql).await.unwrap(); + let expected = "Projection: *\ + \n Filter: test.ts >= TimestampSecond(1764547200, None) AND test.ts < TimestampSecond(1767225600, None)\ + \n TableScan: test"; + assert_eq!(expected, plan.to_string()); + + // 2025-12-1T00:00:00 <= timestamp < 2025-12-2T00:00:00 + let sql = "SELECT * FROM test WHERE ts WITHIN '2015-12-1'"; + let plan = do_query(sql).await.unwrap(); + let expected = "Projection: *\ + \n Filter: test.ts >= TimestampSecond(1448928000, None) AND test.ts < TimestampSecond(1449014400, None)\ + \n TableScan: test"; + assert_eq!(expected, plan.to_string()); + + // 2025-12-1T01:00:00 <= timestamp < 2025-12-1T02:00:00 + let sql = "SELECT * FROM test WHERE ts WITHIN '2025-12-1T01'"; + let plan = do_query(sql).await.unwrap(); + let expected = "Projection: *\ + \n Filter: test.ts >= TimestampSecond(1764550800, None) AND test.ts < TimestampSecond(1764554400, None)\ + \n TableScan: test"; + assert_eq!(expected, plan.to_string()); + + // 2025-12-1T01:12:00 <= timestamp < 2025-12-1T01:13:00 + let sql = "SELECT * FROM test WHERE ts WITHIN '2025-12-1T01:12'"; + let plan = do_query(sql).await.unwrap(); + let expected = "Projection: *\ + \n Filter: test.ts >= TimestampSecond(1764551520, None) AND test.ts < TimestampSecond(1764551580, None)\ + \n TableScan: test"; + assert_eq!(expected, plan.to_string()); + + // 2025-12-1T01:12:01 <= timestamp < 2025-12-1T01:12:02 + let sql = "SELECT * FROM test WHERE ts WITHIN '2025-12-1T01:12:01'"; + let plan = do_query(sql).await.unwrap(); + let expected = "Projection: *\ + \n Filter: test.ts >= TimestampSecond(1764551521, None) AND test.ts < TimestampSecond(1764551522, None)\ + \n TableScan: test"; + assert_eq!(expected, plan.to_string()); + } +} diff --git a/tests/cases/standalone/common/function/within_filter.result b/tests/cases/standalone/common/function/within_filter.result new file mode 100644 index 000000000000..bc9056e8220e --- /dev/null +++ b/tests/cases/standalone/common/function/within_filter.result @@ -0,0 +1,231 @@ +CREATE TABLE system_metrics ( + host STRING, + memory_util DOUBLE, + ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP(), + PRIMARY KEY(host), + TIME INDEX(ts) +); + +Affected Rows: 0 + +INSERT INTO system_metrics +VALUES + ("host1", 10.3, "2024-10-10T13:59:00Z"), + ("host2", 90.0, "2024-12-10T13:59:00Z"), + ("host2", 90.0, "2024-12-31T13:59:00Z"), + ("host1", 10.3, "2025-01-01T00:00:00Z"), + ("host2", 90.0, "2025-01-03T13:59:00Z"), + ("host1", 10.3, "2025-01-13T18:59:00Z"), + ("host2", 90.0, "2025-02-01T00:00:00Z"), + ("host2", 90.0, "2025-02-13T12:11:06Z"), + ("host1", 40.6, "2025-11-01T00:00:00Z"), + ("host1", 10.3, "2025-11-30T23:59:59Z"), + ("host2", 90.0, "2025-12-01T00:00:00Z"), + ("host2", 90.0, "2025-12-01T01:00:00Z"), + ("host2", 90.0, "2025-12-01T01:59:59Z"), + ("host2", 90.0, "2025-12-01T01:00:00Z"), + ("host2", 90.0, "2025-12-01T01:00:00Z"), + ("host2", 90.0, "2025-12-01T01:00:59Z"), + ("host2", 90.0, "2025-12-01T01:01:00Z"), + ("host1", 10.3, "2025-12-31T01:11:59Z"), + ("host1", 10.3, "2025-12-31T01:12:00Z"), + ("host2", 90.0, "2025-12-31T01:12:00Z"), + ("host1", 10.3, "2025-12-31T01:12:59Z"), + ("host1", 10.3, "2025-12-31T01:13:00Z"), + ("host2", 90.0, "2025-12-01T02:00:00Z"), + ("host2", 90.0, "2025-12-01T23:59:59Z"), + ("host2", 90.0, "2025-12-02T00:00:00Z"), + ("host2", 90.0, "2025-12-15T13:42:12Z"), + ("host1", 40.6, "2025-12-31T23:59:00Z"), + ("host1", 40.6, "2025-12-31T23:59:59Z"), + ("host1", 10.3, "2026-01-01T00:00:00Z"), + ("host2", 90.0, "2026-01-01T23:59:59Z"), + ("host2", 90.0, "2026-12-31T13:59:00Z"); + +Affected Rows: 31 + +SELECT * FROM system_metrics WHERE ts WITHIN '2023' ORDER BY host, ts; + +++ +++ + +SELECT * FROM system_metrics WHERE ts WITHIN '2024' ORDER BY host, ts; + ++-------+-------------+---------------------+ +| host | memory_util | ts | ++-------+-------------+---------------------+ +| host1 | 10.3 | 2024-10-10T13:59:00 | +| host2 | 90.0 | 2024-12-10T13:59:00 | +| host2 | 90.0 | 2024-12-31T13:59:00 | ++-------+-------------+---------------------+ + +SELECT * FROM system_metrics WHERE ts WITHIN '2025' ORDER BY host, ts; + ++-------+-------------+---------------------+ +| host | memory_util | ts | ++-------+-------------+---------------------+ +| host1 | 10.3 | 2025-01-01T00:00:00 | +| host1 | 10.3 | 2025-01-13T18:59:00 | +| host1 | 40.6 | 2025-11-01T00:00:00 | +| host1 | 10.3 | 2025-11-30T23:59:59 | +| host1 | 10.3 | 2025-12-31T01:11:59 | +| host1 | 10.3 | 2025-12-31T01:12:00 | +| host1 | 10.3 | 2025-12-31T01:12:59 | +| host1 | 10.3 | 2025-12-31T01:13:00 | +| host1 | 40.6 | 2025-12-31T23:59:00 | +| host1 | 40.6 | 2025-12-31T23:59:59 | +| host2 | 90.0 | 2025-01-03T13:59:00 | +| host2 | 90.0 | 2025-02-01T00:00:00 | +| host2 | 90.0 | 2025-02-13T12:11:06 | +| host2 | 90.0 | 2025-12-01T00:00:00 | +| host2 | 90.0 | 2025-12-01T01:00:00 | +| host2 | 90.0 | 2025-12-01T01:00:59 | +| host2 | 90.0 | 2025-12-01T01:01:00 | +| host2 | 90.0 | 2025-12-01T01:59:59 | +| host2 | 90.0 | 2025-12-01T02:00:00 | +| host2 | 90.0 | 2025-12-01T23:59:59 | +| host2 | 90.0 | 2025-12-02T00:00:00 | +| host2 | 90.0 | 2025-12-15T13:42:12 | +| host2 | 90.0 | 2025-12-31T01:12:00 | ++-------+-------------+---------------------+ + +SELECT * FROM system_metrics WHERE ts WITHIN '2026' ORDER BY host, ts; + ++-------+-------------+---------------------+ +| host | memory_util | ts | ++-------+-------------+---------------------+ +| host1 | 10.3 | 2026-01-01T00:00:00 | +| host2 | 90.0 | 2026-01-01T23:59:59 | +| host2 | 90.0 | 2026-12-31T13:59:00 | ++-------+-------------+---------------------+ + +SELECT * FROM system_metrics WHERE ts WITHIN '2027' ORDER BY host, ts; + +++ +++ + +SELECT * FROM system_metrics WHERE ts WITHIN '2025-01' ORDER BY host, ts; + ++-------+-------------+---------------------+ +| host | memory_util | ts | ++-------+-------------+---------------------+ +| host1 | 10.3 | 2025-01-01T00:00:00 | +| host1 | 10.3 | 2025-01-13T18:59:00 | +| host2 | 90.0 | 2025-01-03T13:59:00 | ++-------+-------------+---------------------+ + +SELECT * FROM system_metrics WHERE ts WITHIN '2025-05' ORDER BY host, ts; + +++ +++ + +SELECT * FROM system_metrics WHERE ts WITHIN '2025-12' ORDER BY host, ts; + ++-------+-------------+---------------------+ +| host | memory_util | ts | ++-------+-------------+---------------------+ +| host1 | 10.3 | 2025-12-31T01:11:59 | +| host1 | 10.3 | 2025-12-31T01:12:00 | +| host1 | 10.3 | 2025-12-31T01:12:59 | +| host1 | 10.3 | 2025-12-31T01:13:00 | +| host1 | 40.6 | 2025-12-31T23:59:00 | +| host1 | 40.6 | 2025-12-31T23:59:59 | +| host2 | 90.0 | 2025-12-01T00:00:00 | +| host2 | 90.0 | 2025-12-01T01:00:00 | +| host2 | 90.0 | 2025-12-01T01:00:59 | +| host2 | 90.0 | 2025-12-01T01:01:00 | +| host2 | 90.0 | 2025-12-01T01:59:59 | +| host2 | 90.0 | 2025-12-01T02:00:00 | +| host2 | 90.0 | 2025-12-01T23:59:59 | +| host2 | 90.0 | 2025-12-02T00:00:00 | +| host2 | 90.0 | 2025-12-15T13:42:12 | +| host2 | 90.0 | 2025-12-31T01:12:00 | ++-------+-------------+---------------------+ + +SELECT * FROM system_metrics WHERE ts WITHIN '2025-12-01' ORDER BY host, ts; + ++-------+-------------+---------------------+ +| host | memory_util | ts | ++-------+-------------+---------------------+ +| host2 | 90.0 | 2025-12-01T00:00:00 | +| host2 | 90.0 | 2025-12-01T01:00:00 | +| host2 | 90.0 | 2025-12-01T01:00:59 | +| host2 | 90.0 | 2025-12-01T01:01:00 | +| host2 | 90.0 | 2025-12-01T01:59:59 | +| host2 | 90.0 | 2025-12-01T02:00:00 | +| host2 | 90.0 | 2025-12-01T23:59:59 | ++-------+-------------+---------------------+ + +SELECT * FROM system_metrics WHERE ts WITHIN '2025-12-24' ORDER BY host, ts; + +++ +++ + +SELECT * FROM system_metrics WHERE ts WITHIN '2025-12-01T01' ORDER BY host, ts; + ++-------+-------------+---------------------+ +| host | memory_util | ts | ++-------+-------------+---------------------+ +| host2 | 90.0 | 2025-12-01T01:00:00 | +| host2 | 90.0 | 2025-12-01T01:00:59 | +| host2 | 90.0 | 2025-12-01T01:01:00 | +| host2 | 90.0 | 2025-12-01T01:59:59 | ++-------+-------------+---------------------+ + +SELECT * FROM system_metrics WHERE ts WITHIN '2025-12-01T15' ORDER BY host, ts; + +++ +++ + +SELECT * FROM system_metrics WHERE ts WITHIN '2025-12-01T01:00' ORDER BY host, ts; + ++-------+-------------+---------------------+ +| host | memory_util | ts | ++-------+-------------+---------------------+ +| host2 | 90.0 | 2025-12-01T01:00:00 | +| host2 | 90.0 | 2025-12-01T01:00:59 | ++-------+-------------+---------------------+ + +SELECT * FROM system_metrics WHERE ts WITHIN '2025-12-01T15:00' ORDER BY host, ts; + +++ +++ + +SELECT * FROM system_metrics WHERE ts WITHIN '2025-12-31T01:12:59' ORDER BY host, ts; + ++-------+-------------+---------------------+ +| host | memory_util | ts | ++-------+-------------+---------------------+ +| host1 | 10.3 | 2025-12-31T01:12:59 | ++-------+-------------+---------------------+ + +SELECT * FROM system_metrics WHERE ts WITHIN '2025-12-31T01:12:55' ORDER BY host, ts; + +++ +++ + +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +EXPLAIN ANALYZE SELECT * FROM system_metrics WHERE ts WITHIN '2025'; + ++-+-+-+ +| stage | node | plan_| ++-+-+-+ +| 0_| 0_|_MergeScanExec: REDACTED +|_|_|_| +| 1_| 0_|_CoalesceBatchesExec: target_batch_size=8192 REDACTED +|_|_|_FilterExec: ts@2 >= 1735689600000 AND ts@2 < 1767225600000 REDACTED +|_|_|_RepartitionExec: partitioning=REDACTED +|_|_|_SeqScan: region=REDACTED, partition_count=1 (1 memtable ranges, 0 file 0 ranges) REDACTED +|_|_|_| +|_|_| Total rows: 23_| ++-+-+-+ + +DROP TABLE system_metrics; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/function/within_filter.sql b/tests/cases/standalone/common/function/within_filter.sql new file mode 100644 index 000000000000..70838188f70c --- /dev/null +++ b/tests/cases/standalone/common/function/within_filter.sql @@ -0,0 +1,83 @@ +CREATE TABLE system_metrics ( + host STRING, + memory_util DOUBLE, + ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP(), + PRIMARY KEY(host), + TIME INDEX(ts) +); + +INSERT INTO system_metrics +VALUES + ("host1", 10.3, "2024-10-10T13:59:00Z"), + ("host2", 90.0, "2024-12-10T13:59:00Z"), + ("host2", 90.0, "2024-12-31T13:59:00Z"), + ("host1", 10.3, "2025-01-01T00:00:00Z"), + ("host2", 90.0, "2025-01-03T13:59:00Z"), + ("host1", 10.3, "2025-01-13T18:59:00Z"), + ("host2", 90.0, "2025-02-01T00:00:00Z"), + ("host2", 90.0, "2025-02-13T12:11:06Z"), + ("host1", 40.6, "2025-11-01T00:00:00Z"), + ("host1", 10.3, "2025-11-30T23:59:59Z"), + ("host2", 90.0, "2025-12-01T00:00:00Z"), + ("host2", 90.0, "2025-12-01T01:00:00Z"), + ("host2", 90.0, "2025-12-01T01:59:59Z"), + ("host2", 90.0, "2025-12-01T01:00:00Z"), + ("host2", 90.0, "2025-12-01T01:00:00Z"), + ("host2", 90.0, "2025-12-01T01:00:59Z"), + ("host2", 90.0, "2025-12-01T01:01:00Z"), + ("host1", 10.3, "2025-12-31T01:11:59Z"), + ("host1", 10.3, "2025-12-31T01:12:00Z"), + ("host2", 90.0, "2025-12-31T01:12:00Z"), + ("host1", 10.3, "2025-12-31T01:12:59Z"), + ("host1", 10.3, "2025-12-31T01:13:00Z"), + ("host2", 90.0, "2025-12-01T02:00:00Z"), + ("host2", 90.0, "2025-12-01T23:59:59Z"), + ("host2", 90.0, "2025-12-02T00:00:00Z"), + ("host2", 90.0, "2025-12-15T13:42:12Z"), + ("host1", 40.6, "2025-12-31T23:59:00Z"), + ("host1", 40.6, "2025-12-31T23:59:59Z"), + ("host1", 10.3, "2026-01-01T00:00:00Z"), + ("host2", 90.0, "2026-01-01T23:59:59Z"), + ("host2", 90.0, "2026-12-31T13:59:00Z"); + +SELECT * FROM system_metrics WHERE ts WITHIN '2023' ORDER BY host, ts; + +SELECT * FROM system_metrics WHERE ts WITHIN '2024' ORDER BY host, ts; + +SELECT * FROM system_metrics WHERE ts WITHIN '2025' ORDER BY host, ts; + +SELECT * FROM system_metrics WHERE ts WITHIN '2026' ORDER BY host, ts; + +SELECT * FROM system_metrics WHERE ts WITHIN '2027' ORDER BY host, ts; + +SELECT * FROM system_metrics WHERE ts WITHIN '2025-01' ORDER BY host, ts; + +SELECT * FROM system_metrics WHERE ts WITHIN '2025-05' ORDER BY host, ts; + +SELECT * FROM system_metrics WHERE ts WITHIN '2025-12' ORDER BY host, ts; + +SELECT * FROM system_metrics WHERE ts WITHIN '2025-12-01' ORDER BY host, ts; + +SELECT * FROM system_metrics WHERE ts WITHIN '2025-12-24' ORDER BY host, ts; + +SELECT * FROM system_metrics WHERE ts WITHIN '2025-12-01T01' ORDER BY host, ts; + +SELECT * FROM system_metrics WHERE ts WITHIN '2025-12-01T15' ORDER BY host, ts; + +SELECT * FROM system_metrics WHERE ts WITHIN '2025-12-01T01:00' ORDER BY host, ts; + +SELECT * FROM system_metrics WHERE ts WITHIN '2025-12-01T15:00' ORDER BY host, ts; + +SELECT * FROM system_metrics WHERE ts WITHIN '2025-12-31T01:12:59' ORDER BY host, ts; + +SELECT * FROM system_metrics WHERE ts WITHIN '2025-12-31T01:12:55' ORDER BY host, ts; + +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +EXPLAIN ANALYZE SELECT * FROM system_metrics WHERE ts WITHIN '2025'; + +DROP TABLE system_metrics;