feat(query): enhance datetime functions (#18027)

* feat(query): enhance datetime functions

* 1. to_char as alias of to_string

2. delete TZM

Author: TCeason

src/query/ast/src/parser/expr.rs

@@ -1214,6 +1214,13 @@ pub fn expr_element(i: Input) -> IResult<WithSpan<ExprElement>> {
         |(_, _, unit, _, date, _)| ExprElement::DateTrunc { unit, date },
     );
 
+    let trunc = map(
+        rule! {
+            TRUNC ~ "(" ~  #subexpr(0) ~ "," ~  #interval_kind ~ ")"
+        },
+        |(_, _, date, _, unit, _)| ExprElement::DateTrunc { unit, date },
+    );
+
     let last_day = map(
         rule! {
             LAST_DAY ~ "(" ~ #subexpr(0) ~ ("," ~ #interval_kind)? ~ ")"
@@ -1326,7 +1333,8 @@ pub fn expr_element(i: Input) -> IResult<WithSpan<ExprElement>> {
                 | #date_diff : "`DATE_DIFF(..., ..., (YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND | DOY | DOW))`"
                 | #date_sub : "`DATE_SUB(..., ..., (YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND | DOY | DOW))`"
                 | #date_between : "`DATE_BETWEEN((YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND | DOY | DOW), ..., ...,)`"
-                | #date_trunc : "`DATE_TRUNC((YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND), ...)`"
+                | #date_trunc : "`DATE_TRUNC((YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND | WEEK), ...)`"
+                | #trunc : "`TRUNC(..., (YEAR | QUARTER | MONTH | DAY | HOUR | MINUTE | SECOND | WEEK))`"
                 | #last_day : "`LAST_DAY(..., (YEAR | QUARTER | MONTH | WEEK)))`"
                 | #previous_day : "`PREVIOUS_DAY(..., (Sunday | Monday | Tuesday | Wednesday | Thursday | Friday | Saturday))`"
                 | #next_day : "`NEXT_DAY(..., (Sunday | Monday | Tuesday | Wednesday | Thursday | Friday | Saturday))`"

src/query/ast/src/parser/token.rs

@@ -517,6 +517,8 @@ pub enum TokenKind {
     DATESUB,
     #[token("DATE_TRUNC", ignore(ascii_case))]
     DATE_TRUNC,
+    #[token("TRUNC", ignore(ascii_case))]
+    TRUNC,
     #[token("DATETIME", ignore(ascii_case))]
     DATETIME,
     #[token("DAY", ignore(ascii_case))]
@@ -1672,6 +1674,7 @@ impl TokenKind {
             | TokenKind::DATE_SUB
             | TokenKind::DATE_BETWEEN
             | TokenKind::DATE_TRUNC
+            | TokenKind::TRUNC
             | TokenKind::LAST_DAY
             | TokenKind::PREVIOUS_DAY
             | TokenKind::NEXT_DAY

src/query/ast/tests/it/parser.rs

@@ -1264,6 +1264,8 @@ fn test_expr() {
         r#"extract(year from d)"#,
         r#"date_part(year, d)"#,
         r#"datepart(year, d)"#,
+        r#"date_trunc(week, to_timestamp(1630812366))"#,
+        r#"trunc(to_timestamp(1630812366), week)"#,
         r#"DATEDIFF(SECOND, to_timestamp('2024-01-01 21:01:35.423179'), to_timestamp('2023-12-31 09:38:18.165575'))"#,
         r#"last_day(to_date('2024-10-22'), week)"#,
         r#"last_day(to_date('2024-10-22'))"#,

src/query/ast/tests/it/testdata/expr-error.txt

@@ -52,7 +52,7 @@ error:
   --> SQL:1:10
   |
 1 | CAST(col1)
-  | ----     ^ unexpected `)`, expecting `AS`, `,`, `(`, `IS`, `NOT`, `IN`, `EXISTS`, `BETWEEN`, `+`, `-`, `*`, `/`, `//`, `DIV`, `%`, `||`, `<->`, `>`, `<`, `>=`, `<=`, `=`, `<>`, `!=`, `^`, `AND`, `OR`, `XOR`, `LIKE`, `REGEXP`, `RLIKE`, `SOUNDS`, <BitWiseOr>, <BitWiseAnd>, <BitWiseXor>, <ShiftLeft>, <ShiftRight>, `->`, `->>`, `#>`, `#>>`, `?`, `?|`, `?&`, `@>`, `<@`, `@?`, `@@`, `#-`, <Factorial>, <SquareRoot>, <BitWiseNot>, <CubeRoot>, <Abs>, `CAST`, `TRY_CAST`, `::`, `POSITION`, `IdentVariable`, `DATE_ADD`, or 41 more ...
+  | ----     ^ unexpected `)`, expecting `AS`, `,`, `(`, `IS`, `NOT`, `IN`, `EXISTS`, `BETWEEN`, `+`, `-`, `*`, `/`, `//`, `DIV`, `%`, `||`, `<->`, `>`, `<`, `>=`, `<=`, `=`, `<>`, `!=`, `^`, `AND`, `OR`, `XOR`, `LIKE`, `REGEXP`, `RLIKE`, `SOUNDS`, <BitWiseOr>, <BitWiseAnd>, <BitWiseXor>, <ShiftLeft>, <ShiftRight>, `->`, `->>`, `#>`, `#>>`, `?`, `?|`, `?&`, `@>`, `<@`, `@?`, `@@`, `#-`, <Factorial>, <SquareRoot>, <BitWiseNot>, <CubeRoot>, <Abs>, `CAST`, `TRY_CAST`, `::`, `POSITION`, `IdentVariable`, `DATE_ADD`, or 42 more ...
   | |         
   | while parsing `CAST(... AS ...)`
   | while parsing expression
@@ -81,7 +81,7 @@ error:
 1 | $ abc + 3
   | ^
   | |
-  | unexpected `$`, expecting `IS`, `IN`, `EXISTS`, `BETWEEN`, `+`, `-`, `*`, `/`, `//`, `DIV`, `%`, `||`, `<->`, `>`, `<`, `>=`, `<=`, `=`, `<>`, `!=`, `^`, `AND`, `OR`, `XOR`, `LIKE`, `NOT`, `REGEXP`, `RLIKE`, `SOUNDS`, <BitWiseOr>, <BitWiseAnd>, <BitWiseXor>, <ShiftLeft>, <ShiftRight>, `->`, `->>`, `#>`, `#>>`, `?`, `?|`, `?&`, `@>`, `<@`, `@?`, `@@`, `#-`, <Factorial>, <SquareRoot>, <BitWiseNot>, <CubeRoot>, <Abs>, `CAST`, `TRY_CAST`, `::`, `POSITION`, `IdentVariable`, `DATE_ADD`, `DATE_DIFF`, `DATEDIFF`, `DATESUB`, or 39 more ...
+  | unexpected `$`, expecting `IS`, `IN`, `EXISTS`, `BETWEEN`, `+`, `-`, `*`, `/`, `//`, `DIV`, `%`, `||`, `<->`, `>`, `<`, `>=`, `<=`, `=`, `<>`, `!=`, `^`, `AND`, `OR`, `XOR`, `LIKE`, `NOT`, `REGEXP`, `RLIKE`, `SOUNDS`, <BitWiseOr>, <BitWiseAnd>, <BitWiseXor>, <ShiftLeft>, <ShiftRight>, `->`, `->>`, `#>`, `#>>`, `?`, `?|`, `?&`, `@>`, `<@`, `@?`, `@@`, `#-`, <Factorial>, <SquareRoot>, <BitWiseNot>, <CubeRoot>, <Abs>, `CAST`, `TRY_CAST`, `::`, `POSITION`, `IdentVariable`, `DATE_ADD`, `DATE_DIFF`, `DATEDIFF`, `DATESUB`, or 40 more ...
   | while parsing expression
 
 

src/query/ast/tests/it/testdata/expr.txt

@@ -1842,6 +1842,92 @@ DatePart {
 }
 
 
+---------- Input ----------
+date_trunc(week, to_timestamp(1630812366))
+---------- Output ---------
+DATE_TRUNC(WEEK, to_timestamp(1630812366))
+---------- AST ------------
+DateTrunc {
+    span: Some(
+        0..42,
+    ),
+    unit: Week,
+    date: FunctionCall {
+        span: Some(
+            17..41,
+        ),
+        func: FunctionCall {
+            distinct: false,
+            name: Identifier {
+                span: Some(
+                    17..29,
+                ),
+                name: "to_timestamp",
+                quote: None,
+                ident_type: None,
+            },
+            args: [
+                Literal {
+                    span: Some(
+                        30..40,
+                    ),
+                    value: UInt64(
+                        1630812366,
+                    ),
+                },
+            ],
+            params: [],
+            order_by: [],
+            window: None,
+            lambda: None,
+        },
+    },
+}
+
+
+---------- Input ----------
+trunc(to_timestamp(1630812366), week)
+---------- Output ---------
+DATE_TRUNC(WEEK, to_timestamp(1630812366))
+---------- AST ------------
+DateTrunc {
+    span: Some(
+        0..37,
+    ),
+    unit: Week,
+    date: FunctionCall {
+        span: Some(
+            6..30,
+        ),
+        func: FunctionCall {
+            distinct: false,
+            name: Identifier {
+                span: Some(
+                    6..18,
+                ),
+                name: "to_timestamp",
+                quote: None,
+                ident_type: None,
+            },
+            args: [
+                Literal {
+                    span: Some(
+                        19..29,
+                    ),
+                    value: UInt64(
+                        1630812366,
+                    ),
+                },
+            ],
+            params: [],
+            order_by: [],
+            window: None,
+            lambda: None,
+        },
+    },
+}
+
+
 ---------- Input ----------
 DATEDIFF(SECOND, to_timestamp('2024-01-01 21:01:35.423179'), to_timestamp('2023-12-31 09:38:18.165575'))
 ---------- Output ---------

src/query/ast/tests/it/testdata/stmt-error.txt

@@ -560,7 +560,7 @@ error:
   --> SQL:1:41
   |
 1 | SELECT * FROM t GROUP BY GROUPING SETS ()
-  | ------                                  ^ unexpected `)`, expecting `(`, `IS`, `IN`, `EXISTS`, `BETWEEN`, `+`, `-`, `*`, `/`, `//`, `DIV`, `%`, `||`, `<->`, `>`, `<`, `>=`, `<=`, `=`, `<>`, `!=`, `^`, `AND`, `OR`, `XOR`, `LIKE`, `NOT`, `REGEXP`, `RLIKE`, `SOUNDS`, <BitWiseOr>, <BitWiseAnd>, <BitWiseXor>, <ShiftLeft>, <ShiftRight>, `->`, `->>`, `#>`, `#>>`, `?`, `?|`, `?&`, `@>`, `<@`, `@?`, `@@`, `#-`, <Factorial>, <SquareRoot>, <BitWiseNot>, <CubeRoot>, <Abs>, `CAST`, `TRY_CAST`, `::`, `POSITION`, `IdentVariable`, `DATE_ADD`, `DATE_DIFF`, `DATEDIFF`, or 39 more ...
+  | ------                                  ^ unexpected `)`, expecting `(`, `IS`, `IN`, `EXISTS`, `BETWEEN`, `+`, `-`, `*`, `/`, `//`, `DIV`, `%`, `||`, `<->`, `>`, `<`, `>=`, `<=`, `=`, `<>`, `!=`, `^`, `AND`, `OR`, `XOR`, `LIKE`, `NOT`, `REGEXP`, `RLIKE`, `SOUNDS`, <BitWiseOr>, <BitWiseAnd>, <BitWiseXor>, <ShiftLeft>, <ShiftRight>, `->`, `->>`, `#>`, `#>>`, `?`, `?|`, `?&`, `@>`, `<@`, `@?`, `@@`, `#-`, <Factorial>, <SquareRoot>, <BitWiseNot>, <CubeRoot>, <Abs>, `CAST`, `TRY_CAST`, `::`, `POSITION`, `IdentVariable`, `DATE_ADD`, `DATE_DIFF`, `DATEDIFF`, or 40 more ...
   | |                                        
   | while parsing `SELECT ...`
 
@@ -982,7 +982,7 @@ error:
   --> SQL:1:65
   |
 1 | CREATE FUNCTION IF NOT EXISTS isnotempty AS(p) -> not(is_null(p)
-  | ------                                   --       ----          ^ unexpected end of input, expecting `)`, `(`, `WITHIN`, `IGNORE`, `RESPECT`, `OVER`, `IS`, `NOT`, `IN`, `EXISTS`, `BETWEEN`, `+`, `-`, `*`, `/`, `//`, `DIV`, `%`, `||`, `<->`, `>`, `<`, `>=`, `<=`, `=`, `<>`, `!=`, `^`, `AND`, `OR`, `XOR`, `LIKE`, `REGEXP`, `RLIKE`, `SOUNDS`, <BitWiseOr>, <BitWiseAnd>, <BitWiseXor>, <ShiftLeft>, <ShiftRight>, `->`, `->>`, `#>`, `#>>`, `?`, `?|`, `?&`, `@>`, `<@`, `@?`, `@@`, `#-`, <Factorial>, <SquareRoot>, <BitWiseNot>, <CubeRoot>, <Abs>, `CAST`, `TRY_CAST`, `::`, or 45 more ...
+  | ------                                   --       ----          ^ unexpected end of input, expecting `)`, `(`, `WITHIN`, `IGNORE`, `RESPECT`, `OVER`, `IS`, `NOT`, `IN`, `EXISTS`, `BETWEEN`, `+`, `-`, `*`, `/`, `//`, `DIV`, `%`, `||`, `<->`, `>`, `<`, `>=`, `<=`, `=`, `<>`, `!=`, `^`, `AND`, `OR`, `XOR`, `LIKE`, `REGEXP`, `RLIKE`, `SOUNDS`, <BitWiseOr>, <BitWiseAnd>, <BitWiseXor>, <ShiftLeft>, <ShiftRight>, `->`, `->>`, `#>`, `#>>`, `?`, `?|`, `?&`, `@>`, `<@`, `@?`, `@@`, `#-`, <Factorial>, <SquareRoot>, <BitWiseNot>, <CubeRoot>, <Abs>, `CAST`, `TRY_CAST`, `::`, or 46 more ...
   | |                                        |        |  |          
   | |                                        |        |  while parsing `(<expr> [, ...])`
   | |                                        |        while parsing expression

src/query/expression/src/function.rs

@@ -172,6 +172,8 @@ pub struct FunctionContext {
     pub parse_datetime_ignore_remainder: bool,
     pub enable_strict_datetime_parser: bool,
     pub random_function_seed: bool,
+    pub week_start: u8,
+    pub date_format_style: String,
 }
 
 impl Default for FunctionContext {
@@ -192,6 +194,8 @@ impl Default for FunctionContext {
             parse_datetime_ignore_remainder: false,
             enable_strict_datetime_parser: true,
             random_function_seed: false,
+            week_start: 0,
+            date_format_style: "mysql".to_string(),
         }
     }
 }

src/query/expression/src/utils/date_helper.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::sync::LazyLock;
+
 use databend_common_exception::Result;
 use jiff::civil::date;
 use jiff::civil::datetime;
@@ -1315,43 +1317,47 @@ pub fn previous_or_next_day(dt: &Zoned, target: Weekday, is_previous: bool) -> i
     datetime_to_date_inner_number(dt) + dir * days_diff
 }
 
-pub struct PGDateTimeFormatter;
-
-impl PGDateTimeFormatter {
-    pub fn format(dt: Zoned, format_string: &str) -> String {
-        let mut result = format_string.to_string();
-        let mut format_map: Vec<(&str, fn(&Zoned) -> String)> = Vec::new();
-
-        format_map.push(("YYYY", |dt| dt.strftime("%Y").to_string()));
-        format_map.push(("YY", |dt| dt.strftime("%y").to_string()));
-        format_map.push(("MM", |dt| dt.strftime("%m").to_string()));
-        format_map.push(("MON", |dt| dt.strftime("%b").to_string()));
-        format_map.push(("MMMM", |dt| dt.strftime("%B").to_string()));
-        format_map.push(("DD", |dt| dt.strftime("%d").to_string()));
-        format_map.push(("DY", |dt| dt.strftime("%a").to_string()));
-        format_map.push(("HH24", |dt| dt.strftime("%H").to_string()));
-        format_map.push(("HH12", |dt| dt.strftime("%I").to_string()));
-        format_map.push(("AM", |dt| dt.strftime("%p").to_string()));
-        format_map.push(("PM", |dt| dt.strftime("%p").to_string()));
-        format_map.push(("MI", |dt| dt.strftime("%M").to_string()));
-        format_map.push(("SS", |dt| dt.strftime("%S").to_string()));
-        format_map.push(("FF", |dt| dt.strftime("%f").to_string()));
-        format_map.push(("TZH", |dt| {
-            dt.strftime("%z").to_string().chars().take(3).collect()
-        }));
-        format_map.push(("TZM", |dt| {
-            dt.strftime("%z")
-                .to_string()
-                .chars()
-                .skip(3)
-                .take(2)
-                .collect()
-        }));
-        format_map.push(("UUUU", |dt| dt.strftime("%G").to_string()));
-        for (key, func) in &format_map {
-            let reg = regex::Regex::new(&format!(r"(?i){}", key)).unwrap();
-            result = reg.replace_all(&result, func(&dt)).to_string();
-        }
-        result
+static PG_STRFTIME_MAPPINGS: LazyLock<Vec<(&'static str, &'static str)>> = LazyLock::new(|| {
+    let mut mappings = vec![
+        ("YYYY", "%Y"),
+        ("YY", "%y"),
+        ("MMMM", "%B"),
+        ("MON", "%b"),
+        ("MM", "%m"),
+        ("DD", "%d"),
+        ("DY", "%a"),
+        ("HH24", "%H"),
+        ("HH12", "%I"),
+        ("AM", "%p"),
+        ("PM", "%p"),
+        ("MI", "%M"),
+        ("SS", "%S"),
+        ("FF", "%f"),
+        ("UUUU", "%G"),
+        ("TZHTZM", "%z"),
+        ("TZH:TZM", "%z"),
+        ("TZH", "%:::z"),
+    ];
+    // Sort by key length in descending order to ensure
+    // longer patterns are matched first and to avoid short patterns replacing part of long patterns prematurely.
+    mappings.sort_by(|a, b| b.0.len().cmp(&a.0.len()));
+    mappings
+});
+
+#[inline]
+pub fn pg_format_to_strftime(pg_format_string: &str) -> String {
+    let mut result = pg_format_string.to_string();
+    for (pg_key, strftime_code) in PG_STRFTIME_MAPPINGS.iter() {
+        let pattern = if *pg_key == "MON" {
+            // should keep "month". Only "MON" as a single string escape it.
+            format!(r"(?i)\b{}\b", regex::escape(pg_key))
+        } else {
+            format!(r"(?i){}", regex::escape(pg_key))
+        };
+        let reg = regex::Regex::new(&pattern).expect("Failed to compile regex for format key");
+
+        // Use replace_all to substitute all occurrences of the PG key with the strftime code.
+        result = reg.replace_all(&result, *strftime_code).to_string();
     }
+    result
 }

src/query/functions/src/scalars/other.rs

@@ -21,8 +21,6 @@ use databend_common_base::base::convert_byte_size;
 use databend_common_base::base::convert_number_size;
 use databend_common_base::base::uuid::Uuid;
 use databend_common_base::base::OrderedFloat;
-use databend_common_expression::date_helper::DateConverter;
-use databend_common_expression::date_helper::PGDateTimeFormatter;
 use databend_common_expression::error_to_null;
 use databend_common_expression::types::boolean::BooleanDomain;
 use databend_common_expression::types::nullable::NullableColumn;
@@ -397,8 +395,9 @@ fn register_grouping(registry: &mut FunctionRegistry) {
 }
 
 fn register_num_to_char(registry: &mut FunctionRegistry) {
+    registry.register_aliases("to_string", &["to_char"]);
     registry.register_passthrough_nullable_2_arg::<Int64Type, StringType, StringType, _, _>(
-        "to_char",
+        "to_string",
         |_, _, _| FunctionDomain::MayThrow,
         vectorize_with_builder_2_arg::<Int64Type, StringType, StringType>(
             |value, fmt, builder, ctx| {
@@ -427,7 +426,7 @@ fn register_num_to_char(registry: &mut FunctionRegistry) {
     );
 
     registry.register_passthrough_nullable_2_arg::<Float32Type, StringType, StringType, _, _>(
-        "to_char",
+        "to_string",
         |_, _, _| FunctionDomain::MayThrow,
         vectorize_with_builder_2_arg::<Float32Type, StringType, StringType>(
             |value, fmt, builder, ctx| {
@@ -457,7 +456,7 @@ fn register_num_to_char(registry: &mut FunctionRegistry) {
     );
 
     registry.register_passthrough_nullable_2_arg::<Float64Type, StringType, StringType, _, _>(
-        "to_char",
+        "to_string",
         |_, _, _| FunctionDomain::MayThrow,
         vectorize_with_builder_2_arg::<Float64Type, StringType, StringType>(
             |value, fmt, builder, ctx| {
@@ -485,19 +484,6 @@ fn register_num_to_char(registry: &mut FunctionRegistry) {
             },
         ),
     );
-
-    registry.register_passthrough_nullable_2_arg::<TimestampType, StringType, StringType, _, _>(
-        "to_char",
-        |_, _, _| FunctionDomain::Full,
-        vectorize_with_builder_2_arg::<TimestampType, StringType, StringType>(
-            |micros, format, output, ctx| {
-                let ts = micros.to_timestamp(ctx.func_ctx.tz.clone());
-                let res = PGDateTimeFormatter::format(ts, format);
-                output.put_str(&res);
-                output.commit_row();
-            },
-        ),
-    );
 }
 
 /// Compute `grouping` by `grouping_id` and `cols`.