Merge branch 'main' into sqllogic-test-split

Author: dqhl76

.github/actions/test_compat_client_cluster/action.yml

@@ -25,7 +25,7 @@ runs:
     - name: Setup Go
       uses: actions/setup-go@v5
       with:
-        go-version: "1.24"
+        go-version: "1.25"
 
     - name: checkout databend-jdbc
       uses: actions/checkout@v4

Cargo.lock

@@ -1,5 +1,4 @@
 # Hashtable implementation in rust
 
-
-This package is a port from its cpp implementation from [ClickHouse](https://github.com/ClickHouse/ClickHouse/blob/master/src/Common/HashTable/HashTable.h).
+This package is a port from its cpp implementation from [ClickHouse].
 It is quite difficult to implement a high-performance hashtable from scratch. After comparing many implementations of hashtables, we found that ClickHouse's hashtable is the most suitable for OLAP systems. Therefore, we rewrote it into this Rust library.

src/common/hashtable/src/README.md

@@ -437,7 +437,11 @@ fn get_local_time(tz: &TimeZone, d: &Date, times: &mut Vec<u32>) -> Result<Zoned
     if let Some(zoned) = fast_local_to_zoned(tz, d, hour, minute, second, 0) {
         return Ok(zoned);
     }
-    tz.to_zoned(d.at(hour as i8, minute as i8, second as i8, 0))
+    let time = Time::new(hour as i8, minute as i8, second as i8, 0)
+        .map_err_to_code(ErrorCode::BadBytes, || {
+            format!("Invalid time {:02}:{:02}:{:02}", hour, minute, second)
+        })?;
+    tz.to_zoned(d.to_datetime(time))
         .map_err_to_code(ErrorCode::BadBytes, || {
             format!("Invalid time provided in times: {:?}", times)
         })
@@ -490,9 +494,23 @@ fn build_zoned_from_components(
     if let Some(zoned) = fast_local_to_zoned(tz, date, hour, minute, second, micro) {
         return Ok(zoned);
     }
+    let time = Time::new(hour as i8, minute as i8, second as i8, 0).map_err_to_code(
+        ErrorCode::BadBytes,
+        || {
+            format!(
+                "Invalid local time {:04}-{:02}-{:02} {:02}:{:02}:{:02}",
+                i32::from(date.year()),
+                date.month(),
+                date.day(),
+                hour,
+                minute,
+                second
+            )
+        },
+    )?;
 
     let base = tz
-        .to_zoned(date.at(hour as i8, minute as i8, second as i8, 0))
+        .to_zoned(date.to_datetime(time))
         .map_err_to_code(ErrorCode::BadBytes, || {
             format!(
                 "Invalid local time {:04}-{:02}-{:02} {:02}:{:02}:{:02}",

src/common/io/src/cursor_ext/cursor_read_datetime_ext.rs

@@ -1,6 +1,6 @@
 [package]
 name = "databend-common-ast"
-version = "0.2.4"
+version = "0.2.5"
 publish = true
 description = "SQL parser for Databend"
 authors = { workspace = true }

src/query/ast/Cargo.toml

@@ -18,9 +18,12 @@ use std::iter::Peekable;
 use std::str::FromStr;
 
 use crate::parser::Dialect;
+use crate::parser::token::is_ident_continue;
+use crate::parser::token::is_ident_start;
 
-// In ANSI SQL, it does not need to quote an identifier if the identifier matches
-// the following regular expression: [A-Za-z_][A-Za-z0-9_$]*.
+// An identifier does not need quoting if it matches the rules defined by
+// is_ident_start / is_ident_continue (ASCII letters, underscore, digits, `$`,
+// plus any Unicode Alphabetic character — CJK, Cyrillic, etc.).
 //
 // There are also two known special cases in Databend which do not require quoting:
 // - "~" is a valid stage name
@@ -37,12 +40,12 @@ pub fn ident_needs_quote(ident: &str) -> bool {
 
     let mut chars = ident.chars();
     let first = chars.next().unwrap();
-    if !first.is_ascii_alphabetic() && first != '_' {
+    if !is_ident_start(first) {
         return true;
     }
 
     for c in chars {
-        if !c.is_ascii_alphanumeric() && c != '_' && c != '$' {
+        if !is_ident_continue(c) {
             return true;
         }
     }
@@ -57,7 +60,11 @@ pub fn display_ident(
     dialect: Dialect,
 ) -> String {
     // Db-s -> "Db-s" ; dbs -> dbs
-    if name.chars().any(|c| c.is_ascii_uppercase()) && quoted_ident_case_sensitive
+    // Quote the identifier if it would change under to_lowercase(), so that
+    // round-tripping through normalize_identifier (which lowercases unquoted
+    // idents) preserves the original name.  This covers both uppercase (Lu)
+    // and titlecase (Lt) Unicode characters — e.g. Dž (U+01C5).
+    if name != name.to_lowercase() && quoted_ident_case_sensitive
         || ident_needs_quote(name)
         || force_quoted_ident
     {
@@ -73,7 +80,7 @@ pub fn ident_opt_quote(
     quoted_ident_case_sensitive: bool,
     dialect: Dialect,
 ) -> Option<char> {
-    if name.chars().any(|c| c.is_ascii_uppercase()) && quoted_ident_case_sensitive
+    if name != name.to_lowercase() && quoted_ident_case_sensitive
         || ident_needs_quote(name)
         || force_quoted_ident
     {

src/query/ast/src/ast/quote.rs

@@ -137,6 +137,45 @@ impl<'a> Iterator for Tokenizer<'a> {
     }
 }
 
+/// Returns true if `c` can start an unquoted identifier.
+///
+/// ASCII letters and underscore are accepted, plus any Unicode character with
+/// the Alphabetic derived property (`char::is_alphabetic()` — covers CJK,
+/// Cyrillic, Arabic, etc. while excluding separators, punctuation, and symbols).
+pub fn is_ident_start(c: char) -> bool {
+    c == '_' || c.is_alphabetic()
+}
+
+/// Returns true if `c` can continue an unquoted identifier.
+///
+/// Same as `is_ident_start`, plus ASCII digits and `$`.
+/// Non-ASCII digits are also accepted via `char::is_alphanumeric()`.
+pub fn is_ident_continue(c: char) -> bool {
+    c == '_' || c == '$' || c.is_alphanumeric()
+}
+
+fn bump_ident_continue(lex: &mut Lexer<TokenKind>) {
+    loop {
+        match lex.remainder().chars().next() {
+            Some(c) if is_ident_continue(c) => lex.bump(c.len_utf8()),
+            _ => break,
+        }
+    }
+}
+
+fn lex_ascii_ident(lex: &mut Lexer<TokenKind>) -> logos::FilterResult<()> {
+    bump_ident_continue(lex);
+    logos::FilterResult::Emit(())
+}
+
+fn lex_unicode_ident(lex: &mut Lexer<TokenKind>) -> logos::FilterResult<()> {
+    if !is_ident_start(lex.slice().chars().next().unwrap()) {
+        return logos::FilterResult::Error;
+    }
+    bump_ident_continue(lex);
+    logos::FilterResult::Emit(())
+}
+
 fn lex_comment_block(lex: &mut Lexer<TokenKind>) -> logos::FilterResult<()> {
     let remainder = lex.remainder().as_bytes();
 
@@ -167,7 +206,8 @@ pub enum TokenKind {
     #[token("/*", lex_comment_block)]
     CommentBlock,
 
-    #[regex(r#"[_a-zA-Z][_$a-zA-Z0-9]*"#)]
+    #[regex(r#"[_a-zA-Z][_$a-zA-Z0-9]*"#, lex_ascii_ident)]
+    #[regex(r"[^\x00-\x7f]", lex_unicode_ident)]
     Ident,
 
     #[regex(r#"\$[_a-zA-Z][_$a-zA-Z0-9]*"#)]

src/query/ast/src/parser/token.rs

@@ -1738,7 +1738,7 @@ fn test_quote() {
         ("a\\\"b", "\"a\\\"\"b\""),
         ("12", "\"12\""),
         ("🍣", "\"🍣\""),
-        ("価格", "\"価格\""),
+        ("価格", "価格"),
         ("\t", "\"\t\""),
         ("complex \"string\"", "\"complex \"\"string\"\"\""),
         ("\"\"\"", "\"\"\"\"\"\"\"\""),
@@ -1759,3 +1759,43 @@ fn test_quote() {
         };
     }
 }
+
+#[test]
+fn test_unicode_ident_tokenize() {
+    let cases = &[
+        ("中文", vec![(TokenKind::Ident, "中文")]),
+        ("価格", vec![(TokenKind::Ident, "価格")]),
+        ("SELECT 'a' AS 中文", vec![
+            (TokenKind::SELECT, "SELECT"),
+            (TokenKind::LiteralString, "'a'"),
+            (TokenKind::AS, "AS"),
+            (TokenKind::Ident, "中文"),
+        ]),
+        // Mixed ASCII and Unicode
+        ("abc中文", vec![(TokenKind::Ident, "abc中文")]),
+        ("abc中文123", vec![(TokenKind::Ident, "abc中文123")]),
+        ("_中文", vec![(TokenKind::Ident, "_中文")]),
+        ("_列名_1", vec![(TokenKind::Ident, "_列名_1")]),
+        // Non-BMP but Alphabetic (CJK Extension B U+20000) — still valid
+        ("𠀀", vec![(TokenKind::Ident, "𠀀")]),
+        ("abc𠀀123", vec![(TokenKind::Ident, "abc𠀀123")]),
+        ("_𠀀$1", vec![(TokenKind::Ident, "_𠀀$1")]),
+    ];
+
+    for (input, expected) in cases {
+        let tokens: Vec<_> = Tokenizer::new(input)
+            .map(|t| t.unwrap())
+            .filter(|t| t.kind != TokenKind::EOI)
+            .map(|t| (t.kind, t.text().to_string()))
+            .collect();
+        let tokens: Vec<_> = tokens.iter().map(|(k, s)| (*k, s.as_str())).collect();
+        assert_eq!(tokens, *expected, "input: {input}");
+    }
+
+    // Emoji is not Alphabetic, tokenizer must return an error
+    let err_cases = &["🍣"];
+    for input in err_cases {
+        let result: std::result::Result<Vec<_>, _> = Tokenizer::new(input).collect();
+        assert!(result.is_err(), "expected error for input: {input}");
+    }
+}

src/query/ast/tests/it/parser.rs

@@ -21,6 +21,7 @@ use databend_common_exception::ErrorCode;
 use databend_common_exception::Result;
 use databend_common_io::datetime::parse_standard_timestamp as parse_iso_timestamp;
 use jiff::civil::Date;
+use jiff::civil::Time;
 use jiff::fmt;
 use jiff::tz;
 use jiff::tz::TimeZone;
@@ -196,8 +197,10 @@ fn build_timestamp_tz_from_components(
             year, month, day
         ))
     })?;
+    let time = Time::new(hour as i8, minute as i8, second as i8, 0)
+        .map_err(|err| ErrorCode::BadBytes(format!("Invalid time value: {}", err)))?;
     let mut zoned = date
-        .at(hour as i8, minute as i8, second as i8, 0)
+        .to_datetime(time)
         .to_zoned(TimeZone::UTC)
         .map_err(|err| ErrorCode::BadBytes(format!("Invalid time value: {}", err)))?;
     if micro > 0 {

src/query/expression/src/types/timestamp_tz.rs

@@ -281,14 +281,20 @@ impl PhysicalPlanBuilder {
                 }
             }
         }
+        // Not-matched expressions are evaluated on the not-matched branch directly.
+        // That branch may bypass RowFetch, so these columns must be in the mutation input.
+        let mut unmatched_required = BTreeSet::new();
         for unmatched_evaluator in unmatched_evaluators {
             if let Some(condition) = &unmatched_evaluator.condition {
                 maybe_udfs.extend(condition.used_columns());
+                unmatched_required.extend(condition.used_columns());
             }
             for value in &unmatched_evaluator.values {
                 maybe_udfs.extend(value.used_columns());
+                unmatched_required.extend(value.used_columns());
             }
         }
+        required.extend(unmatched_required);
         for filter_value in direct_filter {
             maybe_udfs.extend(filter_value.used_columns());
         }
@@ -486,17 +492,20 @@ impl PhysicalPlanBuilder {
             );
 
         for item in unmatched_evaluators {
+            // The not-matched branch may bypass RowFetch (see RowFetch on MutationSplit),
+            // so expressions must be bound against the original mutation input schema.
             let filter = if let Some(filter_expr) = &item.condition {
-                Some(self.scalar_expr_to_remote_expr(filter_expr, output_schema.clone())?)
+                Some(self.scalar_expr_to_remote_expr(filter_expr, mutation_input_schema.clone())?)
             } else {
                 None
             };
 
             let mut values_exprs = Vec::<RemoteExpr>::with_capacity(item.values.len());
 
             for scalar_expr in &item.values {
-                values_exprs
-                    .push(self.scalar_expr_to_remote_expr(scalar_expr, output_schema.clone())?)
+                values_exprs.push(
+                    self.scalar_expr_to_remote_expr(scalar_expr, mutation_input_schema.clone())?,
+                )
             }
 
             unmatched.push((item.source_schema.clone(), filter, values_exprs))