From 4e3b6c13680fb510d212a9ec3287efefd4141d6c Mon Sep 17 00:00:00 2001 From: Sergey Olontsev Date: Fri, 20 Jun 2025 10:23:25 +0100 Subject: [PATCH 1/4] Support for Map values in ClickHouse settings --- src/ast/query.rs | 2 +- src/ast/value.rs | 1 - src/parser/mod.rs | 25 +++++-- src/test_utils.rs | 5 ++ tests/sqlparser_clickhouse.rs | 133 ++++++++++++++++++++++++++-------- 5 files changed, 124 insertions(+), 42 deletions(-) diff --git a/src/ast/query.rs b/src/ast/query.rs index 1fb93b6c6..6b28f7980 100644 --- a/src/ast/query.rs +++ b/src/ast/query.rs @@ -1047,7 +1047,7 @@ impl fmt::Display for ConnectBy { #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct Setting { pub key: Ident, - pub value: Value, + pub value: Expr, } impl fmt::Display for Setting { diff --git a/src/ast/value.rs b/src/ast/value.rs index 98616407c..769a7e533 100644 --- a/src/ast/value.rs +++ b/src/ast/value.rs @@ -116,7 +116,6 @@ impl From for Value { derive(Visit, VisitMut), visit(with = "visit_value") )] - pub enum Value { /// Numeric literal #[cfg(not(feature = "bigdecimal"))] diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 19c11d296..4484120e6 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -2771,7 +2771,7 @@ impl<'a> Parser<'a> { if self.dialect.supports_dictionary_syntax() { self.prev_token(); // Put back the '{' - return self.parse_duckdb_struct_literal(); + return self.parse_duckdb_and_clickhouse_struct_literal(); } self.expected("an expression", token) @@ -3147,7 +3147,7 @@ impl<'a> Parser<'a> { Ok(fields) } - /// DuckDB specific: Parse a duckdb [dictionary] + /// DuckDB and ClickHouse specific: Parse a duckdb [dictionary] or a clickhouse [map] setting /// /// Syntax: /// @@ -3156,18 +3156,21 @@ impl<'a> Parser<'a> { /// ``` /// /// [dictionary]: https://duckdb.org/docs/sql/data_types/struct#creating-structs - fn parse_duckdb_struct_literal(&mut self) -> Result { + /// [map]: https://clickhouse.com/docs/operations/settings/settings#additional_table_filters + fn parse_duckdb_and_clickhouse_struct_literal(&mut self) -> Result { self.expect_token(&Token::LBrace)?; - let fields = - self.parse_comma_separated0(Self::parse_duckdb_dictionary_field, Token::RBrace)?; + let fields = self.parse_comma_separated0( + Self::parse_duckdb_and_clickhouse_struct_field, + Token::RBrace, + )?; self.expect_token(&Token::RBrace)?; Ok(Expr::Dictionary(fields)) } - /// Parse a field for a duckdb [dictionary] + /// Parse a field for a duckdb [dictionary] or a clickhouse [map] setting /// /// Syntax /// @@ -3176,7 +3179,8 @@ impl<'a> Parser<'a> { /// ``` /// /// [dictionary]: https://duckdb.org/docs/sql/data_types/struct#creating-structs - fn parse_duckdb_dictionary_field(&mut self) -> Result { + /// [map]: https://clickhouse.com/docs/operations/settings/settings#additional_table_filters + fn parse_duckdb_and_clickhouse_struct_field(&mut self) -> Result { let key = self.parse_identifier()?; self.expect_token(&Token::Colon)?; @@ -11190,7 +11194,12 @@ impl<'a> Parser<'a> { let key_values = self.parse_comma_separated(|p| { let key = p.parse_identifier()?; p.expect_token(&Token::Eq)?; - let value = p.parse_value()?.value; + + let value = if p.peek_token_ref().token == Token::LBrace { + p.parse_duckdb_and_clickhouse_struct_literal()? + } else { + Expr::Value(p.parse_value()?) + }; Ok(Setting { key, value }) })?; Some(key_values) diff --git a/src/test_utils.rs b/src/test_utils.rs index 24c0ca575..331f7b8a3 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -366,6 +366,11 @@ pub fn number(n: &str) -> Value { Value::Number(n.parse().unwrap(), false) } +/// Creates a `Value::SingleQuotedString` +pub fn single_quoted_string(s: &str) -> Value { + Value::SingleQuotedString(s.to_string()) +} + pub fn table_alias(name: impl Into) -> Option { Some(TableAlias { name: Ident::new(name), diff --git a/tests/sqlparser_clickhouse.rs b/tests/sqlparser_clickhouse.rs index 93b4c4f59..26c343be2 100644 --- a/tests/sqlparser_clickhouse.rs +++ b/tests/sqlparser_clickhouse.rs @@ -28,7 +28,7 @@ use test_utils::*; use sqlparser::ast::Expr::{BinaryOp, Identifier}; use sqlparser::ast::SelectItem::UnnamedExpr; use sqlparser::ast::TableFactor::Table; -use sqlparser::ast::Value::Number; +use sqlparser::ast::Value::Boolean; use sqlparser::ast::*; use sqlparser::dialect::ClickHouseDialect; use sqlparser::dialect::GenericDialect; @@ -965,38 +965,107 @@ fn parse_limit_by() { #[test] fn parse_settings_in_query() { - match clickhouse_and_generic() - .verified_stmt(r#"SELECT * FROM t SETTINGS max_threads = 1, max_block_size = 10000"#) - { - Statement::Query(query) => { - assert_eq!( - query.settings, - Some(vec![ - Setting { - key: Ident::new("max_threads"), - value: Number("1".parse().unwrap(), false) - }, - Setting { - key: Ident::new("max_block_size"), - value: Number("10000".parse().unwrap(), false) - }, - ]) - ); + fn check_settings(sql: &str, expected: Vec) { + match clickhouse_and_generic().verified_stmt(sql) { + Statement::Query(q) => { + assert_eq!(q.settings, Some(expected)); + } + _ => unreachable!(), } - _ => unreachable!(), + } + + for (sql, expected_settings) in vec![ + ( + r#"SELECT * FROM t SETTINGS max_threads = 1, max_block_size = 10000"#, + vec![ + Setting { + key: Ident::new("max_threads"), + value: Expr::value(number("1")), + }, + Setting { + key: Ident::new("max_block_size"), + value: Expr::value(number("10000")), + }, + ], + ), + ( + r#"SELECT * FROM t SETTINGS additional_table_filters = {'table_1': 'x != 2'}"#, + vec![Setting { + key: Ident::new("additional_table_filters"), + value: Expr::Dictionary(vec![DictionaryField { + key: Ident::with_quote('\'', "table_1"), + value: Expr::value(single_quoted_string("x != 2")).into(), + }]), + }], + ), + ( + r#"SELECT * FROM t SETTINGS additional_result_filter = 'x != 2', query_plan_optimize_lazy_materialization = false"#, + vec![ + Setting { + key: Ident::new("additional_result_filter"), + value: Expr::value(single_quoted_string("x != 2")), + }, + Setting { + key: Ident::new("query_plan_optimize_lazy_materialization"), + value: Expr::value(Boolean(false)), + }, + ], + ), + ] { + check_settings(sql, expected_settings); } let invalid_cases = vec![ - "SELECT * FROM t SETTINGS a", - "SELECT * FROM t SETTINGS a=", - "SELECT * FROM t SETTINGS a=1, b", - "SELECT * FROM t SETTINGS a=1, b=", - "SELECT * FROM t SETTINGS a=1, b=c", + ("SELECT * FROM t SETTINGS a", "Expected: =, found: EOF"), + ( + "SELECT * FROM t SETTINGS a=", + "Expected: a value, found: EOF", + ), + ("SELECT * FROM t SETTINGS a=1, b", "Expected: =, found: EOF"), + ( + "SELECT * FROM t SETTINGS a=1, b=", + "Expected: a value, found: EOF", + ), + ( + "SELECT * FROM t SETTINGS a=1, b=c", + "Expected: a concrete value, found: c", + ), + ( + "SELECT * FROM t SETTINGS a = {", + "Expected: identifier, found: EOF", + ), + ( + "SELECT * FROM t SETTINGS a = {'b'", + "Expected: :, found: EOF", + ), + ( + "SELECT * FROM t SETTINGS a = {'b': ", + "Expected: an expression, found: EOF", + ), + ( + "SELECT * FROM t SETTINGS a = {'b': 'c',}", + "Expected: identifier, found: }", + ), + ( + "SELECT * FROM t SETTINGS a = {'b': 'c', 'd'}", + "Expected: :, found: }", + ), + ( + "SELECT * FROM t SETTINGS a = {'b': 'c', 'd': }", + "Expected: an expression, found: }", + ), + ( + "SELECT * FROM t SETTINGS a = {ANY(b)}", + "Expected: :, found: (", + ), ]; - for sql in invalid_cases { - clickhouse_and_generic() - .parse_sql_statements(sql) - .expect_err("Expected: SETTINGS key = value, found: "); + for (sql, error_msg) in invalid_cases { + assert_eq!( + clickhouse_and_generic() + .parse_sql_statements(sql) + .unwrap_err(), + ParserError(error_msg.to_string()) + ); } } #[test] @@ -1550,11 +1619,11 @@ fn parse_select_table_function_settings() { settings: Some(vec![ Setting { key: "s0".into(), - value: Value::Number("3".parse().unwrap(), false), + value: Expr::value(number("3")), }, Setting { key: "s1".into(), - value: Value::SingleQuotedString("s".into()), + value: Expr::value(single_quoted_string("s")), }, ]), }, @@ -1575,11 +1644,11 @@ fn parse_select_table_function_settings() { settings: Some(vec![ Setting { key: "s0".into(), - value: Value::Number("3".parse().unwrap(), false), + value: Expr::value(number("3")), }, Setting { key: "s1".into(), - value: Value::SingleQuotedString("s".into()), + value: Expr::value(single_quoted_string("s")), }, ]), }, From 39be2350e117b2773a54bbe7491088d1a588183d Mon Sep 17 00:00:00 2001 From: Sergey Olontsev Date: Fri, 20 Jun 2025 12:21:14 +0100 Subject: [PATCH 2/4] cargo clippy fix --- tests/sqlparser_clickhouse.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sqlparser_clickhouse.rs b/tests/sqlparser_clickhouse.rs index 26c343be2..e10118f5d 100644 --- a/tests/sqlparser_clickhouse.rs +++ b/tests/sqlparser_clickhouse.rs @@ -974,7 +974,7 @@ fn parse_settings_in_query() { } } - for (sql, expected_settings) in vec![ + for (sql, expected_settings) in [ ( r#"SELECT * FROM t SETTINGS max_threads = 1, max_block_size = 10000"#, vec![ From 2575081556e61c6a6dc25785063274246d621c82 Mon Sep 17 00:00:00 2001 From: Sergey Olontsev Date: Sun, 22 Jun 2025 09:56:11 +0100 Subject: [PATCH 3/4] code improvements --- src/parser/mod.rs | 18 +++++------------- tests/sqlparser_clickhouse.rs | 9 ++------- 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 4484120e6..762089a03 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -2771,7 +2771,7 @@ impl<'a> Parser<'a> { if self.dialect.supports_dictionary_syntax() { self.prev_token(); // Put back the '{' - return self.parse_duckdb_and_clickhouse_struct_literal(); + return self.parse_dictionary(); } self.expected("an expression", token) @@ -3157,13 +3157,10 @@ impl<'a> Parser<'a> { /// /// [dictionary]: https://duckdb.org/docs/sql/data_types/struct#creating-structs /// [map]: https://clickhouse.com/docs/operations/settings/settings#additional_table_filters - fn parse_duckdb_and_clickhouse_struct_literal(&mut self) -> Result { + fn parse_dictionary(&mut self) -> Result { self.expect_token(&Token::LBrace)?; - let fields = self.parse_comma_separated0( - Self::parse_duckdb_and_clickhouse_struct_field, - Token::RBrace, - )?; + let fields = self.parse_comma_separated0(Self::parse_dictionary_field, Token::RBrace)?; self.expect_token(&Token::RBrace)?; @@ -3180,7 +3177,7 @@ impl<'a> Parser<'a> { /// /// [dictionary]: https://duckdb.org/docs/sql/data_types/struct#creating-structs /// [map]: https://clickhouse.com/docs/operations/settings/settings#additional_table_filters - fn parse_duckdb_and_clickhouse_struct_field(&mut self) -> Result { + fn parse_dictionary_field(&mut self) -> Result { let key = self.parse_identifier()?; self.expect_token(&Token::Colon)?; @@ -11194,12 +11191,7 @@ impl<'a> Parser<'a> { let key_values = self.parse_comma_separated(|p| { let key = p.parse_identifier()?; p.expect_token(&Token::Eq)?; - - let value = if p.peek_token_ref().token == Token::LBrace { - p.parse_duckdb_and_clickhouse_struct_literal()? - } else { - Expr::Value(p.parse_value()?) - }; + let value = p.parse_expr()?; Ok(Setting { key, value }) })?; Some(key_values) diff --git a/tests/sqlparser_clickhouse.rs b/tests/sqlparser_clickhouse.rs index e10118f5d..bafb3169f 100644 --- a/tests/sqlparser_clickhouse.rs +++ b/tests/sqlparser_clickhouse.rs @@ -1019,16 +1019,12 @@ fn parse_settings_in_query() { ("SELECT * FROM t SETTINGS a", "Expected: =, found: EOF"), ( "SELECT * FROM t SETTINGS a=", - "Expected: a value, found: EOF", + "Expected: an expression, found: EOF", ), ("SELECT * FROM t SETTINGS a=1, b", "Expected: =, found: EOF"), ( "SELECT * FROM t SETTINGS a=1, b=", - "Expected: a value, found: EOF", - ), - ( - "SELECT * FROM t SETTINGS a=1, b=c", - "Expected: a concrete value, found: c", + "Expected: an expression, found: EOF", ), ( "SELECT * FROM t SETTINGS a = {", @@ -1658,7 +1654,6 @@ fn parse_select_table_function_settings() { "SELECT * FROM t(SETTINGS a=)", "SELECT * FROM t(SETTINGS a=1, b)", "SELECT * FROM t(SETTINGS a=1, b=)", - "SELECT * FROM t(SETTINGS a=1, b=c)", ]; for sql in invalid_cases { clickhouse_and_generic() From f693735167357668d388407bc9747623d8f6f946 Mon Sep 17 00:00:00 2001 From: Sergey Olontsev Date: Tue, 24 Jun 2025 07:43:56 +0100 Subject: [PATCH 4/4] single_quoted_string updated to accept various string types Co-authored-by: Ifeanyi Ubah --- src/test_utils.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test_utils.rs b/src/test_utils.rs index 331f7b8a3..395586793 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -366,9 +366,9 @@ pub fn number(n: &str) -> Value { Value::Number(n.parse().unwrap(), false) } -/// Creates a `Value::SingleQuotedString` -pub fn single_quoted_string(s: &str) -> Value { - Value::SingleQuotedString(s.to_string()) +/// Creates a [Value::SingleQuotedString] +pub fn single_quoted_string(s: impl Into) -> Value { + Value::SingleQuotedString(s.into()) } pub fn table_alias(name: impl Into) -> Option {