From 9cfc3252f68a8dd3349ac43a75de83f2488bd54f Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 6 Nov 2024 16:57:19 -0500 Subject: [PATCH 1/3] chore: Update to sqlparser-rs 0.52.0 pre-release --- Cargo.toml | 2 +- datafusion-cli/Cargo.lock | 32 +++++++++++++++----------------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 91f09102ce48..211beea65357 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -144,7 +144,7 @@ rand = "0.8" regex = "1.8" rstest = "0.23.0" serde_json = "1" -sqlparser = { version = "0.51.0", features = ["visitor"] } +sqlparser = { version = "0.52.0", features = ["visitor"], git = "https://github.com/apache/datafusion-sqlparser-rs.git", rev = "a4aa6e28d68b83099db77e594e9209d85820ec96" } tempfile = "3" tokio = { version = "1.36", features = ["macros", "rt", "sync"] } url = "2.2" diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index 5140ae7eb0dc..d10072fa4faa 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -456,9 +456,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.5.9" +version = "1.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6448cfb224dd6a9b9ac734f58622dd0d4751f3589f3b777345745f46b2eb14" +checksum = "9b49afaa341e8dd8577e1a2200468f98956d6eda50bcf4a53246cc00174ba924" dependencies = [ "aws-credential-types", "aws-runtime", @@ -523,9 +523,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.48.0" +version = "1.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded855583fa1d22e88fe39fd6062b062376e50a8211989e07cf5e38d52eb3453" +checksum = "09677244a9da92172c8dc60109b4a9658597d4d298b188dd0018b6a66b410ca4" dependencies = [ "aws-credential-types", "aws-runtime", @@ -545,9 +545,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.49.0" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9177ea1192e6601ae16c7273385690d88a7ed386a00b74a6bc894d12103cd933" +checksum = "81fea2f3a8bb3bd10932ae7ad59cc59f65f270fc9183a7e91f501dc5efbef7ee" dependencies = [ "aws-credential-types", "aws-runtime", @@ -567,9 +567,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.48.0" +version = "1.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "823ef553cf36713c97453e2ddff1eb8f62be7f4523544e2a5db64caf80100f0a" +checksum = "53dcf5e7d9bd1517b8b998e170e650047cea8a2b85fe1835abe3210713e541b7" dependencies = [ "aws-credential-types", "aws-runtime", @@ -917,9 +917,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.35" +version = "1.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f57c4b4da2a9d619dd035f27316d7a426305b75be93d09e92f2b9229c34feaf" +checksum = "baee610e9452a8f6f0a1b6194ec09ff9e2d85dea54432acdae41aa0761c95d70" dependencies = [ "jobserver", "libc", @@ -3043,9 +3043,9 @@ dependencies = [ [[package]] name = "quad-rand" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b76f1009795ca44bb5aaae8fd3f18953e209259c33d9b059b1f53d58ab7511db" +checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" [[package]] name = "quick-xml" @@ -3681,9 +3681,8 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "sqlparser" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fe11944a61da0da3f592e19a45ebe5ab92dc14a779907ff1f08fbb797bfefc7" +version = "0.52.0" +source = "git+https://github.com/apache/datafusion-sqlparser-rs.git?rev=a4aa6e28d68b83099db77e594e9209d85820ec96#a4aa6e28d68b83099db77e594e9209d85820ec96" dependencies = [ "log", "sqlparser_derive", @@ -3692,8 +3691,7 @@ dependencies = [ [[package]] name = "sqlparser_derive" version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" +source = "git+https://github.com/apache/datafusion-sqlparser-rs.git?rev=a4aa6e28d68b83099db77e594e9209d85820ec96#a4aa6e28d68b83099db77e594e9209d85820ec96" dependencies = [ "proc-macro2", "quote", From d2f3304c579c85374deba48f443cb1cd7a5a5b53 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 6 Nov 2024 17:25:55 -0500 Subject: [PATCH 2/3] Update for API changes --- datafusion/sql/src/expr/mod.rs | 11 +++++++++++ datafusion/sql/src/parser.rs | 4 ++-- datafusion/sql/src/statement.rs | 9 ++++++++- datafusion/sql/src/unparser/ast.rs | 1 + datafusion/sql/src/unparser/expr.rs | 1 + 5 files changed, 23 insertions(+), 3 deletions(-) diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rs index 432e8668c52e..3f1174e115e0 100644 --- a/datafusion/sql/src/expr/mod.rs +++ b/datafusion/sql/src/expr/mod.rs @@ -364,6 +364,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { expr, pattern, escape_char, + any, } => self.sql_like_to_expr( negated, *expr, @@ -372,6 +373,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { schema, planner_context, false, + any, ), SQLExpr::ILike { @@ -379,6 +381,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { expr, pattern, escape_char, + any, } => self.sql_like_to_expr( negated, *expr, @@ -387,6 +390,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { schema, planner_context, true, + any, ), SQLExpr::SimilarTo { @@ -529,6 +533,9 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { left, compare_op, right, + // ANY/SOME are equivalent, this field specifies which the user + // specified but it doesn't affect the plan so ignore the field + is_some: _, } => { let mut binary_expr = RawBinaryExpr { op: compare_op, @@ -776,7 +783,11 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { schema: &DFSchema, planner_context: &mut PlannerContext, case_insensitive: bool, + any: bool, ) -> Result { + if any { + return not_impl_err!("ANY in LIKE expression"); + } let pattern = self.sql_expr_to_logical_expr(pattern, schema, planner_context)?; let pattern_type = pattern.get_type(schema)?; if pattern_type != DataType::Utf8 && pattern_type != DataType::Null { diff --git a/datafusion/sql/src/parser.rs b/datafusion/sql/src/parser.rs index 8a984f1645e9..bd1ed3145ef5 100644 --- a/datafusion/sql/src/parser.rs +++ b/datafusion/sql/src/parser.rs @@ -141,7 +141,7 @@ pub enum CopyToSource { /// `COPY TO ...` Relation(ObjectName), /// COPY (...query...) TO ... - Query(Query), + Query(Box), } impl fmt::Display for CopyToSource { @@ -1444,7 +1444,7 @@ mod tests { }; let query = if let SQLStatement::Query(query) = statement { - *query + query } else { panic!("Expected query, got {statement:?}"); }; diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs index 00949aa13ae1..96ac44821abe 100644 --- a/datafusion/sql/src/statement.rs +++ b/datafusion/sql/src/statement.rs @@ -152,6 +152,10 @@ fn calc_inline_constraints_from_columns(columns: &[ColumnDef]) -> Vec {} } } @@ -672,6 +676,9 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { full, db_name, filter, + // specifies IN/FROM, both of which DataFusion trets the same + // so ignored + clause: _, } => self.show_tables_to_plan(extended, full, db_name, filter), Statement::ShowColumns { @@ -1103,7 +1110,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { (plan, input_schema, Some(table_ref)) } CopyToSource::Query(query) => { - let plan = self.query_to_plan(query, &mut PlannerContext::new())?; + let plan = self.query_to_plan(*query, &mut PlannerContext::new())?; let input_schema = Arc::clone(plan.schema()); (plan, input_schema, None) } diff --git a/datafusion/sql/src/unparser/ast.rs b/datafusion/sql/src/unparser/ast.rs index 2de1ce9125a7..cc0812cd71e1 100644 --- a/datafusion/sql/src/unparser/ast.rs +++ b/datafusion/sql/src/unparser/ast.rs @@ -241,6 +241,7 @@ impl SelectBuilder { pub fn build(&self) -> Result { Ok(ast::Select { distinct: self.distinct.clone(), + top_before_distinct: false, top: self.top.clone(), projection: self.projection.clone(), into: self.into.clone(), diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rs index 0678e7d0306c..8f6ffa51f76a 100644 --- a/datafusion/sql/src/unparser/expr.rs +++ b/datafusion/sql/src/unparser/expr.rs @@ -282,6 +282,7 @@ impl Unparser<'_> { expr: Box::new(self.expr_to_sql_inner(expr)?), pattern: Box::new(self.expr_to_sql_inner(pattern)?), escape_char: escape_char.map(|c| c.to_string()), + any: false, }), Expr::AggregateFunction(agg) => { let func_name = agg.func.name(); From d925806406a908439edf382c415cc52afec50fdb Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 6 Nov 2024 17:26:55 -0500 Subject: [PATCH 3/3] Update for better error messages --- datafusion/sqllogictest/test_files/distinct_on.slt | 2 +- datafusion/sqllogictest/test_files/select.slt | 4 ++-- datafusion/sqllogictest/test_files/unnest.slt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/datafusion/sqllogictest/test_files/distinct_on.slt b/datafusion/sqllogictest/test_files/distinct_on.slt index 604ac95ff476..cc0ebf83a843 100644 --- a/datafusion/sqllogictest/test_files/distinct_on.slt +++ b/datafusion/sqllogictest/test_files/distinct_on.slt @@ -153,7 +153,7 @@ b 1 29 -18218 994303988 5983957848665088916 204 9489 3275293996 1485709125918647 c 2 1 18109 2033001162 -6513304855495910254 25 43062 1491205016 5863949479783605708 0.110830784 0.929409733247 6WfVFBVGJSQb7FhA7E0lBwdvjfZnSW # can't distinct on * -query error DataFusion error: SQL error: ParserError\("Expected: an expression:, found: \*"\) +query error DataFusion error: SQL error: ParserError\("Expected: an expression, found: \*"\) SELECT DISTINCT ON (*) c1 FROM aggregate_test_100 ORDER BY c1 LIMIT 3; diff --git a/datafusion/sqllogictest/test_files/select.slt b/datafusion/sqllogictest/test_files/select.slt index c096f6e692af..c687429ae6ec 100644 --- a/datafusion/sqllogictest/test_files/select.slt +++ b/datafusion/sqllogictest/test_files/select.slt @@ -339,10 +339,10 @@ NULL 1 statement error DataFusion error: SQL error: ParserError\("Expected: \(, found: EOF"\) VALUES -statement error DataFusion error: SQL error: ParserError\("Expected: an expression:, found: \)"\) +statement error DataFusion error: SQL error: ParserError\("Expected: an expression, found: \)"\) VALUES () -statement error DataFusion error: SQL error: ParserError\("Expected: an expression:, found: \)"\) +statement error DataFusion error: SQL error: ParserError\("Expected: an expression, found: \)"\) VALUES (1),() statement error DataFusion error: Error during planning: Inconsistent data length across values list: got 2 values in row 1 but expected 1 diff --git a/datafusion/sqllogictest/test_files/unnest.slt b/datafusion/sqllogictest/test_files/unnest.slt index 947eb8630b52..8ebed5b25ca9 100644 --- a/datafusion/sqllogictest/test_files/unnest.slt +++ b/datafusion/sqllogictest/test_files/unnest.slt @@ -295,7 +295,7 @@ query error DataFusion error: Error during planning: unnest\(\) requires exactly select unnest(); ## Unnest empty expression in from clause -query error DataFusion error: SQL error: ParserError\("Expected: an expression:, found: \)"\) +query error DataFusion error: SQL error: ParserError\("Expected: an expression, found: \)"\) select * from unnest();