diff --git a/Cargo.lock b/Cargo.lock index c7f2804cff3f..7ca2f19bb545 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2409,7 +2409,7 @@ dependencies = [ [[package]] name = "sqlparser" version = "0.16.0" -source = "git+https://github.com/cube-js/sqlparser-rs.git?rev=34f22de680caa5fe586def5b336d56efe43c8cc4#34f22de680caa5fe586def5b336d56efe43c8cc4" +source = "git+https://github.com/cube-js/sqlparser-rs.git?rev=e14d5bf45367edd8679cbc15ccee56693da8e4fb#e14d5bf45367edd8679cbc15ccee56693da8e4fb" dependencies = [ "log", ] diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index be46ed28a508..677c4543e492 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -1531,7 +1531,7 @@ checksum = "45456094d1983e2ee2a18fdfebce3189fa451699d0502cb8e3b49dba5ba41451" [[package]] name = "sqlparser" version = "0.16.0" -source = "git+https://github.com/cube-js/sqlparser-rs.git?rev=34f22de680caa5fe586def5b336d56efe43c8cc4#34f22de680caa5fe586def5b336d56efe43c8cc4" +source = "git+https://github.com/cube-js/sqlparser-rs.git?rev=e14d5bf45367edd8679cbc15ccee56693da8e4fb#e14d5bf45367edd8679cbc15ccee56693da8e4fb" dependencies = [ "log", ] diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml index 0b26c392786c..f10ed580e2e3 100644 --- a/datafusion/common/Cargo.toml +++ b/datafusion/common/Cargo.toml @@ -44,4 +44,4 @@ cranelift-module = { version = "0.82.0", optional = true } ordered-float = "2.10" parquet = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "a03d4eef5640e05dddf99fc2357ad6d58b5337cb", features = ["arrow"], optional = true } pyo3 = { version = "0.16", optional = true } -sqlparser = { git = 'https://github.com/cube-js/sqlparser-rs.git', rev = "34f22de680caa5fe586def5b336d56efe43c8cc4" } +sqlparser = { git = 'https://github.com/cube-js/sqlparser-rs.git', rev = "e14d5bf45367edd8679cbc15ccee56693da8e4fb" } diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index d3d6a1d4c4d3..82bcefd7b0d8 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -79,7 +79,7 @@ pin-project-lite= "^0.2.7" pyo3 = { version = "0.16", optional = true } rand = "0.8" smallvec = { version = "1.6", features = ["union"] } -sqlparser = { git = 'https://github.com/cube-js/sqlparser-rs.git', rev = "34f22de680caa5fe586def5b336d56efe43c8cc4" } +sqlparser = { git = 'https://github.com/cube-js/sqlparser-rs.git', rev = "e14d5bf45367edd8679cbc15ccee56693da8e4fb" } tempfile = "3" tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync", "fs", "parking_lot"] } tokio-stream = "0.1" diff --git a/datafusion/core/src/execution/context.rs b/datafusion/core/src/execution/context.rs index 4f5ba06a9252..dcd2adca7fb5 100644 --- a/datafusion/core/src/execution/context.rs +++ b/datafusion/core/src/execution/context.rs @@ -2402,6 +2402,23 @@ mod tests { ]; assert_batches_eq!(expected, &result); + let result = ctx + .sql("SELECT 1 UNION DISTINCT SELECT 1") + .await + .unwrap() + .collect() + .await + .unwrap(); + + let expected = vec![ + "+----------+", + "| Int64(1) |", + "+----------+", + "| 1 |", + "+----------+", + ]; + assert_batches_eq!(expected, &result); + Ok(()) } diff --git a/datafusion/core/src/sql/planner.rs b/datafusion/core/src/sql/planner.rs index 0245249a314c..ef4ecd1c6bb2 100644 --- a/datafusion/core/src/sql/planner.rs +++ b/datafusion/core/src/sql/planner.rs @@ -57,8 +57,8 @@ use sqlparser::ast::{ ArrayAgg, BinaryOperator, DataType as SQLDataType, DateTimeField, Expr as SQLExpr, Fetch, FunctionArg, FunctionArgExpr, Ident, Join, JoinConstraint, JoinOperator, ObjectName, Offset as SQLOffset, Query, Select, SelectItem, SetExpr, SetOperator, - ShowStatementFilter, TableFactor, TableWithJoins, TrimWhereField, UnaryOperator, - Value, Values as SQLValues, WithinGroup, + SetOperatorOption, ShowStatementFilter, TableFactor, TableWithJoins, TrimWhereField, + UnaryOperator, Value, Values as SQLValues, WithinGroup, }; use sqlparser::ast::{ColumnDef as SQLColumnDef, ColumnOption}; use sqlparser::ast::{ObjectType, OrderByExpr, Statement}; @@ -365,27 +365,34 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { op, left, right, - all, + option, } => { let left_plan = self.set_expr_to_plan(*left, None)?; let right_plan = self.set_expr_to_plan(*right, None)?; - match (op, all) { - (SetOperator::Union, true) => LogicalPlanBuilder::from(left_plan) - .union(right_plan)? - .build(), - (SetOperator::Union, false) => LogicalPlanBuilder::from(left_plan) - .union_distinct(right_plan)? - .build(), - (SetOperator::Intersect, true) => { + match (op, option) { + (SetOperator::Union, Some(SetOperatorOption::All)) => { + LogicalPlanBuilder::from(left_plan) + .union(right_plan)? + .build() + } + (SetOperator::Union, None) + | (SetOperator::Union, Some(SetOperatorOption::Distinct)) => { + LogicalPlanBuilder::from(left_plan) + .union_distinct(right_plan)? + .build() + } + (SetOperator::Intersect, Some(SetOperatorOption::All)) => { LogicalPlanBuilder::intersect(left_plan, right_plan, true) } - (SetOperator::Intersect, false) => { + (SetOperator::Intersect, None) + | (SetOperator::Intersect, Some(SetOperatorOption::Distinct)) => { LogicalPlanBuilder::intersect(left_plan, right_plan, false) } - (SetOperator::Except, true) => { + (SetOperator::Except, Some(SetOperatorOption::All)) => { LogicalPlanBuilder::except(left_plan, right_plan, true) } - (SetOperator::Except, false) => { + (SetOperator::Except, None) + | (SetOperator::Except, Some(SetOperatorOption::Distinct)) => { LogicalPlanBuilder::except(left_plan, right_plan, false) } } diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.toml index 7e06573a397d..f760a0d3e26a 100644 --- a/datafusion/expr/Cargo.toml +++ b/datafusion/expr/Cargo.toml @@ -38,4 +38,4 @@ path = "src/lib.rs" ahash = { version = "0.7", default-features = false } arrow = { git = 'https://github.com/cube-js/arrow-rs.git', rev = "a03d4eef5640e05dddf99fc2357ad6d58b5337cb", features = ["prettyprint"] } datafusion-common = { path = "../common", version = "7.0.0" } -sqlparser = { git = 'https://github.com/cube-js/sqlparser-rs.git', rev = "34f22de680caa5fe586def5b336d56efe43c8cc4" } +sqlparser = { git = 'https://github.com/cube-js/sqlparser-rs.git', rev = "e14d5bf45367edd8679cbc15ccee56693da8e4fb" }