merge from main

Author: yuval-illumex

Cargo.toml

@@ -37,15 +37,18 @@ name = "sqlparser"
 path = "src/lib.rs"
 
 [features]
-default = ["std"]
+default = ["std", "recursive-protection"]
 std = []
+recursive-protection = ["std", "recursive"]
 # Enable JSON output in the `cli` example:
 json_example = ["serde_json", "serde"]
 visitor = ["sqlparser_derive"]
 
 [dependencies]
 bigdecimal = { version = "0.4.1", features = ["serde"], optional = true }
 log = "0.4"
+recursive = { version = "0.1.1", optional = true}
+
 serde = { version = "1.0", features = ["derive"], optional = true }
 # serde_json is only used in examples/cli, but we have to put it outside
 # of dev-dependencies because of

README.md

@@ -63,7 +63,7 @@ The following optional [crate  features](https://doc.rust-lang.org/cargo/referen
 
 * `serde`: Adds [Serde](https://serde.rs/) support by implementing  `Serialize` and `Deserialize` for all AST nodes.
 * `visitor`: Adds a `Visitor` capable of recursively walking the AST tree.
-
+* `recursive-protection` (enabled by default), uses [recursive](https://docs.rs/recursive/latest/recursive/) for stack overflow protection. 
 
 ## Syntax vs Semantics
 

derive/src/lib.rs

@@ -78,7 +78,10 @@ fn derive_visit(input: proc_macro::TokenStream, visit_type: &VisitType) -> proc_
 
     let expanded = quote! {
         // The generated impl.
+        // Note that it uses [`recursive::recursive`] to protect from stack overflow.
+        // See tests in https://github.com/apache/datafusion-sqlparser-rs/pull/1522/ for more info.
         impl #impl_generics sqlparser::ast::#visit_trait for #name #ty_generics #where_clause {
+             #[cfg_attr(feature = "recursive-protection", recursive::recursive)]
             fn visit<V: sqlparser::ast::#visitor_trait>(
                 &#modifier self,
                 visitor: &mut V

sqlparser_bench/benches/sqlparser_bench.rs

@@ -42,6 +42,46 @@ fn basic_queries(c: &mut Criterion) {
     group.bench_function("sqlparser::with_select", |b| {
         b.iter(|| Parser::parse_sql(&dialect, with_query).unwrap());
     });
+
+    let large_statement = {
+        let expressions = (0..1000)
+            .map(|n| format!("FN_{}(COL_{})", n, n))
+            .collect::<Vec<_>>()
+            .join(", ");
+        let tables = (0..1000)
+            .map(|n| format!("TABLE_{}", n))
+            .collect::<Vec<_>>()
+            .join(" JOIN ");
+        let where_condition = (0..1000)
+            .map(|n| format!("COL_{} = {}", n, n))
+            .collect::<Vec<_>>()
+            .join(" OR ");
+        let order_condition = (0..1000)
+            .map(|n| format!("COL_{} DESC", n))
+            .collect::<Vec<_>>()
+            .join(", ");
+
+        format!(
+            "SELECT {} FROM {} WHERE {} ORDER BY {}",
+            expressions, tables, where_condition, order_condition
+        )
+    };
+
+    group.bench_function("parse_large_statement", |b| {
+        b.iter(|| Parser::parse_sql(&dialect, criterion::black_box(large_statement.as_str())));
+    });
+
+    let large_statement = Parser::parse_sql(&dialect, large_statement.as_str())
+        .unwrap()
+        .pop()
+        .unwrap();
+
+    group.bench_function("format_large_statement", |b| {
+        b.iter(|| {
+            let formatted_query = large_statement.to_string();
+            assert_eq!(formatted_query, large_statement);
+        });
+    });
 }
 
 criterion_group!(benches, basic_queries);

src/ast/data_type.rs

@@ -254,7 +254,7 @@ pub enum DataType {
     /// [postgresql]: https://www.postgresql.org/docs/15/datatype.html
     Float8,
     /// Double
-    Double,
+    Double(ExactNumberInfo),
     /// Double PRECISION e.g. [standard], [postgresql]
     ///
     /// [standard]: https://jakewheat.github.io/sql-overview/sql-2016-foundation-grammar.html#approximate-numeric-type
@@ -508,7 +508,7 @@ impl fmt::Display for DataType {
             DataType::Float4 => write!(f, "FLOAT4"),
             DataType::Float32 => write!(f, "Float32"),
             DataType::Float64 => write!(f, "FLOAT64"),
-            DataType::Double => write!(f, "DOUBLE"),
+            DataType::Double(info) => write!(f, "DOUBLE{info}"),
             DataType::Float8 => write!(f, "FLOAT8"),
             DataType::DoublePrecision => write!(f, "DOUBLE PRECISION"),
             DataType::Bool => write!(f, "BOOL"),

src/ast/ddl.rs

@@ -885,12 +885,14 @@ impl fmt::Display for TableConstraint {
             } => {
                 write!(
                     f,
-                    "{}FOREIGN KEY ({}) REFERENCES {}({})",
+                    "{}FOREIGN KEY ({}) REFERENCES {}",
                     display_constraint_name(name),
                     display_comma_separated(columns),
                     foreign_table,
-                    display_comma_separated(referred_columns),
                 )?;
+                if !referred_columns.is_empty() {
+                    write!(f, "({})", display_comma_separated(referred_columns))?;
+                }
                 if let Some(action) = on_delete {
                     write!(f, " ON DELETE {action}")?;
                 }

src/ast/mod.rs

@@ -1291,6 +1291,7 @@ impl fmt::Display for CastFormat {
 }
 
 impl fmt::Display for Expr {
+    #[cfg_attr(feature = "recursive-protection", recursive::recursive)]
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
             Expr::Identifier(s) => write!(f, "{s}"),
@@ -2354,6 +2355,7 @@ pub enum Statement {
         cache_metadata: bool,
         noscan: bool,
         compute_statistics: bool,
+        has_table_keyword: bool,
     },
     /// ```sql
     /// TRUNCATE
@@ -3239,6 +3241,9 @@ pub enum Statement {
         ///
         /// [SQLite](https://sqlite.org/lang_explain.html)
         query_plan: bool,
+        /// `EXPLAIN ESTIMATE`
+        /// [Clickhouse](https://clickhouse.com/docs/en/sql-reference/statements/explain#explain-estimate)
+        estimate: bool,
         /// A SQL query that specifies what to explain
         statement: Box<Statement>,
         /// Optional output format of explain
@@ -3471,6 +3476,7 @@ impl fmt::Display for Statement {
                 verbose,
                 analyze,
                 query_plan,
+                estimate,
                 statement,
                 format,
                 options,
@@ -3483,6 +3489,9 @@ impl fmt::Display for Statement {
                 if *analyze {
                     write!(f, "ANALYZE ")?;
                 }
+                if *estimate {
+                    write!(f, "ESTIMATE ")?;
+                }
 
                 if *verbose {
                     write!(f, "VERBOSE ")?;
@@ -3644,8 +3653,13 @@ impl fmt::Display for Statement {
                 cache_metadata,
                 noscan,
                 compute_statistics,
+                has_table_keyword,
             } => {
-                write!(f, "ANALYZE TABLE {table_name}")?;
+                write!(
+                    f,
+                    "ANALYZE{}{table_name}",
+                    if *has_table_keyword { " TABLE " } else { " " }
+                )?;
                 if let Some(ref parts) = partitions {
                     if !parts.is_empty() {
                         write!(f, " PARTITION ({})", display_comma_separated(parts))?;

src/ast/spans.rs

@@ -284,6 +284,7 @@ impl Spanned for Statement {
                 cache_metadata: _,
                 noscan: _,
                 compute_statistics: _,
+                has_table_keyword: _,
             } => union_spans(
                 core::iter::once(table_name.span())
                     .chain(partitions.iter().flat_map(|i| i.iter().map(|k| k.span())))

src/ast/visitor.rs

@@ -894,4 +894,29 @@ mod tests {
             assert_eq!(actual, expected)
         }
     }
+
+    struct QuickVisitor; // [`TestVisitor`] is too slow to iterate over thousands of nodes
+
+    impl Visitor for QuickVisitor {
+        type Break = ();
+    }
+
+    #[test]
+    fn overflow() {
+        let cond = (0..1000)
+            .map(|n| format!("X = {}", n))
+            .collect::<Vec<_>>()
+            .join(" OR ");
+        let sql = format!("SELECT x where {0}", cond);
+
+        let dialect = GenericDialect {};
+        let tokens = Tokenizer::new(&dialect, sql.as_str()).tokenize().unwrap();
+        let s = Parser::new(&dialect)
+            .with_tokens(tokens)
+            .parse_statement()
+            .unwrap();
+
+        let mut visitor = QuickVisitor {};
+        s.visit(&mut visitor);
+    }
 }

src/dialect/snowflake.rs

@@ -377,6 +377,10 @@ pub fn parse_create_table(
                     parser.expect_token(&Token::RParen)?;
                     builder = builder.with_tags(Some(tags));
                 }
+                Keyword::ON if parser.parse_keyword(Keyword::COMMIT) => {
+                    let on_commit = Some(parser.parse_create_table_on_commit()?);
+                    builder = builder.on_commit(on_commit);
+                }
                 _ => {
                     return parser.expected("end of statement", next_token);
                 }