Merge branch 'main' into dev

.github/workflows/rust.yml

@@ -37,6 +37,14 @@ jobs:
         uses: ./.github/actions/setup-builder
       - run: cargo clippy --all-targets --all-features -- -D warnings
 
+  benchmark-lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust Toolchain
+        uses: ./.github/actions/setup-builder
+      - run: cd sqlparser_bench && cargo clippy --all-targets --all-features -- -D warnings
+
   compile:
     runs-on: ubuntu-latest
     steps:

Cargo.toml

@@ -37,15 +37,18 @@ name = "sqlparser"
 path = "src/lib.rs"
 
 [features]
-default = ["std"]
+default = ["std", "recursive-protection"]
 std = []
+recursive-protection = ["std", "recursive"]
 # Enable JSON output in the `cli` example:
 json_example = ["serde_json", "serde"]
 visitor = ["sqlparser_derive"]
 
 [dependencies]
 bigdecimal = { version = "0.4.1", features = ["serde"], optional = true }
 log = "0.4"
+recursive = { version = "0.1.1", optional = true}
+
 serde = { version = "1.0", features = ["derive"], optional = true }
 # serde_json is only used in examples/cli, but we have to put it outside
 # of dev-dependencies because of

README.md

@@ -71,7 +71,7 @@ The following optional [crate  features](https://doc.rust-lang.org/cargo/referen
 
 * `serde`: Adds [Serde](https://serde.rs/) support by implementing  `Serialize` and `Deserialize` for all AST nodes.
 * `visitor`: Adds a `Visitor` capable of recursively walking the AST tree.
-
+* `recursive-protection` (enabled by default), uses [recursive](https://docs.rs/recursive/latest/recursive/) for stack overflow protection. 
 
 ## Syntax vs Semantics
 
@@ -248,11 +248,14 @@ You can run them with:
 ```
 git checkout main
 cd sqlparser_bench
-cargo bench
+cargo bench -- --save-baseline main
 git checkout <your branch>
-cargo bench
+cargo bench -- --baseline main
 ```
 
+By adding the `--save-baseline main` and `--baseline main` you can track the
+progress of your improvements as you continue working on the feature branch.
+
 ## Licensing
 
 All code in this repository is licensed under the [Apache Software License 2.0](LICENSE.txt).

derive/src/lib.rs

@@ -78,7 +78,10 @@ fn derive_visit(input: proc_macro::TokenStream, visit_type: &VisitType) -> proc_
 
     let expanded = quote! {
         // The generated impl.
+        // Note that it uses [`recursive::recursive`] to protect from stack overflow.
+        // See tests in https://github.com/apache/datafusion-sqlparser-rs/pull/1522/ for more info.
         impl #impl_generics sqlparser::ast::#visit_trait for #name #ty_generics #where_clause {
+             #[cfg_attr(feature = "recursive-protection", recursive::recursive)]
             fn visit<V: sqlparser::ast::#visitor_trait>(
                 &#modifier self,
                 visitor: &mut V

sqlparser_bench/README.md

@@ -17,4 +17,26 @@
   under the License.
 -->
 
-Benchmarks for sqlparser. See [the main README](../README.md) for more information.
+Benchmarks for sqlparser. See [the main README](../README.md) for more information.
+
+Note: this is in a separate, non workspace crate to avoid adding a dependency 
+on `criterion` to the main crate (which complicates testing without std).
+
+# Running Benchmarks
+
+```shell
+cargo bench --bench sqlparser_bench
+```
+
+# Profiling
+
+Note you can generate a [flamegraph] using the following command:
+
+```shell
+cargo flamegraph --bench sqlparser_bench
+```
+
+[flamegraph]: https://crates.io/crates/flamegraph
+
+Here is an example flamegraph:
+![flamegraph](img/flamegraph.svg)

sqlparser_bench/benches/sqlparser_bench.rs

@@ -42,6 +42,45 @@ fn basic_queries(c: &mut Criterion) {
     group.bench_function("sqlparser::with_select", |b| {
         b.iter(|| Parser::parse_sql(&dialect, with_query).unwrap());
     });
+
+    let large_statement = {
+        let expressions = (0..1000)
+            .map(|n| format!("FN_{}(COL_{})", n, n))
+            .collect::<Vec<_>>()
+            .join(", ");
+        let tables = (0..1000)
+            .map(|n| format!("TABLE_{}", n))
+            .collect::<Vec<_>>()
+            .join(" JOIN ");
+        let where_condition = (0..1000)
+            .map(|n| format!("COL_{} = {}", n, n))
+            .collect::<Vec<_>>()
+            .join(" OR ");
+        let order_condition = (0..1000)
+            .map(|n| format!("COL_{} DESC", n))
+            .collect::<Vec<_>>()
+            .join(", ");
+
+        format!(
+            "SELECT {} FROM {} WHERE {} ORDER BY {}",
+            expressions, tables, where_condition, order_condition
+        )
+    };
+
+    group.bench_function("parse_large_statement", |b| {
+        b.iter(|| Parser::parse_sql(&dialect, criterion::black_box(large_statement.as_str())));
+    });
+
+    let large_statement = Parser::parse_sql(&dialect, large_statement.as_str())
+        .unwrap()
+        .pop()
+        .unwrap();
+
+    group.bench_function("format_large_statement", |b| {
+        b.iter(|| {
+            let _formatted_query = large_statement.to_string();
+        });
+    });
 }
 
 criterion_group!(benches, basic_queries);