Merge branch 'main' into feature/column-extraction

Author: teaguesterling

.github/workflows/MainDistributionPipeline.yml

@@ -12,18 +12,18 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-#  duckdb-next-build:
-#    name: Build extension binaries
-#    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main
-#    with:
-#      duckdb_version: main
-#      ci_tools_version: main
-#      extension_name: parser_tools
+  duckdb-next-build:
+   name: Build extension binaries
+   uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main
+   with:
+     duckdb_version: main
+     ci_tools_version: main
+     extension_name: parser_tools
 
   duckdb-stable-build:
     name: Build extension binaries
-    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.3.0
+    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.4.0
     with:
-      duckdb_version: v1.3.0
-      ci_tools_version: v1.3.0
+      duckdb_version: v1.4.0
+      ci_tools_version: v1.4.0
       extension_name: parser_tools

CMakeLists.txt

@@ -9,12 +9,13 @@ set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension)
 project(${TARGET_NAME})
 include_directories(src/include)
 
-set(EXTENSION_SOURCES 
+set(EXTENSION_SOURCES
   src/parser_tools_extension.cpp
   src/parse_tables.cpp
   src/parse_where.cpp
   src/parse_functions.cpp
   src/parse_columns.cpp
+  src/parse_statements.cpp
 )
 
 build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES})

README.md

@@ -4,14 +4,15 @@ An experimental DuckDB extension that exposes functionality from DuckDB's native
 
 ## Overview
 
-`parser_tools` is a DuckDB extension designed to provide SQL parsing capabilities within the database. It allows you to analyze SQL queries and extract structural information directly in SQL. This extension provides parsing functions for tables, WHERE clauses, and function calls (see [Functions](#functions) below).
+`parser_tools` is a DuckDB extension designed to provide SQL parsing capabilities within the database. It allows you to analyze SQL queries and extract structural information directly in SQL. This extension provides parsing functions for tables, WHERE clauses, function calls, and statements.
 
 ## Features
 
 - **Extract table references** from a SQL query with context information (e.g. `FROM`, `JOIN`, etc.)
 - **Extract function calls** from a SQL query with context information (e.g. `SELECT`, `WHERE`, `HAVING`, etc.)
 - **Extract column references** from a SQL query with comprehensive dependency tracking
 - **Parse WHERE clauses** to extract conditions and operators
+- **Parse multi-statement SQL** to extract individual statements or count the number of statements
 - Support for **window functions**, **nested functions**, and **CTEs**
 - **Alias chain tracking** for complex column dependencies
 - **Nested struct field access** parsing (e.g., `table.column.field.subfield`)
@@ -106,7 +107,7 @@ Context helps identify where elements are used in the query.
 
 ## Functions
 
-This extension provides parsing functions for tables, functions, columns, and WHERE clauses. Each category includes both table functions (for detailed results) and scalar functions (for programmatic use).
+This extension provides parsing functions for tables, functions, columns, statements, and WHERE clauses. Each category includes both table functions (for detailed results) and scalar functions (for programmatic use).
 
 In general, errors (e.g. Parse Exception) will not be exposed to the user, but instead will result in an empty result. This simplifies batch processing. When validity is needed, [is_parsable](#is_parsablesql_query--scalar-function) can be used.
 
@@ -395,6 +396,92 @@ FROM (VALUES
 └───────────────────────────────────────────────┴────────┘
 ```
 
+---
+
+### Statement Parsing Functions
+
+These functions parse multi-statement SQL strings and extract individual statements or count them.
+
+#### `parse_statements(sql_query)` – Table Function
+
+Parses a SQL string containing multiple statements and returns each statement as a separate row.
+
+##### Usage
+```sql
+SELECT * FROM parse_statements('SELECT 42; SELECT 43;');
+```
+
+##### Returns
+A table with:
+- `statement`: the SQL statement text
+
+##### Example
+```sql
+SELECT * FROM parse_statements($$
+    SELECT * FROM users WHERE active = true;
+    INSERT INTO log VALUES ('query executed');
+    SELECT count(*) FROM transactions;
+$$);
+```
+
+| statement |
+|-----------|
+| SELECT * FROM users WHERE (active = true) |
+| INSERT INTO log (VALUES ('query executed')) |
+| SELECT count_star() FROM transactions |
+
+---
+
+#### `parse_statements(sql_query)` – Scalar Function
+
+Returns a list of statement strings from a multi-statement SQL query.
+
+##### Usage
+```sql
+SELECT parse_statements('SELECT 42; SELECT 43;');
+----
+[SELECT 42, SELECT 43]
+```
+
+##### Returns
+A list of strings, each being a SQL statement.
+
+##### Example
+```sql
+SELECT parse_statements('SELECT 1; INSERT INTO test VALUES (2); SELECT 3;');
+----
+[SELECT 1, 'INSERT INTO test (VALUES (2))', SELECT 3]
+```
+
+---
+
+#### `num_statements(sql_query)` – Scalar Function
+
+Returns the number of statements in a multi-statement SQL query.
+
+##### Usage
+```sql
+SELECT num_statements('SELECT 42; SELECT 43;');
+----
+2
+```
+
+##### Returns
+An integer count of the number of SQL statements.
+
+##### Example
+```sql
+SELECT num_statements($$
+    WITH cte AS (SELECT 1) SELECT * FROM cte;
+    UPDATE users SET last_seen = now();
+    SELECT count(*) FROM users;
+    DELETE FROM temp_data;
+$$);
+----
+4
+```
+
+---
 
 ## Development
 

duckdb

@@ -7,15 +7,15 @@
 namespace duckdb {
 
 // Forward declarations
-class DatabaseInstance;
+class ExtensionLoader;
 
 struct FunctionResult {
 	std::string function_name;
 	std::string schema;
 	std::string context;     // The context where this function appears (SELECT, WHERE, etc.)
 };
 
-void RegisterParseFunctionsFunction(DatabaseInstance &db);
-void RegisterParseFunctionScalarFunction(DatabaseInstance &db);
+void RegisterParseFunctionsFunction(ExtensionLoader &loader);
+void RegisterParseFunctionScalarFunction(ExtensionLoader &loader);
 
 } // namespace duckdb

src/include/parse_functions.hpp

@@ -0,0 +1,19 @@
+#pragma once
+
+#include "duckdb.hpp"
+#include <string>
+#include <vector>
+
+namespace duckdb {
+
+// Forward declarations
+class ExtensionLoader;
+
+struct StatementResult {
+	std::string statement;
+};
+
+void RegisterParseStatementsFunction(ExtensionLoader &loader);
+void RegisterParseStatementsScalarFunction(ExtensionLoader &loader);
+
+} // namespace duckdb

src/include/parse_statements.hpp

@@ -33,7 +33,7 @@ static void ExtractTablesFromQueryNode(
     const duckdb::CommonTableExpressionMap *cte_map = nullptr
 );
 
-void RegisterParseTablesFunction(duckdb::DatabaseInstance &db);
-void RegisterParseTableScalarFunction(DatabaseInstance &db);
+void RegisterParseTablesFunction(duckdb::ExtensionLoader &loader);
+void RegisterParseTableScalarFunction(ExtensionLoader &loader);
 
 } // namespace duckdb

src/include/parse_tables.hpp

@@ -7,7 +7,7 @@
 namespace duckdb {
 
 // Forward declarations
-class DatabaseInstance;
+class ExtensionLoader;
 
 struct WhereConditionResult {
     std::string condition;
@@ -23,8 +23,8 @@ struct DetailedWhereConditionResult {
     std::string context;        // The context where this condition appears (WHERE, HAVING, etc.)
 };
 
-void RegisterParseWhereFunction(DatabaseInstance &db);
-void RegisterParseWhereScalarFunction(DatabaseInstance &db);
-void RegisterParseWhereDetailedFunction(DatabaseInstance &db);
+void RegisterParseWhereFunction(ExtensionLoader &loader);
+void RegisterParseWhereScalarFunction(ExtensionLoader &loader);
+void RegisterParseWhereDetailedFunction(ExtensionLoader &loader);
 
 } // namespace duckdb 

src/include/parse_where.hpp

@@ -6,7 +6,7 @@ namespace duckdb {
 
 class ParserToolsExtension : public Extension {
 public:
-	void Load(DuckDB &db) override;
+	void Load(ExtensionLoader &loader) override;
 	std::string Name() override;
 	std::string Version() const override;
 };

src/include/parser_tools_extension.hpp

@@ -2,12 +2,12 @@
 #include "duckdb.hpp"
 #include "duckdb/parser/parser.hpp"
 #include "duckdb/parser/statement/select_statement.hpp"
+#include "duckdb/parser/query_node/cte_node.hpp"
 #include "duckdb/parser/query_node/select_node.hpp"
 #include "duckdb/parser/expression/function_expression.hpp"
 #include "duckdb/parser/expression/window_expression.hpp"
 #include "duckdb/parser/parsed_expression_iterator.hpp"
 #include "duckdb/parser/result_modifier.hpp"
-#include "duckdb/main/extension_util.hpp"
 #include "duckdb/function/scalar/nested_functions.hpp"
 
 
@@ -202,7 +202,14 @@ static void ExtractFunctionsFromQueryNode(const QueryNode &node, std::vector<Fun
 				}
 			}
 		}
-	}
+	// additional step necessary for duckdb v1.4.0: unwrap CTE node
+	}  else if (node.type == QueryNodeType::CTE_NODE) {
+        auto &cte_node = (CTENode &)node;
+
+        if (cte_node.child) {
+            ExtractFunctionsFromQueryNode(*cte_node.child, results);
+        }
+    }
 }
 
 static void ExtractFunctionsFromSQL(const std::string &sql, std::vector<FunctionResult> &results) {
@@ -328,15 +335,15 @@ static void ParseFunctionsScalarFunction_struct(DataChunk &args, ExpressionState
 // Extension scaffolding
 // ---------------------------------------------------
 
-void RegisterParseFunctionsFunction(DatabaseInstance &db) {
+void RegisterParseFunctionsFunction(ExtensionLoader &loader) {
 	TableFunction tf("parse_functions", {LogicalType::VARCHAR}, ParseFunctionsFunction, ParseFunctionsBind, ParseFunctionsInit);
-	ExtensionUtil::RegisterFunction(db, tf);
+	loader.RegisterFunction(tf);
 }
 
-void RegisterParseFunctionScalarFunction(DatabaseInstance &db) {
+void RegisterParseFunctionScalarFunction(ExtensionLoader &loader) {
 	// parse_function_names is a scalar function that returns a list of function names
 	ScalarFunction sf("parse_function_names", {LogicalType::VARCHAR}, LogicalType::LIST(LogicalType::VARCHAR), ParseFunctionNamesScalarFunction);
-	ExtensionUtil::RegisterFunction(db, sf);
+	loader.RegisterFunction(sf);
 
 	// parse_functions_struct is a scalar function that returns a list of structs
 	auto return_type = LogicalType::LIST(LogicalType::STRUCT({
@@ -345,7 +352,7 @@ void RegisterParseFunctionScalarFunction(DatabaseInstance &db) {
 		{"context", LogicalType::VARCHAR}
 	}));
 	ScalarFunction sf_struct("parse_functions", {LogicalType::VARCHAR}, return_type, ParseFunctionsScalarFunction_struct);
-	ExtensionUtil::RegisterFunction(db, sf_struct);
+	loader.RegisterFunction(sf_struct);
 }