Duckdb 1.2 update

.github/workflows/build_and_test.yaml

@@ -22,7 +22,7 @@ jobs:
       - name: Install clang-format and ruff
         run: python3 -m pip install -r dev_requirements.txt
       - name: Run clang-format
-        run: git clang-format refs/remotes/origin/main --diff
+        run: find src include -iname '*.hpp' -o -iname '*.h' -o -iname '*.cpp' -o -iname '*.c' | xargs git clang-format --diff origin/main
       - name: Run ruff check
         run: ruff check --output-format=github .
       - name: Run ruff format

Makefile

@@ -13,7 +13,7 @@ OBJS += $(subst .c,.o, $(C_SRCS))
 # set to `make` to disable ninja
 DUCKDB_GEN ?= ninja
 # used to know what version of extensions to download
-DUCKDB_VERSION = v1.1.3
+DUCKDB_VERSION = v1.2.0
 # duckdb build tweaks
 DUCKDB_CMAKE_VARS = -DBUILD_SHELL=0 -DBUILD_PYTHON=0 -DBUILD_UNITTESTS=0
 # set to 1 to disable asserts in DuckDB. This is particularly useful in combinition with MotherDuck.
@@ -119,7 +119,7 @@ lintcheck:
 	ruff check
 
 format:
-	git clang-format origin/main
+	find src include -iname '*.hpp' -o -iname '*.h' -o -iname '*.cpp' -o -iname '*.c' | xargs git clang-format origin/main
 	ruff format
 
 format-all:

include/pgduckdb/scan/postgres_scan.hpp

@@ -22,6 +22,10 @@ struct PostgresScanGlobalState : public duckdb::GlobalTableFunctionState {
 	}
 	void ConstructTableScanQuery(const duckdb::TableFunctionInitInput &input);
 
+private:
+	int ExtractQueryFilters(duckdb::TableFilter *filter, const char *column_name, duckdb::string &filters,
+	                        bool is_optional_filter_parent);
+
 public:
 	Snapshot snapshot;
 	Relation rel;
@@ -50,7 +54,6 @@ struct PostgresScanLocalState : public duckdb::LocalTableFunctionState {
 struct PostgresScanFunctionData : public duckdb::TableFunctionData {
 	PostgresScanFunctionData(Relation rel, uint64_t cardinality, Snapshot snapshot);
 	~PostgresScanFunctionData() override;
-
 	duckdb::vector<duckdb::string> complex_filters;
 	Relation rel;
 	uint64_t cardinality;
@@ -74,7 +77,7 @@ struct PostgresScanTableFunction : public duckdb::TableFunction {
 
 	static duckdb::unique_ptr<duckdb::NodeStatistics> PostgresScanCardinality(duckdb::ClientContext &context,
 	                                                                          const duckdb::FunctionData *data);
-	static std::string ToString(const duckdb::FunctionData *bind_data);
+	static duckdb::InsertionOrderPreservingMap<duckdb::string> ToString(duckdb::TableFunctionToStringInput &input);
 };
 
 } // namespace pgduckdb

src/pgduckdb_types.cpp

@@ -797,7 +797,7 @@ ConvertDuckToPostgresValue(TupleTableSlot *slot, duckdb::Value &value, idx_t col
 		break;
 	}
 	case TIMESTAMPTZOID: {
-		duckdb::timestamp_t timestamp = value.GetValue<duckdb::timestamp_t>();
+		duckdb::timestamp_tz_t timestamp = value.GetValue<duckdb::timestamp_tz_t>();
 		slot->tts_values[col] = timestamp.value - pgduckdb::PGDUCKDB_DUCK_TIMESTAMP_OFFSET;
 		break;
 	}
@@ -1236,7 +1236,7 @@ ConvertPostgresParameterToDuckValue(Datum value, Oid postgres_type) {
 		return duckdb::Value::TIMESTAMP(duckdb::timestamp_t(DatumGetTimestamp(value) + PGDUCKDB_DUCK_TIMESTAMP_OFFSET));
 	case TIMESTAMPTZOID:
 		return duckdb::Value::TIMESTAMPTZ(
-		    duckdb::timestamp_t(DatumGetTimestampTz(value) + PGDUCKDB_DUCK_TIMESTAMP_OFFSET));
+		    duckdb::timestamp_tz_t(DatumGetTimestampTz(value) + PGDUCKDB_DUCK_TIMESTAMP_OFFSET));
 	case INTERVALOID:
 		return duckdb::Value::INTERVAL(DatumGetInterval(value));
 	case FLOAT4OID:
@@ -1305,8 +1305,8 @@ ConvertPostgresToDuckValue(Oid attr_type, Datum value, duckdb::Vector &result, i
 		    result, duckdb::timestamp_t(static_cast<int64_t>(value + PGDUCKDB_DUCK_TIMESTAMP_OFFSET)), offset);
 		break;
 	case duckdb::LogicalTypeId::TIMESTAMP_TZ:
-		Append<duckdb::timestamp_t>(
-		    result, duckdb::timestamp_t(static_cast<int64_t>(value + PGDUCKDB_DUCK_TIMESTAMP_OFFSET)), offset);
+		Append<duckdb::timestamp_tz_t>(
+		    result, duckdb::timestamp_tz_t(static_cast<int64_t>(value + PGDUCKDB_DUCK_TIMESTAMP_OFFSET)), offset);
 		break;
 	case duckdb::LogicalTypeId::INTERVAL:
 		Append<duckdb::interval_t>(result, DatumGetInterval(value), offset);

src/scan/postgres_scan.cpp

@@ -1,3 +1,5 @@
+#include "duckdb/planner/filter/optional_filter.hpp"
+
 #include "pgduckdb/scan/postgres_scan.hpp"
 #include "pgduckdb/scan/postgres_table_reader.hpp"
 #include "pgduckdb/pgduckdb_types.hpp"
@@ -7,12 +9,71 @@
 #include "pgduckdb/pgduckdb_process_lock.hpp"
 #include "pgduckdb/logger.hpp"
 
+#include <numeric> // std::accumulate
+
 namespace pgduckdb {
 
 //
 // PostgresScanGlobalState
 //
 
+static duckdb::string
+FilterJoin(duckdb::vector<duckdb::string> &filters, duckdb::string &&delimiter) {
+	return std::accumulate(filters.begin() + 1, filters.end(), filters[0],
+	                       [&delimiter](duckdb::string l, duckdb::string r) { return l + delimiter + r; });
+}
+
+int
+PostgresScanGlobalState::ExtractQueryFilters(duckdb::TableFilter *filter, const char *column_name,
+                                             duckdb::string &query_filters, bool is_inside_optional_filter) {
+	switch (filter->filter_type) {
+	case duckdb::TableFilterType::CONSTANT_COMPARISON:
+	case duckdb::TableFilterType::IS_NULL:
+	case duckdb::TableFilterType::IS_NOT_NULL:
+	case duckdb::TableFilterType::IN_FILTER: {
+		query_filters += filter->ToString(column_name).c_str();
+		return 1;
+	}
+	case duckdb::TableFilterType::CONJUNCTION_OR:
+	case duckdb::TableFilterType::CONJUNCTION_AND: {
+		auto conjuction_filter = reinterpret_cast<duckdb::ConjunctionFilter *>(filter);
+		duckdb::vector<std::string> conjuction_child_filters;
+		for (idx_t i = 0; i < conjuction_filter->child_filters.size(); i++) {
+			std::string child_filter;
+			if (ExtractQueryFilters(conjuction_filter->child_filters[i].get(), column_name, child_filter,
+			                        is_inside_optional_filter)) {
+				conjuction_child_filters.emplace_back(child_filter);
+			}
+		}
+		duckdb::string conjuction_delimiter =
+		    filter->filter_type == duckdb::TableFilterType::CONJUNCTION_OR ? " OR " : " AND ";
+		if (conjuction_child_filters.size()) {
+			query_filters += "(" + FilterJoin(conjuction_child_filters, std::move(conjuction_delimiter)) + ")";
+		}
+		return conjuction_child_filters.size();
+	}
+	case duckdb::TableFilterType::OPTIONAL_FILTER: {
+		auto optional_filter = reinterpret_cast<duckdb::OptionalFilter *>(filter);
+		return ExtractQueryFilters(optional_filter->child_filter.get(), column_name, query_filters, true);
+	}
+	/* DYNAMIC_FILTER is push down filter from topN execution. STRUCT_EXTRACT is
+	 * only received if struct_extract function is used. Default will catch all
+	 * filter that could be added in future in DuckDB.
+	 */
+	case duckdb::TableFilterType::DYNAMIC_FILTER:
+	case duckdb::TableFilterType::STRUCT_EXTRACT:
+	default: {
+		if (is_inside_optional_filter) {
+			pd_log(DEBUG1, "(DuckDB/ExtractQueryFilters) Unsupported optional filter: %s",
+			       filter->ToString(column_name).c_str());
+			return 0;
+		}
+		throw duckdb::Exception(duckdb::ExceptionType::EXECUTOR,
+		                        "Invalid Filter Type: " + filter->ToString(column_name));
+	}
+	}
+}
+
 void
 PostgresScanGlobalState::ConstructTableScanQuery(const duckdb::TableFunctionInitInput &input) {
 	/* SELECT COUNT(*) FROM */
@@ -82,27 +143,23 @@ PostgresScanGlobalState::ConstructTableScanQuery(const duckdb::TableFunctionInit
 
 	scan_query << " FROM " << GenerateQualifiedRelationName(rel);
 
-	first = true;
-
+	duckdb::vector<duckdb::string> query_filters;
 	for (auto const &[attr_num, duckdb_scanned_index] : columns_to_scan) {
 		auto filter = column_filters[duckdb_scanned_index];
-
 		if (!filter) {
 			continue;
 		}
-
-		if (first) {
-			scan_query << " WHERE ";
-		} else {
-			scan_query << " AND ";
-		}
-
-		first = false;
-		scan_query << "(";
+		duckdb::string column_query_filters;
 		auto attr = GetAttr(table_tuple_desc, attr_num - 1);
 		auto col = pgduckdb::QuoteIdentifier(GetAttName(attr));
-		scan_query << filter->ToString(col).c_str();
-		scan_query << ") ";
+		if (ExtractQueryFilters(filter, col, column_query_filters, false)) {
+			query_filters.emplace_back(column_query_filters);
+		};
+	}
+
+	if (query_filters.size()) {
+		scan_query << " WHERE ";
+		scan_query << FilterJoin(query_filters, " AND ");
 	}
 }
 
@@ -157,12 +214,12 @@ PostgresScanTableFunction::PostgresScanTableFunction()
 	to_string = ToString;
 }
 
-std::string
-PostgresScanTableFunction::ToString(const duckdb::FunctionData *data) {
-	auto &bind_data = data->Cast<PostgresScanFunctionData>();
-	std::ostringstream oss;
-	oss << "(POSTGRES_SCAN) " << GetRelationName(bind_data.rel);
-	return oss.str();
+duckdb::InsertionOrderPreservingMap<duckdb::string>
+PostgresScanTableFunction::ToString(duckdb::TableFunctionToStringInput &input) {
+	auto &bind_data = input.bind_data->Cast<PostgresScanFunctionData>();
+	duckdb::InsertionOrderPreservingMap<duckdb::string> result;
+	result["Table"] = GetRelationName(bind_data.rel);
+	return result;
 }
 
 duckdb::unique_ptr<duckdb::GlobalTableFunctionState>

test/pycheck/explain_test.py

@@ -28,7 +28,7 @@ def test_explain(cur: Cursor):
     result = cur.sql("EXPLAIN SELECT count(*) FROM test_table where id = %s", (1,))
     plan = "\n".join(result)
     assert "UNGROUPED_AGGREGATE" in plan
-    assert "id=1 AND id IS NOT NULL" in plan
+    assert "id=1" in plan
     assert "Total Time:" not in plan
     assert "Output:" not in plan
 
@@ -37,7 +37,7 @@ def test_explain(cur: Cursor):
     )
     plan = "\n".join(result)
     assert "UNGROUPED_AGGREGATE" in plan
-    assert "id=1 AND id IS NOT NULL" in plan
+    assert "id=1" in plan
     assert "Total Time:" in plan
     assert "Output:" not in plan
 
@@ -66,7 +66,7 @@ def test_explain_ctas(cur: Cursor):
         "EXPLAIN ANALYZE CREATE TEMP TABLE heap2(id) AS SELECT * from heap1"
     )
     plan = "\n".join(result)
-    assert "POSTGRES_SCAN" in plan
+    assert "TABLE_SCAN" in plan
     assert "Total Time:" in plan
 
     result = cur.sql(

test/pycheck/non_superuser_test.py

@@ -18,26 +18,26 @@ def test_community_extensions(pg: Postgres):
             match="Permission Error: File system LocalFileSystem has been disabled by configuration",
         ):
             cur.sql(
-                "SELECT * FROM duckdb.raw_query($$ INSTALL avro FROM community; $$)"
+                "SELECT * FROM duckdb.raw_query($$ INSTALL duckpgq FROM community; $$)"
             )
 
     # Even if such community extensions somehow get installed, it's not possible
     # to load them without changing allow_community_extensions. Not even for a
     # superuser.
     with pg.cur() as cur:
         cur.sql("SET duckdb.force_execution = false")
-        cur.sql("SELECT * FROM duckdb.raw_query($$ INSTALL avro FROM community; $$)")
+        cur.sql("SELECT * FROM duckdb.raw_query($$ INSTALL duckpgq FROM community; $$)")
         with pytest.raises(
             Exception,
             match="IO Error: Extension .* could not be loaded because its signature is either missing or invalid and unsigned extensions are disabled by configuration",
         ):
-            cur.sql("SELECT * FROM duckdb.raw_query($$ LOAD avro; $$)")
+            cur.sql("SELECT * FROM duckdb.raw_query($$ LOAD duckpgq; $$)")
 
     # But it should be possible to load them after changing that setting.
     with pg.cur() as cur:
         cur.sql("SET duckdb.allow_community_extensions = true")
         cur.sql("SET duckdb.force_execution = false")
-        cur.sql("SELECT * FROM duckdb.raw_query($$ LOAD avro; $$)")
+        cur.sql("SELECT * FROM duckdb.raw_query($$ LOAD duckpgq; $$)")
 
     # And that setting is only changeable by superusers
     with pg.cur() as cur:

test/regression/data/delta_table/_delta_log/00000000000000000000.json

@@ -1,3 +1,3 @@
-{"protocol":{"minReaderVersion":3,"minWriterVersion":7,"writerFeatures":[]}}
-{"metaData":{"id":"bec316ba-98b6-4d9c-8ad1-a41b4a786d9b","name":"delta_table","description":null,"format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"a\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"b\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"createdTime":1730972838217,"configuration":{"delta.minReaderVersion":"3","delta.minWriterVersion":"7"}}}
+{"protocol":{"minReaderVersion":1,"minWriterVersion":7,"writerFeatures":[]}}
+{"metaData":{"id":"bec316ba-98b6-4d9c-8ad1-a41b4a786d9b","name":"delta_table","description":null,"format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"a\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"b\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"createdTime":1730972838217,"configuration":{"delta.minReaderVersion":"1","delta.minWriterVersion":"7"}}}
 {"commitInfo":{"timestamp":1730972838217,"operation":"CREATE TABLE","operationParameters":{"metadata":"{\"configuration\":{\"delta.minReaderVersion\":\"3\",\"delta.minWriterVersion\":\"7\"},\"createdTime\":1730972838217,\"description\":null,\"format\":{\"options\":{},\"provider\":\"parquet\"},\"id\":\"bec316ba-98b6-4d9c-8ad1-a41b4a786d9b\",\"name\":\"delta_table\",\"partitionColumns\":[],\"schemaString\":\"{\\\"type\\\":\\\"struct\\\",\\\"fields\\\":[{\\\"name\\\":\\\"a\\\",\\\"type\\\":\\\"integer\\\",\\\"nullable\\\":true,\\\"metadata\\\":{}},{\\\"name\\\":\\\"b\\\",\\\"type\\\":\\\"string\\\",\\\"nullable\\\":true,\\\"metadata\\\":{}}]}\"}","mode":"ErrorIfExists","protocol":"{\"minReaderVersion\":3,\"minWriterVersion\":7,\"writerFeatures\":[]}"},"creator":"pg_duckdb","clientVersion":"delta-rs.0.21.0"}}

test/regression/expected/duckdb_recycle.out

@@ -15,7 +15,7 @@ EXPLAIN SELECT count(*) FROM ta;
  ┌─────────────┴─────────────┐
  │       POSTGRES_SCAN       │
  │    ────────────────────   │
- │     (POSTGRES_SCAN) ta    │
+ │         Table: ta         │
  │                           │
  │         ~2550 Rows        │
  └───────────────────────────┘
@@ -39,7 +39,7 @@ EXPLAIN SELECT count(*) FROM ta;
  ┌─────────────┴─────────────┐
  │       POSTGRES_SCAN       │
  │    ────────────────────   │
- │     (POSTGRES_SCAN) ta    │
+ │         Table: ta         │
  │                           │
  │         ~2550 Rows        │
  └───────────────────────────┘

test/regression/expected/execution_error.out

@@ -5,6 +5,7 @@ INSERT INTO int_as_varchar SELECT * from (
 ) t(a);
 SELECT a::INTEGER FROM int_as_varchar;
 ERROR:  (PGDuckDB/Duckdb_ExecCustomScan) Conversion Error: Could not convert string 'abc' to INT32
-LINE 1: SELECT (a)::integer AS a FROM pgduckdb.public.int...
+
+LINE 1: SELECT (a)::integer AS a FROM pgduckdb.public.int_as_varchar
                   ^
 DROP TABLE int_as_varchar;

test/regression/expected/extensions.out

@@ -2,8 +2,9 @@ SET duckdb.force_execution TO false;
 SELECT * FROM duckdb.raw_query($$ SELECT extension_name, loaded, installed FROM duckdb_extensions() WHERE loaded and extension_name != 'jemalloc' $$);
 NOTICE:  result: extension_name	loaded	installed	
 VARCHAR	BOOLEAN	BOOLEAN	
-[ Rows: 5]
+[ Rows: 6]
 cached_httpfs	true	true
+core_functions	true	true
 icu	true	true
 json	true	true
 parquet	true	true
@@ -39,8 +40,9 @@ SELECT last_value FROM duckdb.extensions_table_seq;
 SELECT * FROM duckdb.raw_query($$ SELECT extension_name, loaded, installed FROM duckdb_extensions() WHERE loaded and extension_name != 'jemalloc' $$);
 NOTICE:  result: extension_name	loaded	installed	
 VARCHAR	BOOLEAN	BOOLEAN	
-[ Rows: 5]
+[ Rows: 6]
 cached_httpfs	true	true
+core_functions	true	true
 icu	true	true
 json	true	true
 parquet	true	true
@@ -70,8 +72,9 @@ SELECT last_value FROM duckdb.extensions_table_seq;
 SELECT * FROM duckdb.raw_query($$ SELECT extension_name, loaded, installed FROM duckdb_extensions() WHERE loaded and extension_name != 'jemalloc' $$);
 NOTICE:  result: extension_name	loaded	installed	
 VARCHAR	BOOLEAN	BOOLEAN	
-[ Rows: 5]
+[ Rows: 6]
 cached_httpfs	true	true
+core_functions	true	true
 icu	true	true
 json	true	true
 parquet	true	true
@@ -98,9 +101,10 @@ SELECT last_value FROM duckdb.extensions_table_seq;
 SELECT * FROM duckdb.raw_query($$ SELECT extension_name, loaded, installed FROM duckdb_extensions() WHERE loaded and extension_name != 'jemalloc' $$);
 NOTICE:  result: extension_name	loaded	installed	
 VARCHAR	BOOLEAN	BOOLEAN	
-[ Rows: 6]
+[ Rows: 7]
 aws	true	true
 cached_httpfs	true	true
+core_functions	true	true
 icu	true	true
 json	true	true
 parquet	true	true
@@ -124,9 +128,10 @@ SELECT last_value FROM duckdb.extensions_table_seq;
 SELECT * FROM duckdb.raw_query($$ SELECT extension_name, loaded, installed FROM duckdb_extensions() WHERE loaded and extension_name != 'jemalloc' $$);
 NOTICE:  result: extension_name	loaded	installed	
 VARCHAR	BOOLEAN	BOOLEAN	
-[ Rows: 6]
+[ Rows: 7]
 aws	true	true
 cached_httpfs	true	true
+core_functions	true	true
 icu	true	true
 json	true	true
 parquet	true	true

test/regression/expected/non_superuser.out

@@ -108,12 +108,14 @@ SELECT * FROM duckdb.install_extension('iceberg');
 
 -- We should handle SQL injections carefully though to only allow INSTALL
 SELECT * FROM duckdb.install_extension($$ '; select * from hacky '' $$);
-ERROR:  (PGDuckDB/install_extension) HTTP Error: Failed to download extension " '; select * from hacky '' " at URL "http://extensions.duckdb.org/v1.1.3/linux_amd64/ '; select * from hacky '' .duckdb_extension.gz" (HTTP 403)
+ERROR:  (PGDuckDB/install_extension) HTTP Error: Failed to download extension " '; select * from hacky '' " at URL "http://extensions.duckdb.org/v1.2.0/linux_amd64/ '; select * from hacky '' .duckdb_extension.gz" (HTTP 403)
 
 Candidate extensions: "delta", "excel", "sqlite_scanner", "inet", "sqlite"
+For more info, visit https://duckdb.org/docs/extensions/troubleshooting/?version=v1.2.0&platform=linux_amd64&extension= '; select * from hacky '' 
 INSERT INTO duckdb.extensions (name) VALUES ($$ '; select * from hacky $$);
 SELECT * FROM duckdb.query($$ SELECT 1 $$);
 ERROR:  (PGDuckDB/DuckdbPlanNode) Parser Error: unterminated quoted string at or near "'; select * from hacky "
+
 LINE 1: LOAD  '; select * from hacky 
               ^
 TRUNCATE duckdb.extensions;

test/regression/expected/read_functions.out

@@ -19,7 +19,7 @@ SELECT r['sepal.length'] FROM read_parquet('../../data/iris.parquet') r ORDER BY
 
 SELECT r['sepal.length'], r['file_row_number'], r['filename']
     FROM read_parquet('../../data/iris.parquet', file_row_number => true, filename => true) r
-    ORDER BY r['sepal.length']  LIMIT 5;
+    ORDER BY r['sepal.length'], r['file_row_number']  LIMIT 5;
  sepal.length | file_row_number |        filename         
 --------------+-----------------+-------------------------
           4.3 |              13 | ../../data/iris.parquet