Duckdb 1.2 update

Makefile

@@ -13,7 +13,7 @@ OBJS += $(subst .c,.o, $(C_SRCS))
 # set to `make` to disable ninja
 DUCKDB_GEN ?= ninja
 # used to know what version of extensions to download
-DUCKDB_VERSION = v1.1.3
+DUCKDB_VERSION = v1.2.0
 # duckdb build tweaks
 DUCKDB_CMAKE_VARS = -DBUILD_SHELL=0 -DBUILD_PYTHON=0 -DBUILD_UNITTESTS=0
 # set to 1 to disable asserts in DuckDB. This is particularly useful in combinition with MotherDuck.

include/pgduckdb/scan/postgres_scan.hpp

@@ -22,6 +22,10 @@ struct PostgresScanGlobalState : public duckdb::GlobalTableFunctionState {
 	}
 	void ConstructTableScanQuery(const duckdb::TableFunctionInitInput &input);
 
+private:
+	int ExtractQueryFilters(duckdb::TableFilter *filter, const char *column_name, duckdb::string &filters,
+	                        bool is_optional_filter_parent);
+
 public:
 	Snapshot snapshot;
 	Relation rel;
@@ -50,7 +54,6 @@ struct PostgresScanLocalState : public duckdb::LocalTableFunctionState {
 struct PostgresScanFunctionData : public duckdb::TableFunctionData {
 	PostgresScanFunctionData(Relation rel, uint64_t cardinality, Snapshot snapshot);
 	~PostgresScanFunctionData() override;
-
 	duckdb::vector<duckdb::string> complex_filters;
 	Relation rel;
 	uint64_t cardinality;
@@ -74,7 +77,7 @@ struct PostgresScanTableFunction : public duckdb::TableFunction {
 
 	static duckdb::unique_ptr<duckdb::NodeStatistics> PostgresScanCardinality(duckdb::ClientContext &context,
 	                                                                          const duckdb::FunctionData *data);
-	static std::string ToString(const duckdb::FunctionData *bind_data);
+	static duckdb::InsertionOrderPreservingMap<duckdb::string> ToString(duckdb::TableFunctionToStringInput &input);
 };
 
 } // namespace pgduckdb

src/pgduckdb_types.cpp

@@ -763,7 +763,7 @@ ConvertDuckToPostgresValue(TupleTableSlot *slot, duckdb::Value &value, idx_t col
 		break;
 	}
 	case TIMESTAMPTZOID: {
-		duckdb::timestamp_t timestamp = value.GetValue<duckdb::timestamp_t>();
+		duckdb::timestamp_tz_t timestamp = value.GetValue<duckdb::timestamp_tz_t>();
 		slot->tts_values[col] = timestamp.value - pgduckdb::PGDUCKDB_DUCK_TIMESTAMP_OFFSET;
 		break;
 	}
@@ -1187,7 +1187,7 @@ ConvertPostgresParameterToDuckValue(Datum value, Oid postgres_type) {
 		return duckdb::Value::TIMESTAMP(duckdb::timestamp_t(DatumGetTimestamp(value) + PGDUCKDB_DUCK_TIMESTAMP_OFFSET));
 	case TIMESTAMPTZOID:
 		return duckdb::Value::TIMESTAMPTZ(
-		    duckdb::timestamp_t(DatumGetTimestampTz(value) + PGDUCKDB_DUCK_TIMESTAMP_OFFSET));
+		    duckdb::timestamp_tz_t(DatumGetTimestampTz(value) + PGDUCKDB_DUCK_TIMESTAMP_OFFSET));
 	case FLOAT4OID:
 		return duckdb::Value::FLOAT(DatumGetFloat4(value));
 	case FLOAT8OID:
@@ -1254,8 +1254,8 @@ ConvertPostgresToDuckValue(Oid attr_type, Datum value, duckdb::Vector &result, i
 		    result, duckdb::timestamp_t(static_cast<int64_t>(value + PGDUCKDB_DUCK_TIMESTAMP_OFFSET)), offset);
 		break;
 	case duckdb::LogicalTypeId::TIMESTAMP_TZ:
-		Append<duckdb::timestamp_t>(
-		    result, duckdb::timestamp_t(static_cast<int64_t>(value + PGDUCKDB_DUCK_TIMESTAMP_OFFSET)), offset);
+		Append<duckdb::timestamp_tz_t>(
+		    result, duckdb::timestamp_tz_t(static_cast<int64_t>(value + PGDUCKDB_DUCK_TIMESTAMP_OFFSET)), offset);
 		break;
 	case duckdb::LogicalTypeId::FLOAT:
 		Append<float>(result, DatumGetFloat4(value), offset);

src/scan/postgres_scan.cpp

@@ -1,3 +1,5 @@
+#include "duckdb/planner/filter/optional_filter.hpp"
+
 #include "pgduckdb/scan/postgres_scan.hpp"
 #include "pgduckdb/scan/postgres_table_reader.hpp"
 #include "pgduckdb/pgduckdb_types.hpp"
@@ -7,12 +9,70 @@
 #include "pgduckdb/pgduckdb_process_lock.hpp"
 #include "pgduckdb/logger.hpp"
 
+#include <numeric> // std::accumulate
+
 namespace pgduckdb {
 
 //
 // PostgresScanGlobalState
 //
 
+static duckdb::string
+FilterJoin(duckdb::vector<duckdb::string> &filters, duckdb::string &&delimiter) {
+	return std::accumulate(filters.begin() + 1, filters.end(), filters[0],
+	                       [&delimiter](duckdb::string l, duckdb::string r) { return l + delimiter + r; });
+}
+
+int
+PostgresScanGlobalState::ExtractQueryFilters(duckdb::TableFilter *filter, const char *column_name,
+                                             duckdb::string &query_filters, bool is_optional_filter_parent) {
+	switch (filter->filter_type) {
+	case duckdb::TableFilterType::CONSTANT_COMPARISON:
+	case duckdb::TableFilterType::IS_NULL:
+	case duckdb::TableFilterType::IS_NOT_NULL:
+	case duckdb::TableFilterType::IN_FILTER: {
+		query_filters += filter->ToString(column_name).c_str();
+		return 1;
+	}
+	case duckdb::TableFilterType::CONJUNCTION_OR:
+	case duckdb::TableFilterType::CONJUNCTION_AND: {
+		auto conjuction_filter = reinterpret_cast<duckdb::ConjunctionFilter *>(filter);
+		duckdb::vector<std::string> conjuction_child_filters;
+		for (idx_t i = 0; i < conjuction_filter->child_filters.size(); i++) {
+			std::string child_filter;
+			if (ExtractQueryFilters(conjuction_filter->child_filters[i].get(), column_name, child_filter, false)) {
+				conjuction_child_filters.emplace_back(child_filter);
+			}
+		}
+		duckdb::string conjuction_delimiter =
+		    filter->filter_type == duckdb::TableFilterType::CONJUNCTION_OR ? " OR " : " AND ";
+		if (conjuction_child_filters.size()) {
+			query_filters += "(" + FilterJoin(conjuction_child_filters, std::move(conjuction_delimiter)) + ")";
+		}
+		return conjuction_child_filters.size();
+	}
+	case duckdb::TableFilterType::OPTIONAL_FILTER: {
+		auto optional_filter = reinterpret_cast<duckdb::OptionalFilter *>(filter);
+		return ExtractQueryFilters(optional_filter->child_filter.get(), column_name, query_filters, true);
+	}
+	/* DYNAMIC_FILTER is push down filter from topN execution. STRUCT_EXTRACT is
+	 * only received if struct_extract function is used. Default will catch all
+	 * filter that could be added in future in DuckDB.
+	 */
+	case duckdb::TableFilterType::DYNAMIC_FILTER:
+	case duckdb::TableFilterType::STRUCT_EXTRACT:
+	default: {
+		if (is_optional_filter_parent) {
+			pd_log(DEBUG1, "(DuckDB/ExtractQueryFilters) Unsupported optional filter: %s",
+			       filter->ToString(column_name).c_str());
+			return 0;
+		}
+		throw duckdb::Exception(duckdb::ExceptionType::EXECUTOR,
+		                        "Invalid Filter Type: " + filter->ToString(column_name));
+	}
+	}
+}
+
 void
 PostgresScanGlobalState::ConstructTableScanQuery(const duckdb::TableFunctionInitInput &input) {
 	/* SELECT COUNT(*) FROM */
@@ -82,27 +142,23 @@ PostgresScanGlobalState::ConstructTableScanQuery(const duckdb::TableFunctionInit
 
 	scan_query << " FROM " << GenerateQualifiedRelationName(rel);
 
-	first = true;
-
+	duckdb::vector<duckdb::string> query_filters;
 	for (auto const &[attr_num, duckdb_scanned_index] : columns_to_scan) {
 		auto filter = column_filters[duckdb_scanned_index];
-
 		if (!filter) {
 			continue;
 		}
-
-		if (first) {
-			scan_query << " WHERE ";
-		} else {
-			scan_query << " AND ";
-		}
-
-		first = false;
-		scan_query << "(";
+		duckdb::string column_query_filters;
 		auto attr = GetAttr(table_tuple_desc, attr_num - 1);
 		auto col = pgduckdb::QuoteIdentifier(GetAttName(attr));
-		scan_query << filter->ToString(col).c_str();
-		scan_query << ") ";
+		if (ExtractQueryFilters(filter, col, column_query_filters, false)) {
+			query_filters.emplace_back(column_query_filters);
+		};
+	}
+
+	if (query_filters.size()) {
+		scan_query << " WHERE ";
+		scan_query << FilterJoin(query_filters, " AND ");
 	}
 }
 
@@ -157,12 +213,12 @@ PostgresScanTableFunction::PostgresScanTableFunction()
 	to_string = ToString;
 }
 
-std::string
-PostgresScanTableFunction::ToString(const duckdb::FunctionData *data) {
-	auto &bind_data = data->Cast<PostgresScanFunctionData>();
-	std::ostringstream oss;
-	oss << "(POSTGRES_SCAN) " << GetRelationName(bind_data.rel);
-	return oss.str();
+duckdb::InsertionOrderPreservingMap<duckdb::string>
+PostgresScanTableFunction::ToString(duckdb::TableFunctionToStringInput &input) {
+	auto &bind_data = input.bind_data->Cast<PostgresScanFunctionData>();
+	duckdb::InsertionOrderPreservingMap<duckdb::string> result;
+	result["Table"] = GetRelationName(bind_data.rel);
+	return result;
 }
 
 duckdb::unique_ptr<duckdb::GlobalTableFunctionState>

test/pycheck/explain_test.py

@@ -28,7 +28,7 @@ def test_explain(cur: Cursor):
     result = cur.sql("EXPLAIN SELECT count(*) FROM test_table where id = %s", (1,))
     plan = "\n".join(result)
     assert "UNGROUPED_AGGREGATE" in plan
-    assert "id=1 AND id IS NOT NULL" in plan
+    assert "id=1" in plan
     assert "Total Time:" not in plan
     assert "Output:" not in plan
 
@@ -37,7 +37,7 @@ def test_explain(cur: Cursor):
     )
     plan = "\n".join(result)
     assert "UNGROUPED_AGGREGATE" in plan
-    assert "id=1 AND id IS NOT NULL" in plan
+    assert "id=1" in plan
     assert "Total Time:" in plan
     assert "Output:" not in plan
 
@@ -66,7 +66,7 @@ def test_explain_ctas(cur: Cursor):
         "EXPLAIN ANALYZE CREATE TEMP TABLE heap2(id) AS SELECT * from heap1"
     )
     plan = "\n".join(result)
-    assert "POSTGRES_SCAN" in plan
+    assert "TABLE_SCAN" in plan
     assert "Total Time:" in plan
 
     result = cur.sql(

test/regression/expected/duckdb_recycle.out

@@ -15,7 +15,7 @@ EXPLAIN SELECT count(*) FROM ta;
  ┌─────────────┴─────────────┐
  │       POSTGRES_SCAN       │
  │    ────────────────────   │
- │     (POSTGRES_SCAN) ta    │
+ │         Table: ta         │
  │                           │
  │         ~2550 Rows        │
  └───────────────────────────┘
@@ -39,7 +39,7 @@ EXPLAIN SELECT count(*) FROM ta;
  ┌─────────────┴─────────────┐
  │       POSTGRES_SCAN       │
  │    ────────────────────   │
- │     (POSTGRES_SCAN) ta    │
+ │         Table: ta         │
  │                           │
  │         ~2550 Rows        │
  └───────────────────────────┘

test/regression/expected/execution_error.out

@@ -5,6 +5,7 @@ INSERT INTO int_as_varchar SELECT * from (
 ) t(a);
 SELECT a::INTEGER FROM int_as_varchar;
 ERROR:  (PGDuckDB/Duckdb_ExecCustomScan) Conversion Error: Could not convert string 'abc' to INT32
-LINE 1: SELECT (a)::integer AS a FROM pgduckdb.public.int...
+
+LINE 1: SELECT (a)::integer AS a FROM pgduckdb.public.int_as_varchar
                   ^
 DROP TABLE int_as_varchar;