Update to v0.36.0

.github/workflows/MainDistributionPipeline.yml

@@ -14,11 +14,10 @@ concurrency:
 jobs:
   duckdb-stable-build:
     name: Build extension binaries
-    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@6a7a4f24c5999355ab36c0a6835baf891fc9d522
+    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@71d20029c5314dfc34f3bbdab808b9bce03b8003
     with:
-      duckdb_version: v1.2.2
-      ci_tools_version: 6a7a4f24c5999355ab36c0a6835baf891fc9d522
+      duckdb_version: v1.3.0
+      ci_tools_version: 71d20029c5314dfc34f3bbdab808b9bce03b8003
       extension_name: vortex
       exclude_archs: "wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw;windows_amd64;linux_arm64"
       extra_toolchains: "rust"
-      override_ci_tools_repository: spiraldb/extension-ci-tools

.gitignore

@@ -7,6 +7,9 @@ testext
 test/python/__pycache__/
 .Rhistory
 compile_commands.json
+corrosion
+CMakeFiles
 
-#proto gen folder
-gen/
+# Keep CMAKE
+
+!CMakeLists.txt

.gitmodules

@@ -8,7 +8,7 @@
 	branch = main
 [submodule "duckdb"]
 	path = duckdb
-	url = https://github.com/spiraldb/duckdb.git
+	url = https://github.com/duckdb/duckdb.git
 [submodule "vcpkg"]
 	path = vcpkg
 	url = [email protected]:microsoft/vcpkg.git

CMakeLists.txt

@@ -6,6 +6,9 @@ project(${TARGET_NAME}_project)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 set(CMAKE_CXX_STANDARD 17)
 
+# Allow C++20 designator syntax in C++17
+add_compile_options(-Wno-c++20-designator)
+
 include(FetchContent)
 FetchContent_Declare(
     Corrosion
@@ -17,19 +20,20 @@ FetchContent_MakeAvailable(Corrosion)
 
 find_package(Catch2 CONFIG REQUIRED)
 find_package(Protobuf CONFIG REQUIRED)
+
 if (APPLE)
     find_library(SECURITY_FRAMEWORK Security)
 endif ()
 
 corrosion_import_crate(MANIFEST_PATH vortex/Cargo.toml
     CRATES vortex-ffi
-    FEATURES duckdb
+    FEATURES duckdb mimalloc
     CRATE_TYPES staticlib
     FLAGS --crate-type=staticlib
 )
 
 set(EXTENSION_NAME ${TARGET_NAME}_extension)
-set(EXTENSION_SOURCES src/vortex_extension.cpp src/expr/expr.cpp src/vortex_write.cpp src/vortex_scan.cpp)
+set(EXTENSION_SOURCES src/vortex_extension.cpp src/vortex_expr.cpp src/vortex_write.cpp src/vortex_scan.cpp)
 set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension)
 
 # Generate C++ code from .proto files.
@@ -41,7 +45,8 @@ protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS ${PROTO_FILES} PROTOC_OUT_DIR ${PROT
 include_directories(src/include ${PROTO_GEN_DIR} vortex/vortex-ffi/cinclude)
 
 build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES} ${PROTO_SRCS})
-build_loadable_extension(${TARGET_NAME} ${EXTENSION_SOURCES} ${PROTO_SRCS})
+set(PARAMETERS "-warnings")
+build_loadable_extension(${TARGET_NAME} ${PARAMETERS} ${EXTENSION_SOURCES} ${PROTO_SRCS})
 
 target_link_libraries(${EXTENSION_NAME}
     vortex_ffi-static

Makefile

@@ -2,11 +2,27 @@ PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 
 EXT_NAME=vortex_duckdb
 EXT_CONFIG=${PROJ_DIR}extension_config.cmake
-EXT_FLAGS=-DCMAKE_OSX_DEPLOYMENT_TARGET=12.0 -DOVERRIDE_GIT_DESCRIBE=v1.2.2
+EXT_FLAGS=-DCMAKE_OSX_DEPLOYMENT_TARGET=12.0
 
 export MACOSX_DEPLOYMENT_TARGET=12.0
-export VCPKG_OSX_DEPLOYMENT_TARGET=12.0
+
+# The version of DuckDB and its Vortex extension is either implicitly set by Git tag, e.g. v1.2.2, or commit
+# SHA if the current commit does not have a tag. The implicitly set version can be overridden by defining the
+# `OVERRIDE_GIT_DESCRIBE` environment variable. In context of the DuckDB community extension build, we have to
+# rely on the Git tag, as DuckDB's CI performs a checkout by Git tag. Therefore, the version can't be explicitly
+# set via environment variable for the community extension build.
+
+export OVERRIDE_GIT_DESCRIBE=v1.3.0
 export VCPKG_FEATURE_FLAGS=-binarycaching
+export VCPKG_OSX_DEPLOYMENT_TARGET=12.0
 export VCPKG_TOOLCHAIN_PATH := ${PROJ_DIR}vcpkg/scripts/buildsystems/vcpkg.cmake
 
+export BUILD_MAIN_DUCKDB_LIBRARY=0
+export DISABLE_BUILTIN_EXTENSIONS=1
+
+# This is not needed on macOS, we don't see a tls error on load there.
+ifeq ($(shell uname), Linux)
+    export CFLAGS=-ftls-model=global-dynamic
+endif
+
 include extension-ci-tools/makefiles/duckdb_extension.Makefile

extension_config.cmake

@@ -4,7 +4,4 @@
 duckdb_extension_load(vortex
     SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}
     LOAD_TESTS
-)
-
-# Any extra extensions that should be built
-# e.g.: duckdb_extension_load(json)
+)

src/include/vortex_common.hpp

@@ -1,77 +1,162 @@
 #pragma once
+#define ENABLE_DUCKDB_FFI
 
 #include "duckdb.hpp"
+#include "duckdb/common/unique_ptr.hpp"
+
 #include "vortex.hpp"
 #include "vortex_error.hpp"
+#include "vortex_session.hpp"
+
+namespace vortex {
+
+struct DType {
+	explicit DType(vx_dtype *dtype) : dtype(dtype) {
+	}
+
+	static duckdb::unique_ptr<DType> FromDuckDBTable(const std::vector<duckdb_logical_type> &column_types,
+	                                                 const std::vector<unsigned char> &column_nullable,
+	                                                 const std::vector<const char *> &column_names) {
+		D_ASSERT(column_names.size() == column_nullable.size());
+		D_ASSERT(column_names.size() == column_types.size());
 
-#include <duckdb/common/unique_ptr.hpp>
+		auto dtype = Try([&](auto err) {
+			return vx_duckdb_logical_type_to_dtype(column_types.data(), column_nullable.data(), column_names.data(),
+			                                       column_names.size(), err);
+		});
 
-struct VortexConversionCache {
-	explicit VortexConversionCache(const unsigned long cache_id) : cache(vx_conversion_cache_create(cache_id)) {
+		return duckdb::make_uniq<DType>(dtype);
 	}
 
-	~VortexConversionCache() {
+	~DType() {
+		if (dtype != nullptr) {
+			vx_dtype_free(dtype);
+		}
+	}
+
+	vx_dtype *dtype;
+};
+
+struct ConversionCache {
+	explicit ConversionCache(const unsigned long cache_id) : cache(vx_conversion_cache_create(cache_id)) {
+	}
+
+	~ConversionCache() {
 		vx_conversion_cache_free(cache);
 	}
 
 	vx_conversion_cache *cache;
 };
 
-struct VortexFileReader {
-	explicit VortexFileReader(vx_file_reader *file) : file(file) {
+struct FileReader {
+	explicit FileReader(vx_file_reader *file) : file(file) {
 	}
 
-	~VortexFileReader() {
+	~FileReader() {
 		vx_file_reader_free(file);
 	}
 
-	static duckdb::unique_ptr<VortexFileReader> Open(const vx_file_open_options *options) {
-		vx_error *error;
-		auto file = vx_file_open_reader(options, &error);
-		HandleError(error);
-		return duckdb::make_uniq<VortexFileReader>(file);
+	static duckdb::unique_ptr<FileReader> Open(const vx_file_open_options *options, VortexSession &session) {
+		auto file = Try([&](auto err) { return vx_file_open_reader(options, session.session, err); });
+		return duckdb::make_uniq<FileReader>(file);
+	}
+
+	vx_array_iterator *Scan(const vx_file_scan_options *options) {
+		return Try([&](auto err) { return vx_file_reader_scan(this->file, options, err); });
+	}
+
+	bool CanPrune(const char *filter_expression, unsigned int filter_expression_len) {
+		return Try([&](auto err) {
+			return vx_file_reader_can_prune(this->file, filter_expression, filter_expression_len, err);
+		});
+	}
+
+	uint64_t FileRowCount() {
+		return Try([&](auto err) { return vx_file_row_count(file, err); });
+	}
+
+	struct DType DType() {
+		return vortex::DType(vx_file_dtype(file));
 	}
 
 	vx_file_reader *file;
 };
 
-struct VortexArray {
-	explicit VortexArray(vx_array *array) : array(array) {
+struct Array {
+	explicit Array(vx_array *array) : array(array) {
 	}
 
-	~VortexArray() {
+	~Array() {
 		if (array != nullptr) {
 			vx_array_free(array);
 		}
 	}
 
-	idx_t ToDuckDBVector(idx_t current_row, duckdb_data_chunk output, const VortexConversionCache *cache) const {
-		vx_error *error;
-		auto idx = vx_array_to_duckdb_chunk(array, current_row, output, cache->cache, &error);
-		HandleError(error);
-		return idx;
+	static duckdb::unique_ptr<Array> FromDuckDBChunk(DType &dtype, duckdb::DataChunk &chunk) {
+		auto array = Try([&](auto err) {
+			return vx_duckdb_chunk_to_array(reinterpret_cast<duckdb_data_chunk>(&chunk), dtype.dtype, err);
+		});
+
+		return duckdb::make_uniq<Array>(array);
+	}
+
+	idx_t ToDuckDBVector(idx_t current_row, duckdb_data_chunk output, const ConversionCache *cache) const {
+		return Try([&](auto err) { return vx_array_to_duckdb_chunk(array, current_row, output, cache->cache, err); });
 	}
 
 	vx_array *array;
 };
 
-struct VortexArrayStream {
-	explicit VortexArrayStream(vx_array_stream *array_stream) : array_stream(array_stream) {
+struct ArrayIterator {
+	explicit ArrayIterator(vx_array_iterator *array_iter) : array_iter(array_iter) {
 	}
 
-	~VortexArrayStream() {
-		vx_array_stream_free(array_stream);
+	~ArrayIterator() {
+		vx_array_iter_free(array_iter);
 	}
 
-	duckdb::unique_ptr<VortexArray> NextArray() const {
-		vx_error *error;
-		auto array = vx_array_stream_next(array_stream, &error);
-		HandleError(error);
+	duckdb::unique_ptr<Array> NextArray() const {
+		auto array = Try([&](auto err) { return vx_array_iter_next(array_iter, err); });
+
 		if (array == nullptr) {
 			return nullptr;
 		}
-		return duckdb::make_uniq<VortexArray>(array);
+
+		return duckdb::make_uniq<Array>(array);
 	}
 
-	vx_array_stream *array_stream;
+	vx_array_iterator *array_iter;
 };
+
+struct ArrayStreamSink {
+	explicit ArrayStreamSink(vx_array_sink *sink, duckdb::unique_ptr<DType> dtype)
+	    : sink(sink), dtype(std::move(dtype)) {
+	}
+
+	static duckdb::unique_ptr<ArrayStreamSink> Create(std::string file_path, duckdb::unique_ptr<DType> &&dtype) {
+		auto sink = Try([&](auto err) { return vx_array_sink_open_file(file_path.c_str(), dtype->dtype, err); });
+		return duckdb::make_uniq<ArrayStreamSink>(sink, std::move(dtype));
+	}
+
+	void PushChunk(duckdb::DataChunk &chunk) {
+		auto array = Array::FromDuckDBChunk(*dtype, chunk);
+		Try([&](auto err) { vx_array_sink_push(sink, array->array, err); });
+	}
+
+	void Close() {
+		Try([&](auto err) { vx_array_sink_close(sink, err); });
+		this->sink = nullptr;
+	}
+
+	~ArrayStreamSink() {
+		// "should dctor a sink, before closing it
+		// If you throw during writes then the stack will be unwound and the destructor is going to be called before the
+		// close method is invoked thus triggering following assertion failure and will clobber the exception printing
+		// D_ASSERT(sink == nullptr);
+	}
+
+	vx_array_sink *sink;
+	duckdb::unique_ptr<DType> dtype;
+};
+
+} // namespace vortex

src/include/vortex_error.hpp

@@ -1,11 +1,33 @@
 #pragma once
 
+#include <string>
+#include <type_traits>
+
+#include "duckdb.hpp"
 #include "vortex.hpp"
 
+namespace vortex {
+
 inline void HandleError(vx_error *error) {
 	if (error != nullptr) {
 		auto msg = std::string(vx_error_get_message(error));
 		vx_error_free(error);
 		throw duckdb::InvalidInputException(msg);
 	}
 }
+
+template <typename Func>
+auto Try(Func func) {
+	vx_error *error = nullptr;
+	// Handle both void and non-void return types.
+	if constexpr (std::is_void_v<std::invoke_result_t<Func, vx_error **>>) {
+		func(&error);
+		HandleError(error);
+	} else {
+		auto result = func(&error);
+		HandleError(error);
+		return result;
+	}
+}
+
+} // namespace vortex

src/include/expr/expr.hpp → src/include/vortex_expr.hpp

@@ -1,14 +1,14 @@
 #pragma once
 
-#include "expr.pb.h"
 #include "duckdb/planner/expression.hpp"
+#include "duckdb/planner/table_filter.hpp"
 
-#include <duckdb/planner/table_filter.hpp>
+#include "expr.pb.h"
 
+namespace vortex {
 vortex::expr::Expr *table_expression_into_expr(google::protobuf::Arena &arena, duckdb::TableFilter &filter,
                                                const std::string &column_name);
-
 vortex::expr::Expr *expression_into_vortex_expr(google::protobuf::Arena &arena, const duckdb::Expression &expr);
-
 vortex::expr::Expr *flatten_exprs(google::protobuf::Arena &arena,
-                                  const duckdb::vector<vortex::expr::Expr *> &child_filters);
+                                  const duckdb::vector<vortex::expr::Expr *> &child_filters);
+} // namespace vortex

src/include/vortex_extension.hpp

@@ -2,13 +2,11 @@
 
 #include "duckdb.hpp"
 
-namespace duckdb {
+// The entry point class API can't be scoped to the vortex namespace.
 
-class VortexExtension : public Extension {
+class VortexExtension : public duckdb::Extension {
 public:
-	void Load(DuckDB &db) override;
+	void Load(duckdb::DuckDB &db) override;
 	std::string Name() override;
 	std::string Version() const override;
 };
-
-} // namespace duckdb