release: bump duckdb to 1.3

.github/workflows/MainDistributionPipeline.yml

@@ -16,9 +16,8 @@ jobs:
     name: Build extension binaries
     uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@6a7a4f24c5999355ab36c0a6835baf891fc9d522
     with:
-      duckdb_version: v1.2.2
-      ci_tools_version: 6a7a4f24c5999355ab36c0a6835baf891fc9d522
+      ci_tools_version: 1f00107ca1eaf1691049907296a8aa796d054e6b
+      duckdb_version: ad1273222186d28b4b351736ed88101044bbe97b
       extension_name: vortex
       exclude_archs: "wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw;windows_amd64;linux_arm64"
       extra_toolchains: "rust"
-      override_ci_tools_repository: spiraldb/extension-ci-tools

CMakeLists.txt

@@ -6,30 +6,34 @@ project(${TARGET_NAME}_project)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 set(CMAKE_CXX_STANDARD 17)
 
+# Allow C++20 designator syntax in C++17
+add_compile_options(-Wno-c++20-designator)
+
 include(FetchContent)
 FetchContent_Declare(
-    Corrosion
-    GIT_REPOSITORY https://github.com/corrosion-rs/corrosion.git
-    GIT_TAG v0.5.2
+        Corrosion
+        GIT_REPOSITORY https://github.com/corrosion-rs/corrosion.git
+        GIT_TAG v0.5.2
 )
 
 FetchContent_MakeAvailable(Corrosion)
 
 find_package(Catch2 CONFIG REQUIRED)
 find_package(Protobuf CONFIG REQUIRED)
+
 if (APPLE)
     find_library(SECURITY_FRAMEWORK Security)
 endif ()
 
 corrosion_import_crate(MANIFEST_PATH vortex/Cargo.toml
-    CRATES vortex-ffi
-    FEATURES duckdb
-    CRATE_TYPES staticlib
-    FLAGS --crate-type=staticlib
+        CRATES vortex-ffi
+        FEATURES duckdb mimalloc
+        CRATE_TYPES staticlib
+        FLAGS --crate-type=staticlib
 )
 
 set(EXTENSION_NAME ${TARGET_NAME}_extension)
-set(EXTENSION_SOURCES src/vortex_extension.cpp src/expr/expr.cpp src/vortex_write.cpp src/vortex_scan.cpp)
+set(EXTENSION_SOURCES src/vortex_extension.cpp src/vortex_expr.cpp src/vortex_write.cpp src/vortex_scan.cpp)
 set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension)
 
 # Generate C++ code from .proto files.
@@ -41,19 +45,20 @@ protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS ${PROTO_FILES} PROTOC_OUT_DIR ${PROT
 include_directories(src/include ${PROTO_GEN_DIR} vortex/vortex-ffi/cinclude)
 
 build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES} ${PROTO_SRCS})
-build_loadable_extension(${TARGET_NAME} ${EXTENSION_SOURCES} ${PROTO_SRCS})
+set(PARAMETERS "-warnings")
+build_loadable_extension(${TARGET_NAME} ${PARAMETERS} ${EXTENSION_SOURCES} ${PROTO_SRCS})
 
 target_link_libraries(${EXTENSION_NAME}
-    vortex_ffi-static
-    protobuf::libprotobuf
-    ${SECURITY_FRAMEWORK}
+        vortex_ffi-static
+        protobuf::libprotobuf
+        ${SECURITY_FRAMEWORK}
 )
 
 add_subdirectory(test)
 
 install(
-    TARGETS ${EXTENSION_NAME}
-    EXPORT "${DUCKDB_EXPORT_SET}"
-    LIBRARY DESTINATION "${INSTALL_LIB_DIR}"
-    ARCHIVE DESTINATION "${INSTALL_LIB_DIR}"
+        TARGETS ${EXTENSION_NAME}
+        EXPORT "${DUCKDB_EXPORT_SET}"
+        LIBRARY DESTINATION "${INSTALL_LIB_DIR}"
+        ARCHIVE DESTINATION "${INSTALL_LIB_DIR}"
 )

Makefile

@@ -2,11 +2,17 @@ PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 
 EXT_NAME=vortex_duckdb
 EXT_CONFIG=${PROJ_DIR}extension_config.cmake
-EXT_FLAGS=-DCMAKE_OSX_DEPLOYMENT_TARGET=12.0 -DOVERRIDE_GIT_DESCRIBE=v1.2.2
+EXT_FLAGS=-DCMAKE_OSX_DEPLOYMENT_TARGET=12.0
 
+export OVERRIDE_GIT_DESCRIBE=v1.3.0
 export MACOSX_DEPLOYMENT_TARGET=12.0
-export VCPKG_OSX_DEPLOYMENT_TARGET=12.0
 export VCPKG_FEATURE_FLAGS=-binarycaching
+export VCPKG_OSX_DEPLOYMENT_TARGET=12.0
 export VCPKG_TOOLCHAIN_PATH := ${PROJ_DIR}vcpkg/scripts/buildsystems/vcpkg.cmake
 
+# This is not needed on macOS, we don't see a tls error on load there.
+ifeq ($(shell uname), Linux)
+    export CFLAGS=-ftls-model=global-dynamic
+endif
+
 include extension-ci-tools/makefiles/duckdb_extension.Makefile

extension_config.cmake

@@ -4,7 +4,4 @@
 duckdb_extension_load(vortex
     SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}
     LOAD_TESTS
-)
-
-# Any extra extensions that should be built
-# e.g.: duckdb_extension_load(json)
+)

src/include/vortex.hpp

@@ -1,9 +1,3 @@
+// This header prefixes the auto-generated vortex.h with #pragma once.
 #pragma once
-
-
-// Include Vortex FFI, with the DuckDB FFI feature
-#ifndef ENABLE_DUCKDB_FFI
-#define ENABLE_DUCKDB_FFI
-#endif
-
-#include "vortex.h"
+#include "vortex.h"

src/include/vortex_common.hpp

@@ -1,77 +1,162 @@
 #pragma once
+#define ENABLE_DUCKDB_FFI
 
 #include "duckdb.hpp"
+#include "duckdb/common/unique_ptr.hpp"
+
 #include "vortex.hpp"
 #include "vortex_error.hpp"
+#include "vortex_session.hpp"
+
+namespace vortex {
+
+struct DType {
+	explicit DType(vx_dtype *dtype) : dtype(dtype) {
+	}
+
+	static duckdb::unique_ptr<DType> FromDuckDBTable(const std::vector<duckdb_logical_type> &column_types,
+	                                                 const std::vector<unsigned char> &column_nullable,
+	                                                 const std::vector<const char *> &column_names) {
+		D_ASSERT(column_names.size() == column_nullable.size());
+		D_ASSERT(column_names.size() == column_types.size());
 
-#include <duckdb/common/unique_ptr.hpp>
+		auto dtype = Try([&](auto err) {
+			return vx_duckdb_logical_type_to_dtype(column_types.data(), column_nullable.data(), column_names.data(),
+			                                       column_names.size(), err);
+		});
 
-struct VortexConversionCache {
-	explicit VortexConversionCache(const unsigned long cache_id) : cache(vx_conversion_cache_create(cache_id)) {
+		return duckdb::make_uniq<DType>(dtype);
 	}
 
-	~VortexConversionCache() {
+	~DType() {
+		if (dtype != nullptr) {
+			vx_dtype_free(dtype);
+		}
+	}
+
+	vx_dtype *dtype;
+};
+
+struct ConversionCache {
+	explicit ConversionCache(const unsigned long cache_id) : cache(vx_conversion_cache_create(cache_id)) {
+	}
+
+	~ConversionCache() {
 		vx_conversion_cache_free(cache);
 	}
 
 	vx_conversion_cache *cache;
 };
 
-struct VortexFileReader {
-	explicit VortexFileReader(vx_file_reader *file) : file(file) {
+struct FileReader {
+	explicit FileReader(vx_file_reader *file) : file(file) {
 	}
 
-	~VortexFileReader() {
+	~FileReader() {
 		vx_file_reader_free(file);
 	}
 
-	static duckdb::unique_ptr<VortexFileReader> Open(const vx_file_open_options *options) {
-		vx_error *error;
-		auto file = vx_file_open_reader(options, &error);
-		HandleError(error);
-		return duckdb::make_uniq<VortexFileReader>(file);
+	static duckdb::unique_ptr<FileReader> Open(const vx_file_open_options *options, VortexSession &session) {
+		auto file = Try([&](auto err) { return vx_file_open_reader(options, session.session, err); });
+		return duckdb::make_uniq<FileReader>(file);
+	}
+
+	vx_array_iterator *Scan(const vx_file_scan_options *options) {
+		return Try([&](auto err) { return vx_file_reader_scan(this->file, options, err); });
+	}
+
+	bool CanPrune(const char *filter_expression, unsigned int filter_expression_len) {
+		return Try([&](auto err) {
+			return vx_file_reader_can_prune(this->file, filter_expression, filter_expression_len, err);
+		});
+	}
+
+	uint64_t FileRowCount() {
+		return Try([&](auto err) { return vx_file_row_count(file, err); });
+	}
+
+	struct DType DType() {
+		return vortex::DType(vx_file_dtype(file));
 	}
 
 	vx_file_reader *file;
 };
 
-struct VortexArray {
-	explicit VortexArray(vx_array *array) : array(array) {
+struct Array {
+	explicit Array(vx_array *array) : array(array) {
 	}
 
-	~VortexArray() {
+	~Array() {
 		if (array != nullptr) {
 			vx_array_free(array);
 		}
 	}
 
-	idx_t ToDuckDBVector(idx_t current_row, duckdb_data_chunk output, const VortexConversionCache *cache) const {
-		vx_error *error;
-		auto idx = vx_array_to_duckdb_chunk(array, current_row, output, cache->cache, &error);
-		HandleError(error);
-		return idx;
+	static duckdb::unique_ptr<Array> FromDuckDBChunk(DType &dtype, duckdb::DataChunk &chunk) {
+		auto array = Try([&](auto err) {
+			return vx_duckdb_chunk_to_array(reinterpret_cast<duckdb_data_chunk>(&chunk), dtype.dtype, err);
+		});
+
+		return duckdb::make_uniq<Array>(array);
+	}
+
+	idx_t ToDuckDBVector(idx_t current_row, duckdb_data_chunk output, const ConversionCache *cache) const {
+		return Try([&](auto err) { return vx_array_to_duckdb_chunk(array, current_row, output, cache->cache, err); });
 	}
 
 	vx_array *array;
 };
 
-struct VortexArrayStream {
-	explicit VortexArrayStream(vx_array_stream *array_stream) : array_stream(array_stream) {
+struct ArrayIterator {
+	explicit ArrayIterator(vx_array_iterator *array_iter) : array_iter(array_iter) {
 	}
 
-	~VortexArrayStream() {
-		vx_array_stream_free(array_stream);
+	~ArrayIterator() {
+		vx_array_iter_free(array_iter);
 	}
 
-	duckdb::unique_ptr<VortexArray> NextArray() const {
-		vx_error *error;
-		auto array = vx_array_stream_next(array_stream, &error);
-		HandleError(error);
+	duckdb::unique_ptr<Array> NextArray() const {
+		auto array = Try([&](auto err) { return vx_array_iter_next(array_iter, err); });
+
 		if (array == nullptr) {
 			return nullptr;
 		}
-		return duckdb::make_uniq<VortexArray>(array);
+
+		return duckdb::make_uniq<Array>(array);
 	}
 
-	vx_array_stream *array_stream;
+	vx_array_iterator *array_iter;
 };
+
+struct ArrayStreamSink {
+	explicit ArrayStreamSink(vx_array_sink *sink, duckdb::unique_ptr<DType> dtype)
+	    : sink(sink), dtype(std::move(dtype)) {
+	}
+
+	static duckdb::unique_ptr<ArrayStreamSink> Create(std::string file_path, duckdb::unique_ptr<DType> &&dtype) {
+		auto sink = Try([&](auto err) { return vx_array_sink_open_file(file_path.c_str(), dtype->dtype, err); });
+		return duckdb::make_uniq<ArrayStreamSink>(sink, std::move(dtype));
+	}
+
+	void PushChunk(duckdb::DataChunk &chunk) {
+		auto array = Array::FromDuckDBChunk(*dtype, chunk);
+		Try([&](auto err) { vx_array_sink_push(sink, array->array, err); });
+	}
+
+	void Close() {
+		Try([&](auto err) { vx_array_sink_close(sink, err); });
+		this->sink = nullptr;
+	}
+
+	~ArrayStreamSink() {
+		// "should dctor a sink, before closing it
+		// If you throw during writes then the stack will be unwound and the destructor is going to be called before the
+		// close method is invoked thus triggering following assertion failure and will clobber the exception printing
+		// D_ASSERT(sink == nullptr);
+	}
+
+	vx_array_sink *sink;
+	duckdb::unique_ptr<DType> dtype;
+};
+
+} // namespace vortex

src/include/vortex_error.hpp

@@ -1,11 +1,33 @@
 #pragma once
 
+#include <string>
+#include <type_traits>
+
+#include "duckdb.hpp"
 #include "vortex.hpp"
 
+namespace vortex {
+
 inline void HandleError(vx_error *error) {
 	if (error != nullptr) {
 		auto msg = std::string(vx_error_get_message(error));
 		vx_error_free(error);
 		throw duckdb::InvalidInputException(msg);
 	}
 }
+
+template <typename Func>
+auto Try(Func func) {
+	vx_error *error = nullptr;
+	// Handle both void and non-void return types.
+	if constexpr (std::is_void_v<std::invoke_result_t<Func, vx_error **>>) {
+		func(&error);
+		HandleError(error);
+	} else {
+		auto result = func(&error);
+		HandleError(error);
+		return result;
+	}
+}
+
+} // namespace vortex

src/include/expr/expr.hpp → src/include/vortex_expr.hpp

@@ -1,14 +1,14 @@
 #pragma once
 
-#include "expr.pb.h"
 #include "duckdb/planner/expression.hpp"
+#include "duckdb/planner/table_filter.hpp"
 
-#include <duckdb/planner/table_filter.hpp>
+#include "expr.pb.h"
 
+namespace vortex {
 vortex::expr::Expr *table_expression_into_expr(google::protobuf::Arena &arena, duckdb::TableFilter &filter,
                                                const std::string &column_name);
-
 vortex::expr::Expr *expression_into_vortex_expr(google::protobuf::Arena &arena, const duckdb::Expression &expr);
-
 vortex::expr::Expr *flatten_exprs(google::protobuf::Arena &arena,
-                                  const duckdb::vector<vortex::expr::Expr *> &child_filters);
+                                  const duckdb::vector<vortex::expr::Expr *> &child_filters);
+} // namespace vortex

src/include/vortex_extension.hpp

@@ -2,13 +2,11 @@
 
 #include "duckdb.hpp"
 
-namespace duckdb {
+// The entry point class API can't be scoped to the vortex namespace.
 
-class VortexExtension : public Extension {
+class VortexExtension : public duckdb::Extension {
 public:
-	void Load(DuckDB &db) override;
+	void Load(duckdb::DuckDB &db) override;
 	std::string Name() override;
 	std::string Version() const override;
 };
-
-} // namespace duckdb