update to v0.1.1

CMakeLists.txt

@@ -6,6 +6,9 @@ project(${TARGET_NAME}_project)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 set(CMAKE_CXX_STANDARD 17)
 
+# Allow C++20 designator syntax in C++17
+add_compile_options(-Wno-c++20-designator)
+
 include(FetchContent)
 FetchContent_Declare(
     Corrosion
@@ -17,19 +20,20 @@ FetchContent_MakeAvailable(Corrosion)
 
 find_package(Catch2 CONFIG REQUIRED)
 find_package(Protobuf CONFIG REQUIRED)
+
 if (APPLE)
     find_library(SECURITY_FRAMEWORK Security)
 endif ()
 
 corrosion_import_crate(MANIFEST_PATH vortex/Cargo.toml
     CRATES vortex-ffi
-    FEATURES duckdb
+    FEATURES duckdb mimalloc
     CRATE_TYPES staticlib
     FLAGS --crate-type=staticlib
 )
 
 set(EXTENSION_NAME ${TARGET_NAME}_extension)
-set(EXTENSION_SOURCES src/vortex_extension.cpp src/expr/expr.cpp src/vortex_write.cpp src/vortex_scan.cpp)
+set(EXTENSION_SOURCES src/vortex_extension.cpp src/vortex_expr.cpp src/vortex_write.cpp src/vortex_scan.cpp)
 set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension)
 
 # Generate C++ code from .proto files.
@@ -38,7 +42,7 @@ set(PROTO_GEN_DIR ${CMAKE_CURRENT_SOURCE_DIR}/gen)
 file(MAKE_DIRECTORY ${PROTO_GEN_DIR})
 protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS ${PROTO_FILES} PROTOC_OUT_DIR ${PROTO_GEN_DIR})
 
-include_directories(src/include ${PROTO_GEN_DIR} vortex/vortex-ffi/cinclude)
+include_directories(src/include ${PROTO_GEN_DIR} ../vortex-ffi/cinclude)
 
 build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES} ${PROTO_SRCS})
 build_loadable_extension(${TARGET_NAME} ${EXTENSION_SOURCES} ${PROTO_SRCS})

Makefile

@@ -5,8 +5,16 @@ EXT_CONFIG=${PROJ_DIR}extension_config.cmake
 EXT_FLAGS=-DCMAKE_OSX_DEPLOYMENT_TARGET=12.0 -DOVERRIDE_GIT_DESCRIBE=v1.2.2
 
 export MACOSX_DEPLOYMENT_TARGET=12.0
-export VCPKG_OSX_DEPLOYMENT_TARGET=12.0
+
+# The version of DuckDB and its Vortex extension is either implicitly set by Git tag, e.g. v1.2.2, or commit
+# SHA if the current commit does not have a tag. The implicitly set version can be overridden by defining the
+# `OVERRIDE_GIT_DESCRIBE` environment variable. In context of the DuckDB community extension build, we have to
+# rely on the Git tag, as DuckDB's CI performs a checkout by Git tag. Therefore, the version can't be explicitly
+# set via environment variable for the community extension build.
+
+# export OVERRIDE_GIT_DESCRIBE=v1.2.2
 export VCPKG_FEATURE_FLAGS=-binarycaching
+export VCPKG_OSX_DEPLOYMENT_TARGET=12.0
 export VCPKG_TOOLCHAIN_PATH := ${PROJ_DIR}vcpkg/scripts/buildsystems/vcpkg.cmake
 
 include extension-ci-tools/makefiles/duckdb_extension.Makefile

src/include/vortex.hpp

@@ -1,9 +1,3 @@
+// This header prefixes the auto-generated vortex.h with #pragma once.
 #pragma once
-
-
-// Include Vortex FFI, with the DuckDB FFI feature
-#ifndef ENABLE_DUCKDB_FFI
-#define ENABLE_DUCKDB_FFI
-#endif
-
-#include "vortex.h"
+#include "vortex.h"

src/include/vortex_common.hpp

@@ -1,77 +1,149 @@
 #pragma once
+#define ENABLE_DUCKDB_FFI
 
 #include "duckdb.hpp"
+#include "duckdb/common/unique_ptr.hpp"
+
 #include "vortex.hpp"
 #include "vortex_error.hpp"
 
-#include <duckdb/common/unique_ptr.hpp>
+namespace vortex {
+
+struct DType {
+	explicit DType(vx_dtype *dtype) : dtype(dtype) {
+	}
+
+	static duckdb::unique_ptr<DType> FromDuckDBTable(const std::vector<duckdb_logical_type> &column_types,
+	                                                 const std::vector<unsigned char> &column_nullable,
+	                                                 const std::vector<const char *> &column_names) {
+		D_ASSERT(column_names.size() == column_nullable.size());
+		D_ASSERT(column_names.size() == column_types.size());
+
+		auto dtype = Try([&](auto err) {
+			return vx_duckdb_logical_type_to_dtype(column_types.data(), column_nullable.data(), column_names.data(),
+			                                       column_names.size(), err);
+		});
 
-struct VortexConversionCache {
-	explicit VortexConversionCache(const unsigned long cache_id) : cache(vx_conversion_cache_create(cache_id)) {
+		return duckdb::make_uniq<DType>(dtype);
 	}
 
-	~VortexConversionCache() {
+	~DType() {
+		if (dtype != nullptr) {
+			vx_dtype_free(dtype);
+		}
+	}
+
+	vx_dtype *dtype;
+};
+
+struct ConversionCache {
+	explicit ConversionCache(const unsigned long cache_id) : cache(vx_conversion_cache_create(cache_id)) {
+	}
+
+	~ConversionCache() {
 		vx_conversion_cache_free(cache);
 	}
 
 	vx_conversion_cache *cache;
 };
 
-struct VortexFileReader {
-	explicit VortexFileReader(vx_file_reader *file) : file(file) {
+struct FileReader {
+	explicit FileReader(vx_file_reader *file) : file(file) {
 	}
 
-	~VortexFileReader() {
+	~FileReader() {
 		vx_file_reader_free(file);
 	}
 
-	static duckdb::unique_ptr<VortexFileReader> Open(const vx_file_open_options *options) {
-		vx_error *error;
-		auto file = vx_file_open_reader(options, &error);
-		HandleError(error);
-		return duckdb::make_uniq<VortexFileReader>(file);
+	static duckdb::unique_ptr<FileReader> Open(const vx_file_open_options *options) {
+		auto file = Try([&](auto err) { return vx_file_open_reader(options, err); });
+		return duckdb::make_uniq<FileReader>(file);
+	}
+
+	uint64_t FileRowCount() {
+		return Try([&](auto err) { return vx_file_row_count(file, err); });
+	}
+
+	struct DType DType() {
+		return vortex::DType(vx_file_dtype(file));
 	}
 
 	vx_file_reader *file;
 };
 
-struct VortexArray {
-	explicit VortexArray(vx_array *array) : array(array) {
+struct Array {
+	explicit Array(vx_array *array) : array(array) {
 	}
 
-	~VortexArray() {
+	~Array() {
 		if (array != nullptr) {
 			vx_array_free(array);
 		}
 	}
 
-	idx_t ToDuckDBVector(idx_t current_row, duckdb_data_chunk output, const VortexConversionCache *cache) const {
-		vx_error *error;
-		auto idx = vx_array_to_duckdb_chunk(array, current_row, output, cache->cache, &error);
-		HandleError(error);
-		return idx;
+	static duckdb::unique_ptr<Array> FromDuckDBChunk(DType &dtype, duckdb::DataChunk &chunk) {
+		auto array = Try([&](auto err) {
+			return vx_duckdb_chunk_to_array(reinterpret_cast<duckdb_data_chunk>(&chunk), dtype.dtype, err);
+		});
+
+		return duckdb::make_uniq<Array>(array);
+	}
+
+	idx_t ToDuckDBVector(idx_t current_row, duckdb_data_chunk output, const ConversionCache *cache) const {
+		return Try([&](auto err) { return vx_array_to_duckdb_chunk(array, current_row, output, cache->cache, err); });
 	}
 
 	vx_array *array;
 };
 
-struct VortexArrayStream {
-	explicit VortexArrayStream(vx_array_stream *array_stream) : array_stream(array_stream) {
+struct ArrayIterator {
+	explicit ArrayIterator(vx_array_iterator *array_iter) : array_iter(array_iter) {
 	}
 
-	~VortexArrayStream() {
-		vx_array_stream_free(array_stream);
+	~ArrayIterator() {
+		vx_array_iter_free(array_iter);
 	}
 
-	duckdb::unique_ptr<VortexArray> NextArray() const {
-		vx_error *error;
-		auto array = vx_array_stream_next(array_stream, &error);
-		HandleError(error);
+	duckdb::unique_ptr<Array> NextArray() const {
+		auto array = Try([&](auto err) { return vx_array_iter_next(array_iter, err); });
+
 		if (array == nullptr) {
 			return nullptr;
 		}
-		return duckdb::make_uniq<VortexArray>(array);
+
+		return duckdb::make_uniq<Array>(array);
 	}
 
-	vx_array_stream *array_stream;
+	vx_array_iterator *array_iter;
 };
+
+struct ArrayStreamSink {
+	explicit ArrayStreamSink(vx_array_sink *sink, duckdb::unique_ptr<DType> dtype)
+	    : sink(sink), dtype(std::move(dtype)) {
+	}
+
+	static duckdb::unique_ptr<ArrayStreamSink> Create(std::string file_path, duckdb::unique_ptr<DType> &&dtype) {
+		auto sink = Try([&](auto err) { return vx_array_sink_open_file(file_path.c_str(), dtype->dtype, err); });
+		return duckdb::make_uniq<ArrayStreamSink>(sink, std::move(dtype));
+	}
+
+	void PushChunk(duckdb::DataChunk &chunk) {
+		auto array = Array::FromDuckDBChunk(*dtype, chunk);
+		Try([&](auto err) { vx_array_sink_push(sink, array->array, err); });
+	}
+
+	void Close() {
+		Try([&](auto err) { vx_array_sink_close(sink, err); });
+		this->sink = nullptr;
+	}
+
+	~ArrayStreamSink() {
+		// "should dctor a sink, before closing it
+		D_ASSERT(sink == nullptr);
+	}
+
+	vx_array_sink *sink;
+	duckdb::unique_ptr<DType> dtype;
+};
+
+} // namespace vortex

src/include/vortex_error.hpp

@@ -1,11 +1,33 @@
 #pragma once
 
+#include <string>
+#include <type_traits>
+
+#include "duckdb.hpp"
 #include "vortex.hpp"
 
+namespace vortex {
+
 inline void HandleError(vx_error *error) {
 	if (error != nullptr) {
 		auto msg = std::string(vx_error_get_message(error));
 		vx_error_free(error);
 		throw duckdb::InvalidInputException(msg);
 	}
 }
+
+template <typename Func>
+auto Try(Func func) {
+	vx_error *error = nullptr;
+	// Handle both void and non-void return types.
+	if constexpr (std::is_void_v<std::invoke_result_t<Func, vx_error **>>) {
+		func(&error);
+		HandleError(error);
+	} else {
+		auto result = func(&error);
+		HandleError(error);
+		return result;
+	}
+}
+
+} // namespace vortex

src/include/expr/expr.hpp → src/include/vortex_expr.hpp

@@ -1,14 +1,14 @@
 #pragma once
 
-#include "expr.pb.h"
 #include "duckdb/planner/expression.hpp"
+#include "duckdb/planner/table_filter.hpp"
 
-#include <duckdb/planner/table_filter.hpp>
+#include "expr.pb.h"
 
+namespace vortex {
 vortex::expr::Expr *table_expression_into_expr(google::protobuf::Arena &arena, duckdb::TableFilter &filter,
                                                const std::string &column_name);
-
 vortex::expr::Expr *expression_into_vortex_expr(google::protobuf::Arena &arena, const duckdb::Expression &expr);
-
 vortex::expr::Expr *flatten_exprs(google::protobuf::Arena &arena,
-                                  const duckdb::vector<vortex::expr::Expr *> &child_filters);
+                                  const duckdb::vector<vortex::expr::Expr *> &child_filters);
+} // namespace vortex

src/include/vortex_extension.hpp

@@ -2,13 +2,11 @@
 
 #include "duckdb.hpp"
 
-namespace duckdb {
+// The entry point class API can't be scoped to the vortex namespace.
 
-class VortexExtension : public Extension {
+class VortexExtension : public duckdb::Extension {
 public:
-	void Load(DuckDB &db) override;
+	void Load(duckdb::DuckDB &db) override;
 	std::string Name() override;
 	std::string Version() const override;
 };
-
-} // namespace duckdb

src/include/vortex_layout_reader.hpp

@@ -0,0 +1,29 @@
+#pragma once
+
+#include "vortex_common.hpp"
+#include "vortex_expr.hpp"
+
+namespace vortex {
+
+class LayoutReader {
+public:
+	explicit LayoutReader(vx_layout_reader *reader) : reader(reader) {
+	}
+
+	~LayoutReader() {
+		vx_layout_reader_free(reader);
+	}
+
+	static std::shared_ptr<LayoutReader> CreateFromFile(vortex::FileReader *file) {
+		auto reader = Try([&](auto err) { return vx_layout_reader_create(file->file, err); });
+		return std::make_shared<LayoutReader>(reader);
+	}
+
+	vx_array_iterator *Scan(const vx_file_scan_options *options) {
+		return Try([&](auto err) { return vx_layout_reader_scan(this->reader, options, err); });
+	}
+
+	vx_layout_reader *reader;
+};
+
+} // namespace vortex

src/include/vortex_scan.hpp

@@ -2,9 +2,6 @@
 
 #include "duckdb/main/extension_util.hpp"
 
-
-namespace duckdb {
-
-void RegisterVortexScanFunction(DatabaseInstance &instance);
-
-}
+namespace vortex {
+void RegisterScanFunction(duckdb::DatabaseInstance &instance);
+}

src/include/vortex_write.hpp

@@ -2,6 +2,6 @@
 
 #include "duckdb/main/extension_util.hpp"
 
-namespace duckdb {
-	void RegisterVortexWriteFunction(duckdb::DatabaseInstance &instance);
-}
+namespace vortex {
+void RegisterWriteFunction(duckdb::DatabaseInstance &instance);
+}