feat: move to latest vortex and duckdb v1.3.1

Makefile

@@ -4,7 +4,7 @@ EXT_NAME=vortex_duckdb
 EXT_CONFIG=${PROJ_DIR}extension_config.cmake
 EXT_FLAGS=-DCMAKE_OSX_DEPLOYMENT_TARGET=12.0
 
-export OVERRIDE_GIT_DESCRIBE=v1.3.0
+export OVERRIDE_GIT_DESCRIBE=v1.3.1
 export MACOSX_DEPLOYMENT_TARGET=12.0
 export VCPKG_FEATURE_FLAGS=-binarycaching
 export VCPKG_OSX_DEPLOYMENT_TARGET=12.0

src/include/vortex_common.hpp

@@ -11,7 +11,7 @@
 namespace vortex {
 
 struct DType {
-	explicit DType(vx_dtype *dtype) : dtype(dtype) {
+	explicit DType(const vx_dtype *dtype) : dtype(dtype) {
 	}
 
 	static duckdb::unique_ptr<DType> FromDuckDBTable(const std::vector<duckdb_logical_type> &column_types,
@@ -34,57 +34,46 @@ struct DType {
 		}
 	}
 
-	vx_dtype *dtype;
+	const vx_dtype *dtype;
 };
 
-struct ConversionCache {
-	explicit ConversionCache(const unsigned long cache_id) : cache(vx_conversion_cache_create(cache_id)) {
+struct VortexFile {
+	explicit VortexFile(const vx_file *file) : file(file) {
 	}
 
-	~ConversionCache() {
-		vx_conversion_cache_free(cache);
+	~VortexFile() {
+		vx_file_free(file);
 	}
 
-	vx_conversion_cache *cache;
-};
-
-struct FileReader {
-	explicit FileReader(vx_file_reader *file) : file(file) {
-	}
-
-	~FileReader() {
-		vx_file_reader_free(file);
-	}
-
-	static duckdb::unique_ptr<FileReader> Open(const vx_file_open_options *options, VortexSession &session) {
+	static duckdb::unique_ptr<VortexFile> Open(const vx_file_open_options *options, VortexSession &session) {
 		auto file = Try([&](auto err) { return vx_file_open_reader(options, session.session, err); });
-		return duckdb::make_uniq<FileReader>(file);
+		return duckdb::make_uniq<VortexFile>(file);
 	}
 
 	vx_array_iterator *Scan(const vx_file_scan_options *options) {
-		return Try([&](auto err) { return vx_file_reader_scan(this->file, options, err); });
+		return Try([&](auto err) { return vx_file_scan(this->file, options, err); });
 	}
 
-	bool CanPrune(const char *filter_expression, unsigned int filter_expression_len) {
+	bool CanPrune(const char *filter_expression, unsigned int filter_expression_len, unsigned long file_idx) {
 		return Try([&](auto err) {
-			return vx_file_reader_can_prune(this->file, filter_expression, filter_expression_len, err);
+			return vx_file_can_prune(this->file, filter_expression, filter_expression_len, file_idx, err);
 		});
 	}
 
-	uint64_t FileRowCount() {
-		return Try([&](auto err) { return vx_file_row_count(file, err); });
+	uint64_t RowCount() {
+		return vx_file_row_count(file);
 	}
 
 	struct DType DType() {
-		return vortex::DType(vx_file_dtype(file));
+		return vortex::DType(vx_dtype_clone(vx_file_dtype(file)));
 	}
 
-	vx_file_reader *file;
+	const vx_file *file;
 };
 
 
 struct Array {
-	explicit Array(vx_array *array) : array(array) {
+	explicit Array(const vx_array *array) : array(array) {
 	}
 
 	~Array() {
@@ -101,11 +90,7 @@ struct Array {
 		return duckdb::make_uniq<Array>(array);
 	}
 
-	idx_t ToDuckDBVector(idx_t current_row, duckdb_data_chunk output, const ConversionCache *cache) const {
-		return Try([&](auto err) { return vx_array_to_duckdb_chunk(array, current_row, output, cache->cache, err); });
-	}
-
-	vx_array *array;
+	const vx_array *array;
 };
 
 
@@ -125,12 +110,12 @@ struct ArrayIterator {
 
 	~ArrayIterator() {
 		if (array_iter) {
-			vx_array_iter_free(array_iter);
+			vx_array_iterator_free(array_iter);
 		}
 	}
 
 	duckdb::unique_ptr<Array> NextArray() const {
-		auto array = Try([&](auto err) { return vx_array_iter_next(array_iter, err); });
+		auto array = Try([&](auto err) { return vx_array_iterator_next(array_iter, err); });
 
 		if (array == nullptr) {
 			return nullptr;
@@ -154,9 +139,7 @@ struct ArrayExporter {
 	}
 
 	static duckdb::unique_ptr<ArrayExporter> FromArrayIterator(duckdb::unique_ptr<ArrayIterator> array_iter) {
-		auto exporter = Try([&](auto err) {
-			return vx_duckdb_exporter_create(array_iter->release(), err);
-		});
+		auto exporter = vx_duckdb_exporter_new(array_iter->release());
 		return duckdb::make_uniq<ArrayExporter>(exporter);
 	}
 

src/include/vortex_error.hpp

@@ -10,7 +10,8 @@ namespace vortex {
 
 inline void HandleError(vx_error *error) {
 	if (error != nullptr) {
-		auto msg = std::string(vx_error_get_message(error));
+		auto msg_str = vx_error_get_message(error);
+		auto msg = std::string(vx_string_ptr(msg_str), vx_string_len(msg_str));
 		vx_error_free(error);
 		throw duckdb::InvalidInputException(msg);
 	}

src/include/vortex_expr.hpp

@@ -10,14 +10,17 @@ namespace vortex {
 const std::string BETWEEN_ID = "between";
 const std::string BINARY_ID = "binary";
 const std::string GET_ITEM_ID = "get_item";
-const std::string IDENTITY_ID = "identity";
+const std::string PACK_ID = "pack";
+const std::string VAR_ID = "var";
 const std::string LIKE_ID = "like";
 const std::string LITERAL_ID = "literal";
 const std::string NOT_ID = "not";
+const std::string LIST_CONTAINS_ID = "list_contains";
 
 vortex::expr::Expr *table_expression_into_expr(google::protobuf::Arena &arena, duckdb::TableFilter &filter,
                                                const std::string &column_name);
 vortex::expr::Expr *expression_into_vortex_expr(google::protobuf::Arena &arena, const duckdb::Expression &expr);
 vortex::expr::Expr *flatten_exprs(google::protobuf::Arena &arena,
                                   const duckdb::vector<vortex::expr::Expr *> &child_filters);
+vortex::expr::Expr *pack_projection_columns(google::protobuf::Arena &arena, duckdb::vector<std::string> columns);
 } // namespace vortex

src/include/vortex_session.hpp

@@ -7,7 +7,7 @@ namespace vortex {
 
 class VortexSession : public duckdb::ObjectCacheEntry {
 public:
-	VortexSession() : session(vx_session_create()) {
+	VortexSession() : session(vx_session_new()) {
 	}
 
 	~VortexSession() override {