diff --git a/.github/workflows/ExtensionTemplate.yml b/.github/workflows/ExtensionTemplate.yml index 16faf17..fe6e74c 100644 --- a/.github/workflows/ExtensionTemplate.yml +++ b/.github/workflows/ExtensionTemplate.yml @@ -34,7 +34,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - submodules: 'true' + submodules: 'recursive' - name: Checkout DuckDB to version if: ${{ matrix.duckdb_version != ''}} @@ -80,7 +80,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - submodules: 'true' + submodules: 'recursive' - name: Install Ninja run: brew install ninja @@ -131,7 +131,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - submodules: 'true' + submodules: 'recursive' - uses: actions/setup-python@v2 with: diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 24525fd..e576646 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -3,27 +3,20 @@ # name: Main Extension Distribution Pipeline on: - push: pull_request: - workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} cancel-in-progress: true jobs: - duckdb-next-build: - name: Build extension binaries - uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main - with: - duckdb_version: main - ci_tools_version: main - extension_name: quack - duckdb-stable-build: name: Build extension binaries - uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.2.1 + uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@6a7a4f24c5999355ab36c0a6835baf891fc9d522 with: - duckdb_version: v1.2.1 - ci_tools_version: v1.2.1 - extension_name: quack \ No newline at end of file + duckdb_version: v1.2.2 + ci_tools_version: 6a7a4f24c5999355ab36c0a6835baf891fc9d522 + extension_name: vortex + exclude_archs: "wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw;windows_amd64;linux_arm64" + extra_toolchains: "rust" + override_ci_tools_repository: spiraldb/extension-ci-tools diff --git a/.gitmodules b/.gitmodules index d07f00b..c47fc9c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -8,4 +8,7 @@ branch = main [submodule "duckdb"] path = duckdb - url = https://github.com/duckdb/duckdb.git + url = https://github.com/spiraldb/duckdb.git +[submodule "vcpkg"] + path = vcpkg + url = git@github.com:microsoft/vcpkg.git diff --git a/CMakeLists.txt b/CMakeLists.txt index e75e430..9bfd569 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,52 +4,24 @@ set(TARGET_NAME vortex) project(${TARGET_NAME}_project) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) -set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD 17) include(FetchContent) FetchContent_Declare( Corrosion GIT_REPOSITORY https://github.com/corrosion-rs/corrosion.git - GIT_TAG v0.5.1 -) -FetchContent_Declare( - vcpkg - GIT_REPOSITORY https://github.com/microsoft/vcpkg.git - GIT_TAG 2025.03.19 -) -FetchContent_MakeAvailable(Corrosion) -FetchContent_MakeAvailable(vcpkg) - -# Install vcpkg dependencies. -execute_process( - COMMAND bash ./bootstrap-vcpkg.sh -disableMetrics - WORKING_DIRECTORY "${vcpkg_SOURCE_DIR}" - RESULT_VARIABLE VCPKG_BOOTSTRAP_RESULT -) -set(VCPKG_INSTALL_DIR "${CMAKE_BINARY_DIR}/vcpkg_installed") -file(MAKE_DIRECTORY "${VCPKG_INSTALL_DIR}") -execute_process( - COMMAND "${vcpkg_SOURCE_DIR}/vcpkg" install --no-print-usage --x-manifest-root=${CMAKE_CURRENT_SOURCE_DIR}/vcpkg --x-install-root=${VCPKG_INSTALL_DIR} - WORKING_DIRECTORY "${vcpkg_SOURCE_DIR}" + GIT_TAG v0.5.2 ) -if (APPLE AND CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") - set(CMAKE_PREFIX_PATH ${VCPKG_INSTALL_DIR}/arm64-osx) -elseif (APPLE) - set(CMAKE_PREFIX_PATH ${VCPKG_INSTALL_DIR}/x64-osx) -elseif (UNIX AND NOT APPLE) - set(CMAKE_PREFIX_PATH ${VCPKG_INSTALL_DIR}/x64-linux) -endif () +FetchContent_MakeAvailable(Corrosion) find_package(Catch2 CONFIG REQUIRED) find_package(Protobuf CONFIG REQUIRED) if (APPLE) - find_library(CORE_FOUNDATION_FRAMEWORK CoreFoundation) find_library(SECURITY_FRAMEWORK Security) endif () corrosion_import_crate(MANIFEST_PATH vortex/Cargo.toml -# CORROSION_TOOLS_RUST_TOOLCHAIN "nightly-2025-02-24" CRATES vortex-ffi FEATURES duckdb IMPORTED_CRATES vortex_ffi @@ -73,13 +45,11 @@ build_loadable_extension(${TARGET_NAME} ${EXTENSION_SOURCES} ${PROTO_SRCS}) target_link_libraries(${EXTENSION_NAME} vortex_ffi-static protobuf::libprotobuf - ${CORE_FOUNDATION_FRAMEWORK} ${SECURITY_FRAMEWORK} ) target_link_libraries(${LOADABLE_EXTENSION_NAME} vortex_ffi-shared protobuf::libprotobuf - ${CORE_FOUNDATION_FRAMEWORK} ${SECURITY_FRAMEWORK} ) diff --git a/Makefile b/Makefile index 9f07bea..d8b80f9 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,12 @@ PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) -# Configuration of extension EXT_NAME=vortex_duckdb EXT_CONFIG=${PROJ_DIR}extension_config.cmake -EXT_FLAGS=-DDISABLE_VPTR_SANITIZER=ON -DOVERRIDE_GIT_DESCRIBE=v1.2.2 -EXT_FLAGS += -DCMAKE_OSX_DEPLOYMENT_TARGET=13.0 +EXT_FLAGS=-DCMAKE_OSX_DEPLOYMENT_TARGET=12.0 -DOVERRIDE_GIT_DESCRIBE=v1.2.2 -export MACOSX_DEPLOYMENT_TARGET=13.0 -export VCPKG_OSX_DEPLOYMENT_TARGET=13.0 +export MACOSX_DEPLOYMENT_TARGET=12.0 +export VCPKG_OSX_DEPLOYMENT_TARGET=12.0 export VCPKG_FEATURE_FLAGS=-binarycaching +export VCPKG_TOOLCHAIN_PATH := ${PROJ_DIR}vcpkg/scripts/buildsystems/vcpkg.cmake -# Include the Makefile from extension-ci-tools include extension-ci-tools/makefiles/duckdb_extension.Makefile diff --git a/duckdb b/duckdb index 7c03946..bf7c548 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit 7c039464e452ddc3330e2691d3fa6d305521d09b +Subproject commit bf7c5488f6eacfef3f719f1979aa9c01b0573185 diff --git a/extension-ci-tools b/extension-ci-tools index 58970c5..6a7a4f2 160000 --- a/extension-ci-tools +++ b/extension-ci-tools @@ -1 +1 @@ -Subproject commit 58970c538d35919db875096460c05806056f4de0 +Subproject commit 6a7a4f24c5999355ab36c0a6835baf891fc9d522 diff --git a/src/include/vortex_error.hpp b/src/include/vortex_error.hpp index dbfa90f..9600285 100644 --- a/src/include/vortex_error.hpp +++ b/src/include/vortex_error.hpp @@ -3,9 +3,9 @@ #include "vortex.hpp" inline void HandleError(vx_error *error) { - if (error != nullptr && error->code != 0) { - auto msg = std::string(error->message); + if (error != nullptr) { + auto msg = std::string(vx_error_get_message(error)); vx_error_free(error); throw duckdb::InvalidInputException(msg); } -} \ No newline at end of file +} diff --git a/src/vortex_extension.cpp b/src/vortex_extension.cpp index 542ad99..121647b 100644 --- a/src/vortex_extension.cpp +++ b/src/vortex_extension.cpp @@ -5,8 +5,6 @@ #include "duckdb/main/extension_util.hpp" #include "vortex_extension.hpp" -#include - #include "vortex_write.hpp" #include "vortex_scan.hpp" diff --git a/src/vortex_scan.cpp b/src/vortex_scan.cpp index 6a1436a..be6fbac 100644 --- a/src/vortex_scan.cpp +++ b/src/vortex_scan.cpp @@ -5,9 +5,10 @@ #include "duckdb/common/multi_file_reader.hpp" #include "duckdb/function/table_function.hpp" #include "duckdb/main/extension_util.hpp" +#include "duckdb/common/file_system.hpp" #include "vortex_extension.hpp" -#include +#include #include "vortex_common.hpp" #include "expr/expr.hpp" @@ -150,23 +151,20 @@ static void ExtractVortexSchema(const vx_dtype *file_dtype, vector } } -std::string EnsureFileProtocol(const std::string &path) { - auto absolute_path = path; - const std::string prefix = "file://"; +const std::regex schema_prefix = std::regex("^[^/]*:\\/\\/.*$"); - std::filesystem::path p = absolute_path; - if (!p.is_absolute()) { - try { - absolute_path = absolute(p).string(); - } catch (const std::exception &e) { - throw InternalException(std::string("Error making path absolute: ") + e.what()); - } +std::string EnsureFileProtocol(FileSystem &fs, const std::string &path) { + // If the path is a URL then don't change it, otherwise try to make the path an absolute path + if (std::regex_match(path, schema_prefix)) { + return path; } - // Check if the string already starts with "file://" - if (absolute_path.size() >= prefix.size() && std::equal(prefix.begin(), prefix.end(), absolute_path.begin())) { - return absolute_path; + const std::string prefix = "file://"; + if (fs.IsPathAbsolute(path)) { + return prefix + path; } + + const auto absolute_path = fs.JoinPath(fs.GetWorkingDirectory(), path); return prefix + absolute_path; } @@ -209,13 +207,14 @@ static void VerifyNewFile(const VortexBindData &bind_data, vector & } } -static unique_ptr OpenFileAndVerify(const std::string &filename, const VortexBindData &bind_data) { +static unique_ptr OpenFileAndVerify(FileSystem &fs, const std::string &filename, + const VortexBindData &bind_data) { auto new_column_names = vector(); new_column_names.reserve(bind_data.column_names.size()); auto new_column_types = vector(); new_column_names.reserve(bind_data.columns_types.size()); - auto file = OpenFile(EnsureFileProtocol(filename), new_column_types, new_column_names); + auto file = OpenFile(EnsureFileProtocol(fs, filename), new_column_types, new_column_names); VerifyNewFile(bind_data, new_column_types, new_column_names); return file; } @@ -268,7 +267,7 @@ static void VortexScanFunction(ClientContext &context, TableFunctionInput &data, } auto file_name = global_state.expanded_files[file_idx]; - auto file = OpenFileAndVerify(file_name, bind_data); + auto file = OpenFileAndVerify(FileSystem::GetFileSystem(context), file_name, bind_data); slot.array_stream = OpenArrayStream(bind_data, global_state, file.get()); } @@ -309,7 +308,7 @@ static unique_ptr VortexBind(ClientContext &context, TableFunction auto vec = duckdb::vector {input.inputs[0].GetValue()}; result->file_list = make_shared_ptr(context, vec, FileGlobOptions::DISALLOW_EMPTY); - auto filename = EnsureFileProtocol(result->file_list->GetFirstFile()); + auto filename = EnsureFileProtocol(FileSystem::GetFileSystem(context), result->file_list->GetFirstFile()); result->initial_file = OpenFile(filename, column_types, column_names); @@ -366,7 +365,7 @@ void RegisterVortexScanFunction(DatabaseInstance &instance) { // Most expressions are extracted from `PushdownComplexFilter`, the final filters come from `input.filters`. vector conjuncts; - std::ranges::copy(bind.conjuncts, std::back_inserter(conjuncts)); + std::copy(bind.conjuncts.begin(), bind.conjuncts.end(), std::back_inserter(conjuncts)); CreateFilterExpression(*bind.arena, bind.column_names, input.filters, input.column_ids, conjuncts); auto column_names = std::vector(); diff --git a/vcpkg b/vcpkg new file mode 160000 index 0000000..d6995a0 --- /dev/null +++ b/vcpkg @@ -0,0 +1 @@ +Subproject commit d6995a0cf3cafda5e9e52749fad075dd62bfd90c diff --git a/vcpkg.json b/vcpkg.json new file mode 100644 index 0000000..cf477f0 --- /dev/null +++ b/vcpkg.json @@ -0,0 +1,14 @@ +{ + "dependencies": [{ "name": "catch2" }, { "name": "protobuf" } ], + "vcpkg-configuration": { + "registries": [ + { + "kind": "git", + "repository": "https://github.com/duckdb/vcpkg-duckdb-ports", + "baseline": "c8f9d504986e76dbcd4585a34028ff00e251f40f", + "packages": [ "vcpkg-cmake" ] + } + ] + }, + "builtin-baseline" : "5e5d0e1cd7785623065e77eff011afdeec1a3574" +} diff --git a/vcpkg/vcpkg.json b/vcpkg/vcpkg.json deleted file mode 100644 index 61c33b9..0000000 --- a/vcpkg/vcpkg.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "dependencies": [{ "name": "catch2" }, { "name": "protobuf" }] -} diff --git a/vortex b/vortex index 07017c1..3165cfd 160000 --- a/vortex +++ b/vortex @@ -1 +1 @@ -Subproject commit 07017c1faf09a2805cf7080dad2fbffc113b808f +Subproject commit 3165cfd612236d367dfd086c18937cbd6c35273a