chore: build extension with rust v2 rewrite (#14)

* chore: build extension with rust v2 rewrite

* install llvm

* vortex latest

* libclang path

* try libclang via apt get

* libclang via apt-get

* drop log lines

* wip

Signed-off-by: Alexander Droste <[email protected]>

* bump vortex

Signed-off-by: Alexander Droste <[email protected]>

* wip

Signed-off-by: Alexander Droste <[email protected]>

* update

---------

Signed-off-by: Alexander Droste <[email protected]>
Co-authored-by: Joe Isaacs <[email protected]>

Author: 0ax1

.github/workflows/MainDistributionPipeline.yml

@@ -14,10 +14,10 @@ concurrency:
 jobs:
   duckdb-stable-build:
     name: Build extension binaries
-    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@6a7a4f24c5999355ab36c0a6835baf891fc9d522
+    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.3.2
     with:
-      ci_tools_version: 1f00107ca1eaf1691049907296a8aa796d054e6b
-      duckdb_version: ad1273222186d28b4b351736ed88101044bbe97b
+      ci_tools_version: v1.3.2
+      duckdb_version: v1.3.2
       extension_name: vortex
       exclude_archs: "wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64_mingw;windows_amd64;linux_arm64"
       extra_toolchains: "rust"

.gitmodules

@@ -1,14 +1,14 @@
+[submodule "duckdb"]
+	path = duckdb
+	url = https://github.com/duckdb/duckdb.git
 [submodule "extension-ci-tools"]
 	path = extension-ci-tools
 	url = https://github.com/duckdb/extension-ci-tools
 	branch = main
+[submodule "vcpkg"]
+	path = vcpkg
+	url = https://github.com/microsoft/vcpkg.git
 [submodule "vortex"]
 	path = vortex
 	url = https://github.com/spiraldb/vortex
 	branch = main
-[submodule "duckdb"]
-	path = duckdb
-	url = https://github.com/spiraldb/duckdb.git
-[submodule "vcpkg"]
-	path = vcpkg
-	url = [email protected]:microsoft/vcpkg.git

CMakeLists.txt

@@ -6,9 +6,6 @@ project(${TARGET_NAME}_project)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 set(CMAKE_CXX_STANDARD 17)
 
-# Allow C++20 designator syntax in C++17
-add_compile_options(-Wno-c++20-designator)
-
 include(FetchContent)
 FetchContent_Declare(
         Corrosion
@@ -18,44 +15,64 @@ FetchContent_Declare(
 
 FetchContent_MakeAvailable(Corrosion)
 
-find_package(Catch2 CONFIG REQUIRED)
-find_package(Protobuf CONFIG REQUIRED)
-
 if (APPLE)
     find_library(SECURITY_FRAMEWORK Security)
+    find_library(CORE_FOUNDATION_FRAMEWORK CoreFoundation)
 endif ()
 
+# Install clang and libclang for Linux Docker builds. macOS has both of these pre-installed.
+if(EXISTS "/.dockerenv")
+    find_program(YUM_EXEC yum)
+    if(YUM_EXEC AND NOT INSTALL_SUCCESS)
+        execute_process(
+        COMMAND yum install -y clang-devel clang
+        RESULT_VARIABLE YUM_RESULT
+    )
+        if(YUM_RESULT EQUAL 0)
+            set(INSTALL_SUCCESS TRUE)
+        endif()
+    endif()
+
+    find_program(APK_EXEC apk)
+    if(APK_EXEC AND NOT INSTALL_SUCCESS)
+        execute_process(
+        COMMAND apk add --no-cache clang-dev clang
+        RESULT_VARIABLE APK_RESULT
+    )
+        if(APK_RESULT EQUAL 0)
+            set(INSTALL_SUCCESS TRUE)
+        endif()
+    endif()
+endif()
+
 corrosion_import_crate(MANIFEST_PATH vortex/Cargo.toml
         CRATES vortex-ffi
-        FEATURES duckdb mimalloc
+        FEATURES mimalloc
         CRATE_TYPES staticlib
         FLAGS --crate-type=staticlib
 )
 
-set(EXTENSION_NAME ${TARGET_NAME}_extension)
-set(EXTENSION_SOURCES src/vortex_extension.cpp src/vortex_expr.cpp src/vortex_write.cpp src/vortex_scan.cpp)
-set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension)
+corrosion_import_crate(MANIFEST_PATH vortex/Cargo.toml
+        CRATES vortex-duckdb
+        CRATE_TYPES staticlib
+        FLAGS --crate-type=staticlib
+)
 
-# Generate C++ code from .proto files.
-file(GLOB PROTO_FILES "vortex/vortex-proto/proto/*.proto")
-set(PROTO_GEN_DIR ${CMAKE_CURRENT_SOURCE_DIR}/gen)
-file(MAKE_DIRECTORY ${PROTO_GEN_DIR})
-protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS ${PROTO_FILES} PROTOC_OUT_DIR ${PROTO_GEN_DIR})
+include_directories(src/include vortex/vortex-duckdb/include)
 
-include_directories(src/include ${PROTO_GEN_DIR} vortex/vortex-ffi/cinclude)
+set(EXTENSION_NAME ${TARGET_NAME}_extension)
+set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension)
 
-build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES} ${PROTO_SRCS})
-set(PARAMETERS "-warnings")
-build_loadable_extension(${TARGET_NAME} ${PARAMETERS} ${EXTENSION_SOURCES} ${PROTO_SRCS})
+build_static_extension(${TARGET_NAME} src/vortex_extension.cpp)
+build_loadable_extension(${TARGET_NAME} -warnings src/vortex_extension.cpp)
 
 target_link_libraries(${EXTENSION_NAME}
+        vortex_duckdb-static
         vortex_ffi-static
-        protobuf::libprotobuf
         ${SECURITY_FRAMEWORK}
+        ${CORE_FOUNDATION_FRAMEWORK}
 )
 
-add_subdirectory(test)
-
 install(
         TARGETS ${EXTENSION_NAME}
         EXPORT "${DUCKDB_EXPORT_SET}"

Makefile

@@ -3,8 +3,6 @@ PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 EXT_NAME=vortex_duckdb
 EXT_CONFIG=${PROJ_DIR}extension_config.cmake
 EXT_FLAGS=-DCMAKE_OSX_DEPLOYMENT_TARGET=12.0
-
-export OVERRIDE_GIT_DESCRIBE=v1.3.2
 export MACOSX_DEPLOYMENT_TARGET=12.0
 export VCPKG_FEATURE_FLAGS=-binarycaching
 export VCPKG_OSX_DEPLOYMENT_TARGET=12.0