feat: Support macOS builds with AppleClang and Homebrew Clang

CMakeLists.txt

@@ -54,7 +54,6 @@ option(PAIMON_ENABLE_LANCE "Whether to enable lance file format" OFF)
 option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF)
 option(PAIMON_ENABLE_LUMINA "Whether to enable lumina vector index" OFF)
 option(PAIMON_ENABLE_LUCENE "Whether to enable lucene index" OFF)
-
 if(PAIMON_ENABLE_ORC)
     add_definitions(-DPAIMON_ENABLE_ORC)
 endif()
@@ -296,8 +295,6 @@ if(PAIMON_ENABLE_LANCE)
             DESTINATION ${CMAKE_INSTALL_LIBDIR})
 endif()
 
-list(APPEND PAIMON_LINK_LIBS ${CMAKE_DL_LIBS})
-list(APPEND PAIMON_SHARED_INSTALL_INTERFACE_LIBS ${CMAKE_DL_LIBS})
 if(PAIMON_ENABLE_LUMINA)
     add_subdirectory(third_party/lumina EXCLUDE_FROM_ALL)
     link_directories(third_party/lumina/lib)
@@ -349,8 +346,11 @@ add_compile_definitions("GLOG_USE_GLOG_EXPORT")
 
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
-set(PAIMON_VERSION_SCRIPT_FLAGS
-    "-Wl,--version-script=${CMAKE_SOURCE_DIR}/src/paimon/symbols.map")
+set(PAIMON_VERSION_SCRIPT_FLAGS)
+if(NOT APPLE)
+    set(PAIMON_VERSION_SCRIPT_FLAGS
+        "-Wl,--version-script=${CMAKE_SOURCE_DIR}/src/paimon/symbols.map")
+endif()
 
 set(ENV{PAIMON_TEST_DATA} "${CMAKE_SOURCE_DIR}/test/test_data")
 
@@ -378,47 +378,66 @@ if(PAIMON_BUILD_TESTS)
     include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
     include_directories("${CMAKE_SOURCE_DIR}/test/")
 
-    set(TEST_STATIC_LINK_LIBS
-        "-Wl,--whole-archive"
+    paimon_link_libraries_whole_archive(
+        TEST_WHOLE_ARCHIVE_LINK_LIBS
         paimon_file_index_static
         paimon_global_index_static
         paimon_local_file_system_static
-        paimon_mock_file_format_static
-        "-Wl,--no-whole-archive"
-        "-Wl,--no-as-needed"
-        paimon_parquet_file_format_shared
-        paimon_blob_file_format_shared
-        "-Wl,--as-needed")
+        paimon_mock_file_format_static)
+    paimon_link_libraries_no_as_needed(
+        TEST_PLUGIN_LINK_LIBS paimon_parquet_file_format_shared
+        paimon_blob_file_format_shared)
+    set(TEST_STATIC_LINK_LIBS ${TEST_WHOLE_ARCHIVE_LINK_LIBS} ${TEST_PLUGIN_LINK_LIBS})
+    paimon_link_libraries_whole_archive(PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS
+                                        paimon_local_file_system_static)
+    paimon_link_libraries_no_as_needed(PAIMON_LOCAL_FILE_SYSTEM_SHARED_LINK_LIBS
+                                       paimon_local_file_system_shared)
+    paimon_link_libraries_whole_archive(PAIMON_BLOB_FILE_FORMAT_STATIC_LINK_LIBS
+                                        paimon_blob_file_format_static)
+    paimon_link_libraries_whole_archive(PAIMON_PARQUET_FILE_FORMAT_STATIC_LINK_LIBS
+                                        paimon_parquet_file_format_static)
 
     if(PAIMON_ENABLE_LANCE)
-        list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed")
-        list(APPEND TEST_STATIC_LINK_LIBS paimon_lance_file_format_shared)
-        list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed")
+        paimon_link_libraries_whole_archive(PAIMON_LANCE_FILE_FORMAT_STATIC_LINK_LIBS
+                                            paimon_lance_file_format_static)
+        paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
+                                           paimon_lance_file_format_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
     endif()
     if(PAIMON_ENABLE_ORC)
-        list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed")
-        list(APPEND TEST_STATIC_LINK_LIBS paimon_orc_file_format_shared)
-        list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed")
+        paimon_link_libraries_whole_archive(PAIMON_ORC_FILE_FORMAT_STATIC_LINK_LIBS
+                                            paimon_orc_file_format_static)
+        paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
+                                           paimon_orc_file_format_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
     endif()
     if(PAIMON_ENABLE_AVRO)
-        list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed")
-        list(APPEND TEST_STATIC_LINK_LIBS paimon_avro_file_format_shared)
-        list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed")
+        paimon_link_libraries_whole_archive(PAIMON_AVRO_FILE_FORMAT_STATIC_LINK_LIBS
+                                            paimon_avro_file_format_static)
+        paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
+                                           paimon_avro_file_format_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
     endif()
     if(PAIMON_ENABLE_JINDO)
-        list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed")
-        list(APPEND TEST_STATIC_LINK_LIBS paimon_jindo_file_system_shared)
-        list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed")
+        paimon_link_libraries_whole_archive(PAIMON_JINDO_FILE_SYSTEM_STATIC_LINK_LIBS
+                                            paimon_jindo_file_system_static)
+        paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
+                                           paimon_jindo_file_system_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
     endif()
     if(PAIMON_ENABLE_LUMINA)
-        list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed")
-        list(APPEND TEST_STATIC_LINK_LIBS paimon_lumina_index_shared)
-        list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed")
+        paimon_link_libraries_whole_archive(PAIMON_LUMINA_INDEX_STATIC_LINK_LIBS
+                                            paimon_lumina_index_static)
+        paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
+                                           paimon_lumina_index_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
     endif()
     if(PAIMON_ENABLE_LUCENE)
-        list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed")
-        list(APPEND TEST_STATIC_LINK_LIBS paimon_lucene_index_shared)
-        list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed")
+        paimon_link_libraries_whole_archive(PAIMON_LUCENE_INDEX_STATIC_LINK_LIBS
+                                            paimon_lucene_index_static)
+        paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
+                                           paimon_lucene_index_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
     endif()
 endif()
 

README.md

@@ -36,7 +36,7 @@ Paimon C++ is a high-performance C++ implementation of [Apache Paimon](https://p
 - **Compatibility**: compatibility with Apache Paimon Java format and communication protocols,
   including commit messages, data splits, and manifests.
 
-Note: The current implementation only supports the x86_64 architecture.
+Note: Linux x86_64 and macOS arm64 builds are currently verified.
 
 ## Write And Commit Example
 

build_support/asan_symbolize.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #===- lib/asan/scripts/asan_symbolize.py -----------------------------------===#
 #
 #                     The LLVM Compiler Infrastructure

build_support/iwyu/iwyu.sh

@@ -47,7 +47,7 @@ affected_files() {
 include-what-you-use --version
 
 if [[ "${1:-}" == "all" ]]; then
-    python $ROOT/build_support/iwyu/iwyu_tool.py -p ${IWYU_COMPILATION_DATABASE_PATH:-.} \
+    ${PYTHON:-python3} $ROOT/build_support/iwyu/iwyu_tool.py -p ${IWYU_COMPILATION_DATABASE_PATH:-.} \
         -- $IWYU_ARGS
      #| awk -f $ROOT/build_support/iwyu/iwyu-filter.awk
 elif [[ "${1:-}" == "match" ]]; then
@@ -60,7 +60,7 @@ elif [[ "${1:-}" == "match" ]]; then
   done
 
   echo "Running IWYU on $IWYU_FILE_LIST"
-  python $ROOT/build_support/iwyu/iwyu_tool.py \
+  ${PYTHON:-python3} $ROOT/build_support/iwyu/iwyu_tool.py \
       -p ${IWYU_COMPILATION_DATABASE_PATH:-.} $IWYU_FILE_LIST  -- \
        $IWYU_ARGS | awk -f $ROOT/build_support/iwyu/iwyu-filter.awk
 else
@@ -77,7 +77,7 @@ else
     IWYU_FILE_LIST="$IWYU_FILE_LIST $ROOT/$p"
   done
 
-  python $ROOT/build_support/iwyu/iwyu_tool.py \
+  ${PYTHON:-python3} $ROOT/build_support/iwyu/iwyu_tool.py \
       -p ${IWYU_COMPILATION_DATABASE_PATH:-.} $IWYU_FILE_LIST  -- \
        $IWYU_ARGS | awk -f $ROOT/build_support/iwyu/iwyu-filter.awk > $IWYU_LOG
 fi

build_support/iwyu/iwyu_tool.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # This file has been imported into the apache source tree from
 # the IWYU source tree as of version 0.8
@@ -65,7 +65,7 @@
     -DCMAKE_C_COMPILER="%VCINSTALLDIR%/VC/bin/cl.exe" \
     -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
     -G Ninja ...
-  $ python iwyu_tool.py -p .
+  $ python3 iwyu_tool.py -p .
 
 See iwyu_tool.py -h for more details on command-line arguments.
 """

build_support/run_clang_format.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information

build_support/run_clang_tidy.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information

cmake_modules/BuildUtils.cmake

@@ -18,6 +18,43 @@
 # Borrowed the file from Apache Arrow:
 # https://github.com/apache/arrow/blob/main/cpp/cmake_modules/BuildUtils.cmake
 
+function(paimon_link_libraries_whole_archive OUT_VAR)
+    set(_paimon_whole_archive_libs)
+    if(APPLE)
+        foreach(_paimon_lib IN LISTS ARGN)
+            list(APPEND _paimon_whole_archive_libs
+                 "-Wl,-force_load,$<TARGET_FILE:${_paimon_lib}>" ${_paimon_lib})
+        endforeach()
+    else()
+        list(APPEND
+             _paimon_whole_archive_libs
+             "-Wl,--whole-archive"
+             ${ARGN}
+             "-Wl,--no-whole-archive")
+    endif()
+    set(${OUT_VAR}
+        ${_paimon_whole_archive_libs}
+        PARENT_SCOPE)
+endfunction()
+
+function(paimon_link_libraries_no_as_needed OUT_VAR)
+    set(_paimon_link_libs)
+    foreach(_paimon_lib IN LISTS ARGN)
+        if(APPLE)
+            list(APPEND _paimon_link_libs ${_paimon_lib})
+        else()
+            list(APPEND
+                 _paimon_link_libs
+                 "-Wl,--no-as-needed"
+                 ${_paimon_lib}
+                 "-Wl,--as-needed")
+        endif()
+    endforeach()
+    set(${OUT_VAR}
+        ${_paimon_link_libs}
+        PARENT_SCOPE)
+endfunction()
+
 function(add_paimon_lib LIB_NAME)
     set(options BUILD_SHARED BUILD_STATIC)
     set(one_value_args SHARED_LINK_FLAGS)
@@ -139,12 +176,14 @@ function(add_paimon_lib LIB_NAME)
         target_link_libraries(${LIB_NAME}_shared
                               PUBLIC "$<BUILD_INTERFACE:paimon_sanitizer_flags>")
 
-        target_link_options(${LIB_NAME}_shared
-                            PRIVATE
-                            -Wl,--exclude-libs,ALL
-                            -Wl,-Bsymbolic
-                            -Wl,-z,defs
-                            -Wl,--gc-sections)
+        if(NOT APPLE)
+            target_link_options(${LIB_NAME}_shared
+                                PRIVATE
+                                -Wl,--exclude-libs,ALL
+                                -Wl,-Bsymbolic
+                                -Wl,-z,defs
+                                -Wl,--gc-sections)
+        endif()
 
         install(TARGETS ${LIB_NAME}_shared ${INSTALL_IS_OPTIONAL}
                 EXPORT ${LIB_NAME}_targets

cmake_modules/DefineOptions.cmake

@@ -98,6 +98,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
 
     define_option(PAIMON_USE_CCACHE "Use ccache when compiling (if available)" ON)
 
+    define_option(PAIMON_USE_APPLE_LIBCXX_WITH_CLANG
+                  "Use Apple SDK libc++ headers when building with upstream Clang on macOS"
+                  ON)
+
     #----------------------------------------------------------------------
     set_option_category("Test")
 

cmake_modules/SetupCxxFlags.cmake

@@ -196,6 +196,24 @@ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STRE
         # the default standard library which does not support C++11. libc++ is the
         # default from 10.9 onward.
         set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -stdlib=libc++")
+    elseif(APPLE AND PAIMON_USE_APPLE_LIBCXX_WITH_CLANG)
+        execute_process(COMMAND xcrun --show-sdk-path
+                        OUTPUT_VARIABLE PAIMON_MACOS_SDK_PATH
+                        OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET)
+        if(NOT PAIMON_MACOS_SDK_PATH)
+            message(FATAL_ERROR "PAIMON_USE_APPLE_LIBCXX_WITH_CLANG is enabled, but xcrun could not find the macOS SDK"
+            )
+        endif()
+        set(PAIMON_MACOS_LIBCXX_INCLUDE_DIR "${PAIMON_MACOS_SDK_PATH}/usr/include/c++/v1")
+        if(NOT EXISTS "${PAIMON_MACOS_LIBCXX_INCLUDE_DIR}/cstdlib")
+            message(FATAL_ERROR "PAIMON_USE_APPLE_LIBCXX_WITH_CLANG is enabled, but libc++ headers were not found at ${PAIMON_MACOS_LIBCXX_INCLUDE_DIR}"
+            )
+        endif()
+        message(STATUS "Using Apple libc++ headers with Clang: ${PAIMON_MACOS_LIBCXX_INCLUDE_DIR}"
+        )
+        set(CMAKE_CXX_FLAGS
+            "${CMAKE_CXX_FLAGS} -nostdinc++ -isystem ${PAIMON_MACOS_LIBCXX_INCLUDE_DIR} -include cstdlib"
+        )
     endif()
 endif()