feat: enable arrow to build parquet

Author: wgtmac

cmake_modules/IcebergThirdpartyToolchain.cmake

@@ -65,13 +65,15 @@ function(resolve_arrow_dependency)
   set(ARROW_BUILD_STATIC
       ON
       CACHE BOOL "" FORCE)
-  # To workaround https://github.com/apache/arrow/pull/45513
   set(ARROW_IPC
-      ON
+      OFF
       CACHE BOOL "" FORCE)
   set(ARROW_FILESYSTEM
       ON
       CACHE BOOL "" FORCE)
+  set(ARROW_PARQUET
+      ON
+      CACHE BOOL "" FORCE)
   set(ARROW_SIMD_LEVEL
       "NONE"
       CACHE STRING "" FORCE)
@@ -82,13 +84,15 @@ function(resolve_arrow_dependency)
       ON
       CACHE BOOL "" FORCE)
   set(ARROW_DEPENDENCY_SOURCE
-      "AUTO"
+      "BUNDLED"
       CACHE STRING "" FORCE)
 
   fetchcontent_declare(Arrow
                        ${FC_DECLARE_COMMON_OPTIONS}
-                       URL ${ARROW_SOURCE_URL}
-                       URL_HASH "SHA256=${ICEBERG_ARROW_BUILD_SHA256_CHECKSUM}"
+                       GIT_REPOSITORY https://github.com/wgtmac/arrow.git
+                       GIT_TAG 7d50c4ac803ad983734de5f418b7cd18f25b0dc9
+                       #URL ${ARROW_SOURCE_URL}
+                       #URL_HASH "SHA256=${ICEBERG_ARROW_BUILD_SHA256_CHECKSUM}"
                        SOURCE_SUBDIR
                        cpp
                        FIND_PACKAGE_ARGS
@@ -107,13 +111,35 @@ function(resolve_arrow_dependency)
                                            ${arrow_SOURCE_DIR}/cpp/src)
     endif()
 
+    if(NOT TARGET Parquet::parquet_static)
+      add_library(Parquet::parquet_static INTERFACE IMPORTED)
+      target_link_libraries(Parquet::parquet_static INTERFACE parquet_static)
+      target_include_directories(Parquet::parquet_static
+                                 INTERFACE ${arrow_BINARY_DIR}/src
+                                           ${arrow_SOURCE_DIR}/cpp/src)
+    endif()
+
     set(ARROW_VENDORED TRUE)
     set_target_properties(arrow_static PROPERTIES OUTPUT_NAME "iceberg_vendored_arrow")
-    install(TARGETS arrow_static
+    set_target_properties(parquet_static PROPERTIES OUTPUT_NAME
+                                                    "iceberg_vendored_parquet")
+    install(TARGETS arrow_static parquet_static
             EXPORT iceberg_targets
             RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}"
             ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}"
             LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}")
+
+    if(TARGET arrow_bundled_dependencies)
+      message(STATUS "arrow_bundled_dependencies found")
+      # arrow_bundled_dependencies is only INSTALL_INTERFACE and will not be built by default.
+      # We need to add it as a dependency to arrow_static so that it will be built.
+      add_dependencies(arrow_static arrow_bundled_dependencies)
+      # We cannot install an IMPORTED target, so we need to install the library manually.
+      get_target_property(arrow_bundled_dependencies_location arrow_bundled_dependencies
+                          IMPORTED_LOCATION)
+      install(FILES ${arrow_bundled_dependencies_location}
+              DESTINATION ${ICEBERG_INSTALL_LIBDIR})
+    endif()
   else()
     set(ARROW_VENDORED FALSE)
     list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Arrow)

src/iceberg/CMakeLists.txt

@@ -89,25 +89,29 @@ if(ICEBERG_BUILD_BUNDLE)
        ICEBERG_BUNDLE_STATIC_BUILD_INTERFACE_LIBS
        "$<IF:$<TARGET_EXISTS:iceberg_static>,iceberg_static,iceberg_shared>"
        "$<IF:$<TARGET_EXISTS:Arrow::arrow_static>,Arrow::arrow_static,Arrow::arrow_shared>"
+       "$<IF:$<TARGET_EXISTS:Parquet::parquet_static>,Parquet::parquet_static,Parquet::parquet_shared>"
        "$<IF:$<TARGET_EXISTS:Avro::avrocpp_static>,Avro::avrocpp_static,Avro::avrocpp_shared>"
   )
   list(APPEND
        ICEBERG_BUNDLE_SHARED_BUILD_INTERFACE_LIBS
        "$<IF:$<TARGET_EXISTS:iceberg_shared>,iceberg_shared,iceberg_static>"
        "$<IF:$<TARGET_EXISTS:Arrow::arrow_shared>,Arrow::arrow_shared,Arrow::arrow_static>"
+       "$<IF:$<TARGET_EXISTS:Parquet::parquet_shared>,Parquet::parquet_shared,Parquet::parquet_static>"
        "$<IF:$<TARGET_EXISTS:Avro::avrocpp_shared>,Avro::avrocpp_shared,Avro::avrocpp_static>"
   )
 
   list(APPEND
        ICEBERG_BUNDLE_STATIC_INSTALL_INTERFACE_LIBS
        "$<IF:$<TARGET_EXISTS:Iceberg::iceberg_static>,Iceberg::iceberg_static,Iceberg::iceberg_shared>"
        "$<IF:$<BOOL:${ARROW_VENDORED}>,Iceberg::arrow_static,$<IF:$<TARGET_EXISTS:Arrow::arrow_static>,Arrow::arrow_static,Arrow::arrow_shared>>"
+       "$<IF:$<BOOL:${ARROW_VENDORED}>,Iceberg::parquet_static,$<IF:$<TARGET_EXISTS:Parquet::parquet_static>,Parquet::parquet_static,Parquet::parquet_shared>>"
        "$<IF:$<BOOL:${AVRO_VENDORED}>,Iceberg::avrocpp_s,$<IF:$<TARGET_EXISTS:Avro::avrocpp_static>,Avro::avrocpp_static,Avro::avrocpp_shared>>"
   )
   list(APPEND
        ICEBERG_BUNDLE_SHARED_INSTALL_INTERFACE_LIBS
        "$<IF:$<TARGET_EXISTS:Iceberg::iceberg_shared>,Iceberg::iceberg_shared,Iceberg::iceberg_static>"
        "$<IF:$<BOOL:${ARROW_VENDORED}>,Iceberg::arrow_static,$<IF:$<TARGET_EXISTS:Arrow::arrow_shared>,Arrow::arrow_shared,Arrow::arrow_static>>"
+       "$<IF:$<BOOL:${ARROW_VENDORED}>,Iceberg::parquet_static,$<IF:$<TARGET_EXISTS:Parquet::parquet_shared>,Parquet::parquet_shared,Parquet::parquet_static>>"
        "$<IF:$<BOOL:${AVRO_VENDORED}>,Iceberg::avrocpp_s,$<IF:$<TARGET_EXISTS:Avro::avrocpp_shared>,Avro::avrocpp_shared,Avro::avrocpp_static>>"
   )
 

src/iceberg/IcebergConfig.cmake.in

@@ -72,6 +72,31 @@ iceberg_find_dependencies("${ICEBERG_SYSTEM_DEPENDENCIES}")
 
 include("${CMAKE_CURRENT_LIST_DIR}/IcebergTargets.cmake")
 
+if(TARGET Iceberg::arrow_static)
+  add_library(Arrow::arrow_static ALIAS Iceberg::arrow_static)
+
+  add_library(Arrow::arrow_bundled_dependencies STATIC IMPORTED)
+  get_target_property(arrow_static_configurations Iceberg::arrow_static
+                      IMPORTED_CONFIGURATIONS)
+  foreach(CONFIGURATION ${arrow_static_configurations})
+    string(TOUPPER "${CONFIGURATION}" CONFIGURATION)
+    get_target_property(arrow_static_location Iceberg::arrow_static
+                        LOCATION_${CONFIGURATION})
+    get_filename_component(arrow_lib_dir "${arrow_static_location}" DIRECTORY)
+    set_property(TARGET Arrow::arrow_bundled_dependencies
+                 APPEND
+                 PROPERTY IMPORTED_CONFIGURATIONS ${CONFIGURATION})
+    set_target_properties(Arrow::arrow_bundled_dependencies
+                          PROPERTIES IMPORTED_LOCATION_${CONFIGURATION}
+                                     "${arrow_lib_dir}/${CMAKE_STATIC_LIBRARY_PREFIX}arrow_bundled_dependencies${CMAKE_STATIC_LIBRARY_SUFFIX}"
+    )
+  endforeach()
+endif()
+
+if(TARGET Iceberg::parquet_static)
+  add_library(Parquet::parquet_static ALIAS Iceberg::parquet_static)
+endif()
+
 # Find required components
 iceberg_find_components("${Iceberg_FIND_COMPONENTS}")
 

src/iceberg/arrow/demo_arrow.cc

@@ -20,13 +20,15 @@
 #include "iceberg/arrow/demo_arrow.h"
 
 #include <arrow/config.h>
+#include <parquet/parquet_version.h>
 
 #include "iceberg/demo.h"
 
 namespace iceberg::arrow {
 
 std::string DemoArrow::print() const {
-  return Demo().print() + ", Arrow version: " + ::arrow::GetBuildInfo().version_string;
+  return Demo().print() + ", Arrow version: " + ::arrow::GetBuildInfo().version_string +
+         ", Parquet version: " + CREATED_BY_VERSION;
 }
 
 }  // namespace iceberg::arrow