Skip to content

feat: enable arrow to build parquet #89

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 29, 2025
Merged

feat: enable arrow to build parquet #89

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 32 additions & 6 deletions cmake_modules/IcebergThirdpartyToolchain.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,13 +65,15 @@ function(resolve_arrow_dependency)
set(ARROW_BUILD_STATIC
ON
CACHE BOOL "" FORCE)
# To workaround https://github.com/apache/arrow/pull/45513
set(ARROW_IPC
ON
OFF
CACHE BOOL "" FORCE)
set(ARROW_FILESYSTEM
ON
CACHE BOOL "" FORCE)
set(ARROW_PARQUET
ON
CACHE BOOL "" FORCE)
set(ARROW_SIMD_LEVEL
"NONE"
CACHE STRING "" FORCE)
Expand All @@ -82,13 +84,15 @@ function(resolve_arrow_dependency)
ON
CACHE BOOL "" FORCE)
set(ARROW_DEPENDENCY_SOURCE
"AUTO"
"BUNDLED"
CACHE STRING "" FORCE)

fetchcontent_declare(Arrow
${FC_DECLARE_COMMON_OPTIONS}
URL ${ARROW_SOURCE_URL}
URL_HASH "SHA256=${ICEBERG_ARROW_BUILD_SHA256_CHECKSUM}"
GIT_REPOSITORY https://github.com/wgtmac/arrow.git
Copy link
Member Author

@wgtmac wgtmac Apr 26, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is still on my fork: apache/arrow#46232. Will update it once merged and released.

GIT_TAG 7d50c4ac803ad983734de5f418b7cd18f25b0dc9
#URL ${ARROW_SOURCE_URL}
#URL_HASH "SHA256=${ICEBERG_ARROW_BUILD_SHA256_CHECKSUM}"
SOURCE_SUBDIR
cpp
FIND_PACKAGE_ARGS
Expand All @@ -107,13 +111,35 @@ function(resolve_arrow_dependency)
${arrow_SOURCE_DIR}/cpp/src)
endif()

if(NOT TARGET Parquet::parquet_static)
add_library(Parquet::parquet_static INTERFACE IMPORTED)
target_link_libraries(Parquet::parquet_static INTERFACE parquet_static)
target_include_directories(Parquet::parquet_static
INTERFACE ${arrow_BINARY_DIR}/src
${arrow_SOURCE_DIR}/cpp/src)
endif()

set(ARROW_VENDORED TRUE)
set_target_properties(arrow_static PROPERTIES OUTPUT_NAME "iceberg_vendored_arrow")
install(TARGETS arrow_static
set_target_properties(parquet_static PROPERTIES OUTPUT_NAME
"iceberg_vendored_parquet")
install(TARGETS arrow_static parquet_static
EXPORT iceberg_targets
RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}"
ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}"
LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}")

if(TARGET arrow_bundled_dependencies)
message(STATUS "arrow_bundled_dependencies found")
# arrow_bundled_dependencies is only INSTALL_INTERFACE and will not be built by default.
# We need to add it as a dependency to arrow_static so that it will be built.
add_dependencies(arrow_static arrow_bundled_dependencies)
# We cannot install an IMPORTED target, so we need to install the library manually.
get_target_property(arrow_bundled_dependencies_location arrow_bundled_dependencies
IMPORTED_LOCATION)
install(FILES ${arrow_bundled_dependencies_location}
DESTINATION ${ICEBERG_INSTALL_LIBDIR})
endif()
else()
set(ARROW_VENDORED FALSE)
list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Arrow)
Expand Down
4 changes: 4 additions & 0 deletions src/iceberg/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,25 +89,29 @@ if(ICEBERG_BUILD_BUNDLE)
ICEBERG_BUNDLE_STATIC_BUILD_INTERFACE_LIBS
"$<IF:$<TARGET_EXISTS:iceberg_static>,iceberg_static,iceberg_shared>"
"$<IF:$<TARGET_EXISTS:Arrow::arrow_static>,Arrow::arrow_static,Arrow::arrow_shared>"
"$<IF:$<TARGET_EXISTS:Parquet::parquet_static>,Parquet::parquet_static,Parquet::parquet_shared>"
"$<IF:$<TARGET_EXISTS:Avro::avrocpp_static>,Avro::avrocpp_static,Avro::avrocpp_shared>"
)
list(APPEND
ICEBERG_BUNDLE_SHARED_BUILD_INTERFACE_LIBS
"$<IF:$<TARGET_EXISTS:iceberg_shared>,iceberg_shared,iceberg_static>"
"$<IF:$<TARGET_EXISTS:Arrow::arrow_shared>,Arrow::arrow_shared,Arrow::arrow_static>"
"$<IF:$<TARGET_EXISTS:Parquet::parquet_shared>,Parquet::parquet_shared,Parquet::parquet_static>"
"$<IF:$<TARGET_EXISTS:Avro::avrocpp_shared>,Avro::avrocpp_shared,Avro::avrocpp_static>"
)

list(APPEND
ICEBERG_BUNDLE_STATIC_INSTALL_INTERFACE_LIBS
"$<IF:$<TARGET_EXISTS:Iceberg::iceberg_static>,Iceberg::iceberg_static,Iceberg::iceberg_shared>"
"$<IF:$<BOOL:${ARROW_VENDORED}>,Iceberg::arrow_static,$<IF:$<TARGET_EXISTS:Arrow::arrow_static>,Arrow::arrow_static,Arrow::arrow_shared>>"
"$<IF:$<BOOL:${ARROW_VENDORED}>,Iceberg::parquet_static,$<IF:$<TARGET_EXISTS:Parquet::parquet_static>,Parquet::parquet_static,Parquet::parquet_shared>>"
"$<IF:$<BOOL:${AVRO_VENDORED}>,Iceberg::avrocpp_s,$<IF:$<TARGET_EXISTS:Avro::avrocpp_static>,Avro::avrocpp_static,Avro::avrocpp_shared>>"
)
list(APPEND
ICEBERG_BUNDLE_SHARED_INSTALL_INTERFACE_LIBS
"$<IF:$<TARGET_EXISTS:Iceberg::iceberg_shared>,Iceberg::iceberg_shared,Iceberg::iceberg_static>"
"$<IF:$<BOOL:${ARROW_VENDORED}>,Iceberg::arrow_static,$<IF:$<TARGET_EXISTS:Arrow::arrow_shared>,Arrow::arrow_shared,Arrow::arrow_static>>"
"$<IF:$<BOOL:${ARROW_VENDORED}>,Iceberg::parquet_static,$<IF:$<TARGET_EXISTS:Parquet::parquet_shared>,Parquet::parquet_shared,Parquet::parquet_static>>"
"$<IF:$<BOOL:${AVRO_VENDORED}>,Iceberg::avrocpp_s,$<IF:$<TARGET_EXISTS:Avro::avrocpp_shared>,Avro::avrocpp_shared,Avro::avrocpp_static>>"
)

Expand Down
25 changes: 25 additions & 0 deletions src/iceberg/IcebergConfig.cmake.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,31 @@ iceberg_find_dependencies("${ICEBERG_SYSTEM_DEPENDENCIES}")

include("${CMAKE_CURRENT_LIST_DIR}/IcebergTargets.cmake")

if(TARGET Iceberg::arrow_static)
add_library(Arrow::arrow_static ALIAS Iceberg::arrow_static)

add_library(Arrow::arrow_bundled_dependencies STATIC IMPORTED)
get_target_property(arrow_static_configurations Iceberg::arrow_static
IMPORTED_CONFIGURATIONS)
foreach(CONFIGURATION ${arrow_static_configurations})
string(TOUPPER "${CONFIGURATION}" CONFIGURATION)
get_target_property(arrow_static_location Iceberg::arrow_static
LOCATION_${CONFIGURATION})
get_filename_component(arrow_lib_dir "${arrow_static_location}" DIRECTORY)
set_property(TARGET Arrow::arrow_bundled_dependencies
APPEND
PROPERTY IMPORTED_CONFIGURATIONS ${CONFIGURATION})
set_target_properties(Arrow::arrow_bundled_dependencies
PROPERTIES IMPORTED_LOCATION_${CONFIGURATION}
"${arrow_lib_dir}/${CMAKE_STATIC_LIBRARY_PREFIX}arrow_bundled_dependencies${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
endforeach()
endif()

if(TARGET Iceberg::parquet_static)
add_library(Parquet::parquet_static ALIAS Iceberg::parquet_static)
endif()

# Find required components
iceberg_find_components("${Iceberg_FIND_COMPONENTS}")

Expand Down
4 changes: 3 additions & 1 deletion src/iceberg/arrow/demo_arrow.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,15 @@
#include "iceberg/arrow/demo_arrow.h"

#include <arrow/config.h>
#include <parquet/parquet_version.h>

#include "iceberg/demo.h"

namespace iceberg::arrow {

std::string DemoArrow::print() const {
return Demo().print() + ", Arrow version: " + ::arrow::GetBuildInfo().version_string;
return Demo().print() + ", Arrow version: " + ::arrow::GetBuildInfo().version_string +
", Parquet version: " + CREATED_BY_VERSION;
}

} // namespace iceberg::arrow
Loading