Skip to content

[feat](paimon) integrate paimon-cpp reader #60676

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
BiteTheDDDDt merged 2 commits into from
Feb 21, 2026
Merged

[feat](paimon) integrate paimon-cpp reader #60676

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
36686d2
feat(paimon): integrate paimon-cpp reader in BE with FE/thrift toggle
xylaaaaa Feb 11, 2026
1038ee8
revert thirdparty/patches/paimon-cpp-buildutils-static-deps.patch
morningman Feb 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 124 additions & 0 deletions be/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,17 @@ message(STATUS "build task executor simulator: ${BUILD_TASK_EXECUTOR_SIMULATOR}"
option(BUILD_FILE_CACHE_LRU_TOOL "ON for building file cache lru tool or OFF for not" OFF)
message(STATUS "build file cache lru tool: ${BUILD_FILE_CACHE_LRU_TOOL}")

option(ENABLE_PAIMON_CPP "Enable Paimon C++ integration" ON)
set(PAIMON_HOME "" CACHE PATH "Paimon install prefix")

# Allow env to override when reconfiguring (avoid picking /usr/local).
if (DEFINED ENV{ENABLE_PAIMON_CPP})
set(ENABLE_PAIMON_CPP "$ENV{ENABLE_PAIMON_CPP}" CACHE BOOL "" FORCE)
endif()
if (DEFINED ENV{PAIMON_HOME} AND NOT PAIMON_HOME)
set(PAIMON_HOME "$ENV{PAIMON_HOME}" CACHE PATH "" FORCE)
endif()

set(CMAKE_SKIP_RPATH TRUE)
set(Boost_USE_STATIC_LIBS ON)
set(Boost_USE_STATIC_RUNTIME ON)
Expand Down Expand Up @@ -550,6 +561,10 @@ set(COMMON_THIRDPARTY
${COMMON_THIRDPARTY}
)

if (ENABLE_PAIMON_CPP)
message(STATUS "Paimon C++ enabled: legacy thirdparty static linkage mode")
endif()

if ((ARCH_AMD64 OR ARCH_AARCH64) AND OS_LINUX)
add_library(hadoop_hdfs STATIC IMPORTED)
set_target_properties(hadoop_hdfs PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/hadoop_hdfs_3_4/native/libhdfs.a)
Expand Down Expand Up @@ -577,6 +592,13 @@ endif()
if (absl_FOUND)
set(COMMON_THIRDPARTY
${COMMON_THIRDPARTY}
absl::cord
absl::cord_internal
absl::cordz_functions
absl::cordz_info
absl::cordz_update_scope
absl::cordz_update_tracker
absl::crc_cord_state
absl::flags
absl::random_random
absl::spinlock_wait
Expand All @@ -602,6 +624,80 @@ if (BUILD_BENCHMARK)
)
endif()

set(PAIMON_FACTORY_REGISTRY_LIBS)
set(PAIMON_ARROW_CORE_LIB)
set(PAIMON_ARROW_FILESYSTEM_LIB)
set(PAIMON_ARROW_DATASET_LIB)
set(PAIMON_ARROW_ACERO_LIB)
if (ENABLE_PAIMON_CPP)
set(_paimon_arrow_core_candidates
${THIRDPARTY_DIR}/paimon-cpp/lib64/paimon_deps/libarrow.a
${THIRDPARTY_DIR}/lib64/libarrow.a
${THIRDPARTY_DIR}/lib/libarrow.a
)
foreach(_paimon_arrow_core_candidate IN LISTS _paimon_arrow_core_candidates)
if (EXISTS "${_paimon_arrow_core_candidate}")
add_library(paimon_arrow_core STATIC IMPORTED)
set_target_properties(paimon_arrow_core PROPERTIES
IMPORTED_LOCATION ${_paimon_arrow_core_candidate})
set(PAIMON_ARROW_CORE_LIB paimon_arrow_core)
break()
endif()
endforeach()
set(_paimon_arrow_filesystem_candidates
${THIRDPARTY_DIR}/paimon-cpp/lib64/paimon_deps/libarrow_filesystem.a
${THIRDPARTY_DIR}/lib64/libarrow_filesystem.a
${THIRDPARTY_DIR}/lib/libarrow_filesystem.a
)
foreach(_paimon_arrow_filesystem_candidate IN LISTS _paimon_arrow_filesystem_candidates)
if (EXISTS "${_paimon_arrow_filesystem_candidate}")
add_library(paimon_arrow_filesystem STATIC IMPORTED)
set_target_properties(paimon_arrow_filesystem PROPERTIES
IMPORTED_LOCATION ${_paimon_arrow_filesystem_candidate})
set(PAIMON_ARROW_FILESYSTEM_LIB paimon_arrow_filesystem)
break()
endif()
endforeach()
set(_paimon_arrow_dataset_candidates
${THIRDPARTY_DIR}/paimon-cpp/lib64/paimon_deps/libarrow_dataset.a
${THIRDPARTY_DIR}/lib64/libarrow_dataset.a
${THIRDPARTY_DIR}/lib/libarrow_dataset.a
)
foreach(_paimon_arrow_dataset_candidate IN LISTS _paimon_arrow_dataset_candidates)
if (EXISTS "${_paimon_arrow_dataset_candidate}")
add_library(paimon_arrow_dataset STATIC IMPORTED)
set_target_properties(paimon_arrow_dataset PROPERTIES
IMPORTED_LOCATION ${_paimon_arrow_dataset_candidate})
set(PAIMON_ARROW_DATASET_LIB paimon_arrow_dataset)
break()
endif()
endforeach()
set(_paimon_arrow_acero_candidates
${THIRDPARTY_DIR}/paimon-cpp/lib64/paimon_deps/libarrow_acero.a
${THIRDPARTY_DIR}/lib64/libarrow_acero.a
${THIRDPARTY_DIR}/lib/libarrow_acero.a
)
foreach(_paimon_arrow_acero_candidate IN LISTS _paimon_arrow_acero_candidates)
if (EXISTS "${_paimon_arrow_acero_candidate}")
add_library(paimon_arrow_acero STATIC IMPORTED)
set_target_properties(paimon_arrow_acero PROPERTIES
IMPORTED_LOCATION ${_paimon_arrow_acero_candidate})
set(PAIMON_ARROW_ACERO_LIB paimon_arrow_acero)
break()
endif()
endforeach()
if (PAIMON_ARROW_DATASET_LIB)
# paimon_parquet_file_format depends on Arrow Dataset symbols.
# Force-link it only when arrow_dataset is available.
set(PAIMON_FACTORY_REGISTRY_LIBS
paimon_parquet_file_format
)
list(REMOVE_ITEM COMMON_THIRDPARTY ${PAIMON_FACTORY_REGISTRY_LIBS})
else()
message(STATUS "Paimon C++: libarrow_dataset.a not found, keep paimon_parquet_file_format as regular static lib")
endif()
endif()

set(DORIS_DEPENDENCIES
${DORIS_DEPENDENCIES}
${WL_START_GROUP}
Expand All @@ -619,6 +715,34 @@ set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} clucene-core-static)
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} clucene-shared-static)
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} clucene-contribs-lib)

if (ENABLE_PAIMON_CPP)
if (PAIMON_FACTORY_REGISTRY_LIBS)
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES}
-Wl,--whole-archive
${PAIMON_FACTORY_REGISTRY_LIBS}
-Wl,--no-whole-archive)
endif()
if (PAIMON_ARROW_CORE_LIB)
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} ${PAIMON_ARROW_CORE_LIB})
endif()
if (PAIMON_ARROW_FILESYSTEM_LIB)
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} ${PAIMON_ARROW_FILESYSTEM_LIB})
endif()
if (PAIMON_ARROW_DATASET_LIB)
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} ${PAIMON_ARROW_DATASET_LIB})
endif()
if (PAIMON_ARROW_ACERO_LIB)
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} ${PAIMON_ARROW_ACERO_LIB})
endif()

# paimon-cpp internal dependencies (renamed with _paimon suffix)
# These must come after paimon libraries to resolve symbols.
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} roaring_bitmap_paimon)
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} xxhash_paimon)
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} fmt_paimon)
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} tbb_paimon)
endif()

set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} ${WL_END_GROUP})

# Add all external dependencies. They should come after the palo libs.
Expand Down
15 changes: 15 additions & 0 deletions be/cmake/thirdparty.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,3 +179,18 @@ add_thirdparty(icudata LIB64)


add_thirdparty(pugixml LIB64)

if (ENABLE_PAIMON_CPP)
add_thirdparty(paimon LIB64)
add_thirdparty(paimon_parquet_file_format LIB64)
add_thirdparty(paimon_orc_file_format LIB64)
add_thirdparty(paimon_blob_file_format LIB64)
add_thirdparty(paimon_local_file_system LIB64)
add_thirdparty(paimon_file_index LIB64)
add_thirdparty(paimon_global_index LIB64)

add_thirdparty(roaring_bitmap_paimon LIB64)
add_thirdparty(xxhash_paimon LIB64)
add_thirdparty(fmt_paimon LIB64)
add_thirdparty(tbb_paimon LIB64)
endif()
Loading
Loading