Merge branch 'main' into fix-bytes-move-assign

Author: lxy-9602

.gitattributes

@@ -3,4 +3,6 @@
 *.jar filter=lfs diff=lfs merge=lfs -text
 *.tar filter=lfs diff=lfs merge=lfs -text
 *.gz filter=lfs diff=lfs merge=lfs -text
-79d01717-8380-4504-86e1-387e6c058d0a filter=lfs diff=lfs merge=lfs -text
+test/test_data/sst/none/79d01717-8380-4504-86e1-387e6c058d0a filter=lfs diff=lfs merge=lfs -text
+test/test_data/sst/lz4/10540951-41d3-4216-aa2c-b15dfd25eb75 filter=lfs diff=lfs merge=lfs -text
+test/test_data/sst/zstd/83d05c53-2353-4160-b756-d50dd851b474 filter=lfs diff=lfs merge=lfs -text

CMakeLists.txt

@@ -53,6 +53,7 @@ option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON)
 option(PAIMON_ENABLE_LANCE "Whether to enable lance file format" OFF)
 option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF)
 option(PAIMON_ENABLE_LUMINA "Whether to enable lumina vector index" ON)
+option(PAIMON_ENABLE_LUCENE "Whether to enable lucene index" ON)
 
 if(PAIMON_ENABLE_ORC)
     add_definitions(-DPAIMON_ENABLE_ORC)
@@ -82,6 +83,10 @@ if(PAIMON_ENABLE_LUMINA)
     add_definitions(-DPAIMON_ENABLE_LUMINA)
 endif()
 
+if(PAIMON_ENABLE_LUCENE)
+    add_definitions(-DPAIMON_ENABLE_LUCENE)
+endif()
+
 add_definitions(-DSNAPPY_CODEC_AVAILABLE)
 add_definitions(-DZSTD_CODEC_AVAILABLE)
 add_definitions(-DRAPIDJSON_HAS_STDSTRING)
@@ -379,6 +384,11 @@ if(PAIMON_BUILD_TESTS)
         list(APPEND TEST_STATIC_LINK_LIBS paimon_lumina_index_shared)
         list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed")
     endif()
+    if(PAIMON_ENABLE_LUCENE)
+        list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed")
+        list(APPEND TEST_STATIC_LINK_LIBS paimon_lucene_index_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed")
+    endif()
 
 endif()
 
@@ -407,6 +417,7 @@ add_subdirectory(src/paimon/format/parquet)
 add_subdirectory(src/paimon/format/avro)
 add_subdirectory(src/paimon/format/lance)
 add_subdirectory(src/paimon/global_index/lumina)
+add_subdirectory(src/paimon/global_index/lucene)
 add_subdirectory(src/paimon/testing/mock)
 add_subdirectory(src/paimon/testing/utils)
 add_subdirectory(test/inte)

LICENSE

@@ -263,6 +263,9 @@ This product includes code from Apache Iceberg C++.
   * .devcontainer/devcontainer.json.template
 * CI utilities:
   * .pre-commit-config.yaml
+* Avro direct decoder/encoder:
+  * src/paimon/format/avro/avro_direct_decoder.cpp
+  * src/paimon/format/avro/avro_direct_decoder.h
 
 Copyright: 2024-2025 The Apache Software Foundation.
 Home page: https://iceberg.apache.org/

cmake_modules/BuildUtils.cmake

@@ -55,7 +55,18 @@ function(add_paimon_lib LIB_NAME)
     # Necessary to make static linking into other shared libraries work properly
     set_property(TARGET ${LIB_NAME}_objlib PROPERTY POSITION_INDEPENDENT_CODE 1)
     if(ARG_DEPENDENCIES)
-        add_dependencies(${LIB_NAME}_objlib ${ARG_DEPENDENCIES})
+        # Avoid add_dependencies on non-existent targets (e.g. when building static only).
+        set(_paimon_objlib_deps)
+        foreach(_paimon_dep IN LISTS ARG_DEPENDENCIES)
+            if(TARGET ${_paimon_dep})
+                list(APPEND _paimon_objlib_deps ${_paimon_dep})
+            endif()
+        endforeach()
+        if(_paimon_objlib_deps)
+            add_dependencies(${LIB_NAME}_objlib ${_paimon_objlib_deps})
+        endif()
+        unset(_paimon_objlib_deps)
+        unset(_paimon_dep)
     endif()
     set(LIB_DEPS $<TARGET_OBJECTS:${LIB_NAME}_objlib>)
     set(LIB_INCLUDES)

cmake_modules/SetupCxxFlags.cmake

@@ -80,10 +80,13 @@ if("${BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN")
         set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-unused-parameter")
         set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-unknown-warning-option")
         set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-constant-logical-operand")
+        set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-deprecated-declarations")
+        set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-deprecated-builtins")
     elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
         set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall")
         set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-conversion")
         set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-deprecated-declarations")
+        set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-deprecated-builtins")
         set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-sign-conversion")
         set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-unused-variable")
     else()
@@ -182,6 +185,8 @@ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STRE
 
     # Don't complain about optimization passes that were not possible
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-pass-failed")
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-deprecated-declarations")
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-deprecated-builtins")
 
     if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
         # Depending on the default OSX_DEPLOYMENT_TARGET (< 10.9), libstdc++ may be

cmake_modules/ThirdpartyToolchain.cmake

@@ -112,6 +112,18 @@ else()
     endif()
 endif()
 
+if(DEFINED ENV{PAIMON_LUCENE_URL})
+    set(LUCENE_SOURCE_URL "$ENV{PAIMON_LUCENE_URL}")
+else()
+    if(EXISTS "${THIRDPARTY_DIR}/${PAIMON_LUCENE_PKG_NAME}")
+        set_urls(LUCENE_SOURCE_URL "${THIRDPARTY_DIR}/${PAIMON_LUCENE_PKG_NAME}")
+    else()
+        set_urls(LUCENE_SOURCE_URL
+                 "${THIRDPARTY_MIRROR_URL}https://github.com/luceneplusplus/LucenePlusPlus/archive/refs/tags/${PAIMON_LUCENE_PKG_NAME}"
+        )
+    endif()
+endif()
+
 if(DEFINED ENV{PAIMON_GLOG_URL})
     set(GLOG_SOURCE_URL "$ENV{PAIMON_GLOG_URL}")
 else()
@@ -275,6 +287,62 @@ set(EP_COMMON_CMAKE_ARGS
     -DCMAKE_C_FLAGS=${EP_C_FLAGS}
     -DCMAKE_INSTALL_LIBDIR=lib)
 
+macro(build_lucene)
+    message(STATUS "Building lucene from source")
+    set(LUCENE_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lucene_ep-install")
+    set(LUCENE_CMAKE_ARGS
+        ${EP_COMMON_CMAKE_ARGS}
+        "-DENABLE_TEST=OFF"
+        "-DCMAKE_C_FLAGS=-pthread"
+        "-DCMAKE_CXX_FLAGS=-pthread"
+        "-DCMAKE_EXE_LINKER_FLAGS=-pthread"
+        "-DBoost_INCLUDE_DIR=${BOOST_INCLUDE_DIR}"
+        "-DBoost_LIBRARY_DIR=${BOOST_LIBRARY_DIR}"
+        "-DBOOST_ROOT=${BOOST_INSTALL}"
+        "-DBoost_CHRONO_FOUND=TRUE"
+        "-DBoost_THREAD_FOUND=TRUE"
+        "-DCMAKE_INSTALL_PREFIX=${LUCENE_PREFIX}")
+
+    set(LUCENE_LIB "${LUCENE_PREFIX}/lib/liblucene++.so.0")
+    externalproject_add(lucene_ep
+                        ${EP_COMMON_OPTIONS}
+                        URL ${LUCENE_SOURCE_URL}
+                        URL_HASH "SHA256=${PAIMON_LUCENE_BUILD_SHA256_CHECKSUM}"
+                        CMAKE_ARGS ${LUCENE_CMAKE_ARGS}
+                        BUILD_BYPRODUCTS ${LUCENE_LIB}
+                        DEPENDS boost_date_time
+                                boost_filesystem
+                                boost_regex
+                                boost_thread
+                                boost_iostreams
+                                boost_system
+                                boost_chrono
+                                boost_atomic)
+
+    set(LUCENE_INCLUDE_DIR "${LUCENE_PREFIX}/include")
+    # The include directory must exist before it is referenced by a target.
+    file(MAKE_DIRECTORY "${LUCENE_INCLUDE_DIR}")
+    include_directories(SYSTEM ${LUCENE_INCLUDE_DIR} ${BOOST_INCLUDE_DIR}
+                        ${BOOST_EXTRA_INCLUDE_DIR})
+    add_library(lucene INTERFACE IMPORTED)
+    target_include_directories(lucene SYSTEM INTERFACE "${LUCENE_INCLUDE_DIR}")
+    target_compile_options(lucene INTERFACE -pthread)
+
+    target_link_libraries(lucene
+                          INTERFACE "${LUCENE_LIB}"
+                                    boost_date_time
+                                    boost_filesystem
+                                    boost_regex
+                                    boost_thread
+                                    boost_iostreams
+                                    boost_system
+                                    boost_chrono
+                                    boost_atomic
+                                    pthread
+                                    dl)
+    add_dependencies(lucene lucene_ep)
+endmacro()
+
 macro(build_rapidjson)
     message(STATUS "Building RapidJSON from source")
     set(RAPIDJSON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/rapidjson_ep-install")
@@ -342,6 +410,99 @@ macro(build_fmt)
     add_dependencies(fmt fmt_ep)
 endmacro(build_fmt)
 
+macro(build_boost)
+    message(STATUS "Building boost from source")
+    set(BOOST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/boost_ep-prefix")
+    set(BOOST_INSTALL "${CMAKE_CURRENT_BINARY_DIR}/boost_ep-install")
+    set(BOOST_INCLUDE_DIR "${BOOST_INSTALL}/include")
+    set(BOOST_LIBRARY_DIR ${BOOST_INSTALL}/lib)
+    file(MAKE_DIRECTORY ${BOOST_INCLUDE_DIR})
+    file(MAKE_DIRECTORY ${BOOST_LIBRARY_DIR})
+
+    set(BOOST_BYPRODUCTS
+        ${BOOST_LIBRARY_DIR}/libboost_date_time.a
+        ${BOOST_LIBRARY_DIR}/libboost_filesystem.a
+        ${BOOST_LIBRARY_DIR}/libboost_system.a
+        ${BOOST_LIBRARY_DIR}/libboost_regex.a
+        ${BOOST_LIBRARY_DIR}/libboost_thread.a
+        ${BOOST_LIBRARY_DIR}/libboost_atomic.a
+        ${BOOST_LIBRARY_DIR}/libboost_chrono.a
+        ${BOOST_LIBRARY_DIR}/libboost_iostreams.a)
+
+    externalproject_add(boost_ep
+                        GIT_REPOSITORY https://github.com/boostorg/boost.git
+                        GIT_TAG boost-${PAIMON_BOOST_BUILD_VERSION}
+                        GIT_SHALLOW FALSE
+                        GIT_PROGRESS TRUE
+                        GIT_SUBMODULES_RECURSE TRUE
+                        CONFIGURE_COMMAND ${BOOST_PREFIX}/src/boost_ep/bootstrap.sh
+                                          --with-libraries=date_time,filesystem,iostreams,regex,system,thread,chrono,atomic
+                        BUILD_IN_SOURCE TRUE
+                        BUILD_COMMAND ${BOOST_PREFIX}/src/boost_ep/b2
+                                      --prefix=${BOOST_INSTALL}
+                                      --libdir=${BOOST_LIBRARY_DIR} link=static
+                                      runtime-link=shared threading=multi variant=release
+                                      cxxflags=-fPIC install
+                        INSTALL_COMMAND bash -c
+                                        "mkdir -p ${BOOST_INSTALL}/include/boost && cp -r ${BOOST_PREFIX}/src/boost_ep/libs/*/include/boost/* ${BOOST_INSTALL}/include/boost && cp -r ${BOOST_PREFIX}/src/boost_ep/libs/*/*/include/boost/* ${BOOST_INSTALL}/include/boost"
+                        BUILD_BYPRODUCTS ${BOOST_BYPRODUCTS}
+                        LOG_DOWNLOAD ON
+                        LOG_CONFIGURE ON
+                        LOG_BUILD ON)
+
+    include_directories(SYSTEM ${BOOST_INCLUDE_DIR})
+
+    add_library(boost_atomic STATIC IMPORTED)
+    set_target_properties(boost_atomic
+                          PROPERTIES IMPORTED_LOCATION
+                                     ${BOOST_LIBRARY_DIR}/libboost_atomic.a
+                                     INTERFACE_INCLUDE_DIRECTORIES ${BOOST_INCLUDE_DIR})
+    add_library(boost_chrono STATIC IMPORTED)
+    set_target_properties(boost_chrono
+                          PROPERTIES IMPORTED_LOCATION
+                                     ${BOOST_LIBRARY_DIR}/libboost_chrono.a
+                                     INTERFACE_INCLUDE_DIRECTORIES ${BOOST_INCLUDE_DIR})
+    add_library(boost_date_time STATIC IMPORTED)
+    set_target_properties(boost_date_time
+                          PROPERTIES IMPORTED_LOCATION
+                                     ${BOOST_LIBRARY_DIR}/libboost_date_time.a
+                                     INTERFACE_INCLUDE_DIRECTORIES ${BOOST_INCLUDE_DIR})
+    add_library(boost_filesystem STATIC IMPORTED)
+    set_target_properties(boost_filesystem
+                          PROPERTIES IMPORTED_LOCATION
+                                     ${BOOST_LIBRARY_DIR}/libboost_filesystem.a
+                                     INTERFACE_INCLUDE_DIRECTORIES ${BOOST_INCLUDE_DIR})
+    add_library(boost_regex STATIC IMPORTED)
+    set_target_properties(boost_regex
+                          PROPERTIES IMPORTED_LOCATION
+                                     ${BOOST_LIBRARY_DIR}/libboost_regex.a
+                                     INTERFACE_INCLUDE_DIRECTORIES ${BOOST_INCLUDE_DIR})
+    add_library(boost_thread STATIC IMPORTED)
+    set_target_properties(boost_thread
+                          PROPERTIES IMPORTED_LOCATION
+                                     ${BOOST_LIBRARY_DIR}/libboost_thread.a
+                                     INTERFACE_INCLUDE_DIRECTORIES ${BOOST_INCLUDE_DIR})
+    add_library(boost_iostreams STATIC IMPORTED)
+    set_target_properties(boost_iostreams
+                          PROPERTIES IMPORTED_LOCATION
+                                     ${BOOST_LIBRARY_DIR}/libboost_iostreams.a
+                                     INTERFACE_INCLUDE_DIRECTORIES ${BOOST_INCLUDE_DIR})
+    add_library(boost_system STATIC IMPORTED)
+    set_target_properties(boost_system
+                          PROPERTIES IMPORTED_LOCATION
+                                     ${BOOST_LIBRARY_DIR}/libboost_system.a
+                                     INTERFACE_INCLUDE_DIRECTORIES ${BOOST_INCLUDE_DIR})
+
+    add_dependencies(boost_atomic boost_ep)
+    add_dependencies(boost_chrono boost_ep)
+    add_dependencies(boost_date_time boost_ep)
+    add_dependencies(boost_filesystem boost_ep)
+    add_dependencies(boost_regex boost_ep)
+    add_dependencies(boost_thread boost_ep)
+    add_dependencies(boost_iostreams boost_ep)
+    add_dependencies(boost_system boost_ep)
+endmacro(build_boost)
+
 macro(build_snappy)
     message(STATUS "Building snappy from source")
     set(SNAPPY_HOME "${CMAKE_CURRENT_BINARY_DIR}/snappy_ep-install")
@@ -1108,3 +1269,7 @@ if(PAIMON_ENABLE_JINDO)
     build_jindosdk_c()
     build_jindosdk_nextarch()
 endif()
+if(PAIMON_ENABLE_LUCENE)
+    build_boost()
+    build_lucene()
+endif()

include/paimon/global_config.h

@@ -29,7 +29,7 @@ namespace paimon {
 /// not necessarily the exact number of threads at a given point in time.
 ///
 /// You can change this number using SetArrowCpuThreadPoolCapacity().
-PAIMON_EXPORT int GetArrowCpuThreadPoolCapacity();
+PAIMON_EXPORT int32_t GetArrowCpuThreadPoolCapacity();
 
 /// Set the capacity of the arrow's global thread pool
 /// This is a simple wrapper of arrow::SetCpuThreadPoolCapacity()
@@ -40,6 +40,6 @@ PAIMON_EXPORT int GetArrowCpuThreadPoolCapacity();
 /// The current number is returned by GetArrowCpuThreadPoolCapacity().
 /// Currently, this capacity will significantly affect the performance
 /// of parquet file batch read.
-PAIMON_EXPORT Status SetArrowCpuThreadPoolCapacity(int threads);
+PAIMON_EXPORT Status SetArrowCpuThreadPoolCapacity(int32_t threads);
 
 }  // namespace paimon

include/paimon/global_index/global_index_reader.h

@@ -38,12 +38,16 @@ namespace paimon {
 class PAIMON_EXPORT GlobalIndexReader : public FunctionVisitor<std::shared_ptr<GlobalIndexResult>> {
  public:
     /// VisitVectorSearch performs approximate vector similarity search.
-    /// @note `VisitVectorSearch` is thread-safe (not coroutine-safe) while other `VisitXXX` is not
-    /// thread-safe.
     /// @warning `VisitVectorSearch` may return error status when it is incorrectly invoked (e.g.,
     /// BitmapGlobalIndexReader call `VisitVectorSearch`).
     virtual Result<std::shared_ptr<VectorSearchGlobalIndexResult>> VisitVectorSearch(
         const std::shared_ptr<VectorSearch>& vector_search) = 0;
+
+    /// @return true if the reader is thread-safe; false otherwise.
+    virtual bool IsThreadSafe() const = 0;
+
+    /// @return An identifier representing the index type. (e.g., "bitmap", "lumina").
+    virtual std::string GetIndexType() const = 0;
 };
 
 }  // namespace paimon