Merge branch 'alibaba:main' into main

Author: lucasfang

.github/workflows/build_release.yaml

@@ -0,0 +1,64 @@
+# Copyright 2025-present Alibaba Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Build Release
+
+on:
+  push:
+    branches:
+      - '**'
+    tags:
+      - '**'
+  pull_request:
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  clang-release:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+    steps:
+      - name: Checkout paimon-cpp
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with:
+          lfs: true
+      - name: Build Paimon
+        shell: bash
+        env:
+          CC: clang
+          CXX: clang++
+        run: ci/scripts/build_paimon.sh $(pwd) false false Release
+  gcc-release:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+    steps:
+      - name: Checkout paimon-cpp
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with:
+          lfs: true
+      - name: Build Paimon
+        shell: bash
+        env:
+          CC: gcc-14
+          CXX: g++-14
+        run: ci/scripts/build_paimon.sh $(pwd) false false Release

CMakeLists.txt

@@ -319,7 +319,6 @@ if(PAIMON_BUILD_TESTS)
                       -L
                       unittest
                       --output-on-failure)
-    add_compile_options(-fno-access-control)
     add_dependencies(unittest paimon-tests)
 
     include_directories(SYSTEM ${GTEST_INCLUDE_DIR})

ci/scripts/build_paimon.sh

@@ -19,14 +19,15 @@ set -eux
 source_dir=${1}
 enable_sanitizer=${2:-false}
 check_clang_tidy=${3:-false}
+build_type=${4:-Debug}
 build_dir=${1}/build
 
 mkdir ${build_dir}
 pushd ${build_dir}
 
 CMAKE_ARGS=(
     "-G Ninja"
-    "-DCMAKE_BUILD_TYPE=Debug"
+    "-DCMAKE_BUILD_TYPE=${build_type}"
     "-DPAIMON_BUILD_TESTS=ON"
     "-DPAIMON_ENABLE_LANCE=ON"
     "-DPAIMON_ENABLE_JINDO=ON"

cmake_modules/BuildUtils.cmake

@@ -256,6 +256,7 @@ function(add_test_case REL_TEST_NAME)
     if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
         target_compile_options(${TEST_NAME} PRIVATE -Wno-global-constructors)
     endif()
+    target_compile_options(${TEST_NAME} PRIVATE -fno-access-control)
 
     add_test(${TEST_NAME}
              ${BUILD_SUPPORT_DIR}/run-test.sh

cmake_modules/ThirdpartyToolchain.cmake

@@ -548,6 +548,10 @@ macro(build_avro)
 
     get_target_property(AVRO_ZSTD_INCLUDE_DIR zstd INTERFACE_INCLUDE_DIRECTORIES)
     get_filename_component(AVRO_ZSTD_ROOT "${AVRO_ZSTD_INCLUDE_DIR}" DIRECTORY)
+
+    get_target_property(AVRO_ZLIB_INCLUDE_DIR zlib INTERFACE_INCLUDE_DIRECTORIES)
+    get_filename_component(AVRO_ZLIB_ROOT "${AVRO_ZLIB_INCLUDE_DIR}" DIRECTORY)
+
     set(AVRO_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -Wno-error")
     set(AVRO_CMAKE_C_FLAGS "${EP_C_FLAGS} -Wno-error")
 
@@ -558,7 +562,7 @@ macro(build_avro)
         "-DCMAKE_C_FLAGS=${AVRO_CMAKE_C_FLAGS}"
         "-DAVRO_BUILD_TESTS=OFF"
         "-DAVRO_BUILD_EXECUTABLES=OFF"
-        "-DZLIB_ROOT=${THIRDPARTY_ZLIB_ROOT}"
+        "-DZLIB_ROOT=${AVRO_ZLIB_ROOT}"
         "-Dzstd_ROOT=${AVRO_ZSTD_ROOT}"
         "-DSnappy_ROOT=${AVRO_SNAPPY_ROOT}")
     externalproject_add(avro_ep

include/paimon/global_index/bitmap_global_index_result.h

@@ -26,7 +26,7 @@
 #include "paimon/visibility.h"
 
 namespace paimon {
-/// Represents a global index query result that **lazily materializes** its matching row IDs as a
+/// Represents a global index query result that **lazily materializes** its matching row ids as a
 /// Roaring bitmap. The underlying 64-bit Roaring bitmap is **not constructed during object
 /// creation**; instead, it is built on-demand the first time GetBitmap() is called. This design
 /// avoids unnecessary computation and memory allocation when the bitmap is not needed (e.g., during
@@ -67,6 +67,8 @@ class PAIMON_EXPORT BitmapGlobalIndexResult : public GlobalIndexResult {
 
     Result<bool> IsEmpty() const override;
 
+    Result<std::shared_ptr<GlobalIndexResult>> AddOffset(int64_t offset) override;
+
     std::string ToString() const override;
 
     /// @return A non-owning, const pointer to the bitmap. The returned pointer is valid as long as

include/paimon/global_index/bitmap_topk_global_index_result.h

@@ -26,12 +26,12 @@
 #include "paimon/visibility.h"
 
 namespace paimon {
-/// Represents a Top-K global index result that combines a Roaring bitmap of candidate row IDs
+/// Represents a Top-K global index result that combines a Roaring bitmap of candidate row ids
 /// with an array of associated relevance scores.
 ///
 /// **Important Ordering Note**: Despite inheriting from TopKGlobalIndexResult, the results are
 /// **NOT sorted by score**. Instead, both the bitmap and the score vector are ordered by
-/// **ascending row ID**. This design enables efficient merging and set operations while preserving
+/// **ascending row id**. This design enables efficient merging and set operations while preserving
 /// row id-to-score mapping.
 class PAIMON_EXPORT BitmapTopKGlobalIndexResult : public TopKGlobalIndexResult {
  public:
@@ -74,16 +74,18 @@ class PAIMON_EXPORT BitmapTopKGlobalIndexResult : public TopKGlobalIndexResult {
     Result<std::shared_ptr<GlobalIndexResult>> Or(
         const std::shared_ptr<GlobalIndexResult>& other) override;
 
+    Result<std::shared_ptr<GlobalIndexResult>> AddOffset(int64_t offset) override;
+
     Result<bool> IsEmpty() const override;
 
     std::string ToString() const override;
 
-    /// @return A non-owning, const pointer to the bitmap. The row IDs in the bitmap are stored in
+    /// @return A non-owning, const pointer to the bitmap. The row ids in the bitmap are stored in
     ///         ascending order (as guaranteed by Roaring64 iteration).
     Result<const RoaringBitmap64*> GetBitmap() const;
 
     /// @return A const reference to a vector of float scores, where the i-th element corresponds to
-    ///         the i-th row ID when iterating the bitmap in **ascending row ID order**.
+    ///         the i-th row id when iterating the bitmap in **ascending row id order**.
     const std::vector<float>& GetScores() const;
 
  private:

include/paimon/global_index/global_index_io_meta.h

@@ -25,17 +25,17 @@
 namespace paimon {
 /// Metadata describing a single file entry in a global index.
 struct PAIMON_EXPORT GlobalIndexIOMeta {
-    GlobalIndexIOMeta(const std::string& _file_name, int64_t _file_size, const Range& _row_id_range,
+    GlobalIndexIOMeta(const std::string& _file_name, int64_t _file_size, int64_t _range_end,
                       const std::shared_ptr<Bytes>& _metadata)
         : file_name(_file_name),
           file_size(_file_size),
-          row_id_range(_row_id_range),
+          range_end(_range_end),
           metadata(_metadata) {}
 
     std::string file_name;
     int64_t file_size;
-    /// The inclusive range of row IDs covered by this file (i.e., [from, to]).
-    Range row_id_range;
+    /// The inclusive range end covered by this file (i.e., the last local row id).
+    int64_t range_end;
     /// Optional binary metadata associated with the file, such as serialized
     /// secondary index structures or inline index bytes.
     /// May be null if no additional metadata is available.

include/paimon/global_index/global_index_reader.h

@@ -26,18 +26,20 @@
 
 namespace paimon {
 /// Reads and evaluates filter predicates against a global file index.
-/// `GlobalIndexReader` is an implementation of the `FunctionVisitor` interface
-/// specialized to produce `std::shared_ptr<GlobalIndexResult>` objects.
 ///
 /// Derived classes are expected to implement the visitor methods (e.g., `VisitEqual`,
 /// `VisitIsNull`, etc.) to return index-based results that indicate which
 /// row satisfy the given predicate.
+///
+/// @note All `GlobalIndexResult` objects returned by implementations of this class use **local row
+/// ids** that start from 0 — not global row ids in the entire table.
+/// The `GlobalIndexResult` can be converted to global row ids by calling `AddOffset()`.
 class PAIMON_EXPORT GlobalIndexReader : public FunctionVisitor<std::shared_ptr<GlobalIndexResult>> {
  public:
     /// TopKPreFilter: A lightweight pre-filtering function applied **before** similarity scoring.
-    /// It operates solely on row_id and is typically driven by other global index, such as bitmap,
-    /// or range index. This filter enables early pruning of irrelevant candidates (e.g., "only
-    /// consider rows with label X"), significantly reducing the search space. Returns true to
+    /// It operates solely on **local row ids** and is typically driven by other global index, such
+    /// as bitmap, or range index. This filter enables early pruning of irrelevant candidates (e.g.,
+    /// "only consider rows with label X"), significantly reducing the search space. Returns true to
     /// include the row in Top-K computation; false to exclude it.
     ///
     /// @note Must be thread-safe.
@@ -47,7 +49,8 @@ class PAIMON_EXPORT GlobalIndexReader : public FunctionVisitor<std::shared_ptr<G
     ///
     /// @param k         Number of top results to return.
     /// @param query     The query vector (must match the dimensionality of the indexed vectors).
-    /// @param filter    A pre-filter based on row_id, implemented by leveraging other global index
+    /// @param filter    A pre-filter based on **local row ids**, implemented by leveraging other
+    /// global index
     ///                   structures (e.g., bitmap index) for efficient candidate pruning.
     /// @param predicate A runtime filtering condition that may involve graph traversal of
     ///                   structured attributes. **Using this parameter often yields better
@@ -58,7 +61,8 @@ class PAIMON_EXPORT GlobalIndexReader : public FunctionVisitor<std::shared_ptr<G
     ///                   context-aware filtering at query time.
     /// @note All fields referenced in the predicate must have been materialized
     ///       in the index during build to ensure availability.
-    /// @note `VisitTopK` is thread-safe while other `VisitXXX` is not.
+    /// @note `VisitTopK` is thread-safe (not coroutine-safe) while other `VisitXXX` is not
+    /// thread-safe.
     virtual Result<std::shared_ptr<TopKGlobalIndexResult>> VisitTopK(
         int32_t k, const std::vector<float>& query, TopKPreFilter filter,
         const std::shared_ptr<Predicate>& predicate) = 0;

include/paimon/global_index/global_index_result.h

@@ -44,7 +44,7 @@ class PAIMON_EXPORT GlobalIndexResult : public std::enable_shared_from_this<Glob
         virtual int64_t Next() = 0;
     };
 
-    /// Checks whether the global index result contains no matching row IDs.
+    /// Checks whether the global index result contains no matching row ids.
     ///
     /// @return A `Result<bool>` where:
     ///         - `true` indicates the result is empty (no matching rows),
@@ -67,6 +67,10 @@ class PAIMON_EXPORT GlobalIndexResult : public std::enable_shared_from_this<Glob
     virtual Result<std::shared_ptr<GlobalIndexResult>> Or(
         const std::shared_ptr<GlobalIndexResult>& other);
 
+    /// Adds the given offset to each row id in current result and returns the new global index
+    /// result.
+    virtual Result<std::shared_ptr<GlobalIndexResult>> AddOffset(int64_t offset) = 0;
+
     virtual std::string ToString() const = 0;
 
     /// Serializes a GlobalIndexResult object into a byte array.
@@ -103,7 +107,7 @@ class PAIMON_EXPORT GlobalIndexResult : public std::enable_shared_from_this<Glob
 };
 
 /// Represents the result of a Top-K query against a global index.
-/// This class encapsulates a set of top-K candidates (row ID + score pairs) and provides
+/// This class encapsulates a set of top-K candidates (row id + score pairs) and provides
 /// an iterator interface to traverse them.
 class PAIMON_EXPORT TopKGlobalIndexResult : public GlobalIndexResult {
  public:
@@ -115,7 +119,7 @@ class PAIMON_EXPORT TopKGlobalIndexResult : public GlobalIndexResult {
      public:
         virtual ~TopKIterator() = default;
 
-        /// Checks whether more row IDs are available.
+        /// Checks whether more row ids are available.
         virtual bool HasNext() const = 0;
 
         /// Retrieves the next (row_id, score) pair and advances the iterator.