[vector index]support vector index by faiss

.github/actions/build-faiss-native/action.yml

@@ -0,0 +1,113 @@
+name: 'Build FAISS Native Library'
+description: 'Build FAISS native library for specified platform'
+inputs:
+  platform:
+    description: 'Target platform (linux-amd64, linux-aarch64, darwin-aarch64)'
+    required: true
+  faiss-version:
+    description: 'FAISS version to build (e.g., 1.7.4)'
+    required: false
+    default: '1.7.4'
+  use-homebrew:
+    description: 'Whether to use Homebrew for dependencies (macOS)'
+    required: false
+    default: 'false'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Install native dependencies (Linux)
+      if: inputs.use-homebrew == 'false'
+      shell: bash
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+          build-essential \
+          libopenblas-dev \
+          liblapack-dev \
+          patchelf \
+          libgomp1 \
+          wget
+
+    - name: Install GCC 9 (Linux)
+      if: inputs.use-homebrew == 'false'
+      shell: bash
+      run: |
+        sudo apt-get install -y gcc-9 g++-9 || sudo apt-get install -y gcc g++
+        if command -v gcc-9 &>/dev/null; then
+          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90
+          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 90
+        fi
+        gcc --version
+
+    - name: Install CMake 3.30.1 (Linux)
+      if: inputs.use-homebrew == 'false'
+      shell: bash
+      run: |
+        CMAKE_VERSION="3.30.1"
+        if [[ "${{ inputs.platform }}" == *"amd64"* ]]; then
+          CMAKE_ARCH="x86_64"
+        else
+          CMAKE_ARCH="aarch64"
+        fi
+        wget -q https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz
+        tar -xzf cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz
+        sudo mv cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH} /opt/cmake
+        sudo ln -sf /opt/cmake/bin/cmake /usr/local/bin/cmake
+        cmake --version
+
+    - name: Install FAISS (Linux)
+      if: inputs.use-homebrew == 'false'
+      shell: bash
+      run: |
+        git clone --depth 1 --branch v${{ inputs.faiss-version }} https://github.com/facebookresearch/faiss.git /tmp/faiss
+        cd /tmp/faiss
+        cmake -B build \
+          -DFAISS_ENABLE_GPU=OFF \
+          -DFAISS_ENABLE_PYTHON=OFF \
+          -DBUILD_TESTING=OFF \
+          -DCMAKE_BUILD_TYPE=Release
+        cmake --build build -j $(nproc)
+        sudo cmake --install build
+
+    - name: Install dependencies (macOS)
+      if: inputs.use-homebrew == 'true'
+      shell: bash
+      run: |
+        brew install cmake libomp openblas faiss
+
+    - name: Build native library
+      shell: bash
+      run: |
+        cd paimon-faiss-jni
+        ./scripts/build-native.sh --clean --fat-lib --faiss-version ${{ inputs.faiss-version }}
+
+    - name: List built libraries (Linux)
+      if: inputs.use-homebrew == 'false' && inputs.platform == 'linux-amd64'
+      shell: bash
+      run: |
+        echo "=== Built libraries ==="
+        ls -la paimon-faiss-jni/src/main/resources/linux/amd64/
+        echo ""
+        echo "=== Library dependencies ==="
+        ldd paimon-faiss-jni/src/main/resources/linux/amd64/libpaimon_faiss_jni.so || true
+
+    - name: List built libraries (Linux AARCH64)
+      if: inputs.use-homebrew == 'false' && inputs.platform == 'linux-aarch64'
+      shell: bash
+      run: |
+        echo "=== Built libraries ==="
+        ls -la paimon-faiss-jni/src/main/resources/linux/aarch64/
+        echo ""
+        echo "=== Library dependencies ==="
+        ldd paimon-faiss-jni/src/main/resources/linux/aarch64/libpaimon_faiss_jni.so || true
+
+    - name: List built libraries (macOS)
+      if: inputs.use-homebrew == 'true'
+      shell: bash
+      run: |
+        echo "=== Built libraries ==="
+        ls -la paimon-faiss-jni/src/main/resources/darwin/aarch64/
+        echo ""
+        echo "=== Library dependencies ==="
+        otool -L paimon-faiss-jni/src/main/resources/darwin/aarch64/libpaimon_faiss_jni.dylib || true

.github/workflows/faiss-vector-index-tests.yml

@@ -0,0 +1,91 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+name: Faiss Vector Index Tests
+
+on:
+  push:
+    paths:
+      - 'paimon-faiss/**'
+      - 'paimon-faiss-jni/**'
+      - '.github/workflows/faiss-vector-index-tests.yml'
+  pull_request:
+    paths:
+      - 'paimon-faiss/**'
+      - 'paimon-faiss-jni/**'
+      - '.github/workflows/faiss-vector-index-tests.yml'
+
+env:
+  JDK_VERSION: 8
+  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  build_test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up JDK ${{ env.JDK_VERSION }}
+        uses: actions/setup-java@v4
+        with:
+          java-version: ${{ env.JDK_VERSION }}
+          distribution: 'temurin'
+
+      - name: Build FAISS native library
+        uses: ./.github/actions/build-faiss-native
+        with:
+          platform: linux-amd64
+
+      - name: Build paimon-faiss-jni
+        shell: bash
+        run: |
+          mvn -B clean install -pl paimon-faiss-jni -am -DskipTests -Ppaimon-faiss-vector
+
+      - name: Build paimon-faiss
+        shell: bash
+        run: |
+          mvn -B clean install -pl paimon-faiss -am -DskipTests -Ppaimon-faiss-vector
+
+      - name: Test paimon-faiss-jni
+        timeout-minutes: 10
+        run: |
+          mvn -T 1C  -B test -pl paimon-faiss-jni -DskipFaissTests=false -Ppaimon-faiss-vector
+        env:
+          MAVEN_OPTS: -Xmx2048m
+
+      - name: Test paimon-faiss
+        timeout-minutes: 30
+        run: |
+          mvn -T 1C  -B test -pl paimon-faiss -Ppaimon-faiss-vector
+        env:
+          MAVEN_OPTS: -Xmx4096m
+
+      - name: Build Vector E2E Test Module
+        run: mvn -T 2C -B clean install -DskipTests -Pspark3,flink1,paimon-faiss-vector -pl paimon-vector-e2e-test -am
+
+      - name: Run Vector E2E Tests
+        timeout-minutes: 30
+        run: mvn -T 2C -B verify -Pspark3,flink1,paimon-faiss-vector -pl paimon-vector-e2e-test
+        env:
+          MAVEN_OPTS: -Xmx4096m

.github/workflows/publish-faiss-snapshot.yml

@@ -0,0 +1,187 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+name: Publish Faiss Snapshot
+
+on:
+  schedule:
+    # At the end of every day
+    - cron: '0 1 * * *'
+  workflow_dispatch:
+  push:
+    paths:
+      - 'paimon-faiss/**'
+      - 'paimon-faiss-jni/**'
+    branches:
+      - master
+
+env:
+  JDK_VERSION: 8
+  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  # Build native library for Linux AMD64
+  build-native-linux-amd64:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up JDK ${{ env.JDK_VERSION }}
+        uses: actions/setup-java@v4
+        with:
+          java-version: ${{ env.JDK_VERSION }}
+          distribution: 'temurin'
+
+      - name: Build FAISS native library
+        uses: ./.github/actions/build-faiss-native
+        with:
+          platform: linux-amd64
+
+      - name: Upload native library
+        uses: actions/upload-artifact@v4
+        with:
+          name: native-linux-amd64
+          path: paimon-faiss-jni/src/main/resources/linux/amd64/
+          retention-days: 1
+
+  # Build native library for Linux AARCH64
+  build-native-linux-aarch64:
+    runs-on: ubuntu-24.04-arm
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up JDK ${{ env.JDK_VERSION }}
+        uses: actions/setup-java@v4
+        with:
+          java-version: ${{ env.JDK_VERSION }}
+          distribution: 'temurin'
+
+      - name: Build FAISS native library
+        uses: ./.github/actions/build-faiss-native
+        with:
+          platform: linux-aarch64
+
+      - name: Upload native library
+        uses: actions/upload-artifact@v4
+        with:
+          name: native-linux-aarch64
+          path: paimon-faiss-jni/src/main/resources/linux/aarch64/
+          retention-days: 1
+
+  # Build native library for macOS ARM (Apple Silicon)
+  build-native-macos-arm:
+    runs-on: macos-14
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up JDK ${{ env.JDK_VERSION }}
+        uses: actions/setup-java@v4
+        with:
+          java-version: ${{ env.JDK_VERSION }}
+          distribution: 'temurin'
+
+      - name: Build FAISS native library
+        uses: ./.github/actions/build-faiss-native
+        with:
+          platform: darwin-aarch64
+          use-homebrew: 'true'
+
+      - name: Upload native library
+        uses: actions/upload-artifact@v4
+        with:
+          name: native-darwin-aarch64
+          path: paimon-faiss-jni/src/main/resources/darwin/aarch64/
+          retention-days: 1
+
+  # Package and publish
+  package-and-publish:
+    if: github.repository == 'apache/paimon'
+    needs: [build-native-linux-amd64, build-native-linux-aarch64, build-native-macos-arm]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up JDK ${{ env.JDK_VERSION }}
+        uses: actions/setup-java@v4
+        with:
+          java-version: ${{ env.JDK_VERSION }}
+          distribution: 'temurin'
+
+      - name: Download Linux AMD64 native library
+        uses: actions/download-artifact@v4
+        with:
+          name: native-linux-amd64
+          path: paimon-faiss-jni/src/main/resources/linux/amd64/
+
+      - name: Download Linux AARCH64 native library
+        uses: actions/download-artifact@v4
+        with:
+          name: native-linux-aarch64
+          path: paimon-faiss-jni/src/main/resources/linux/aarch64/
+
+      - name: Download macOS ARM native library
+        uses: actions/download-artifact@v4
+        with:
+          name: native-darwin-aarch64
+          path: paimon-faiss-jni/src/main/resources/darwin/aarch64/
+
+      - name: List all native libraries
+        run: |
+          echo "=== All native libraries ==="
+          find paimon-faiss-jni/src/main/resources -type f \( -name "*.so" -o -name "*.so.*" -o -name "*.dylib" \) -exec ls -la {} \;
+
+      - name: Cache local Maven repository
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2/repository
+          key: faiss-snapshot-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            faiss-snapshot-maven-
+
+      - name: Build and package paimon-faiss-jni
+        run: |
+          mvn -B clean install -pl paimon-faiss-jni -am -DskipTests -Ppaimon-faiss-vector -Drat.skip
+
+      - name: Build and package paimon-faiss
+        run: |
+          mvn -B clean install -pl paimon-faiss -am -DskipTests -Ppaimon-faiss-vector -Drat.skip
+
+      - name: Publish snapshot
+        env:
+          ASF_USERNAME: ${{ secrets.NEXUS_USER }}
+          ASF_PASSWORD: ${{ secrets.NEXUS_PW }}
+          MAVEN_OPTS: -Xmx4096m
+        run: |
+          tmp_settings="tmp-settings.xml"
+          echo "<settings><servers><server>" > $tmp_settings
+          echo "<id>apache.snapshots.https</id><username>$ASF_USERNAME</username>" >> $tmp_settings
+          echo "<password>$ASF_PASSWORD</password>" >> $tmp_settings
+          echo "</server></servers></settings>" >> $tmp_settings
+
+          mvn --settings $tmp_settings deploy -pl paimon-faiss-jni,paimon-faiss -Dgpg.skip -Drat.skip -DskipTests -Ppaimon-faiss-vector
+
+          rm $tmp_settings
+