Skip to content

Commit 31d67c6

Browse files
jerry-024jerry-024
committed
fix test fail
1 parent 4de0e9c commit 31d67c6

File tree

6 files changed

+138
-216
lines changed

6 files changed

+138
-216
lines changed

‎.github/actions/build-faiss-native/action.yml‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
name: 'Build FAISS Native Library'
2+
description: 'Build FAISS native library for specified platform'
3+
inputs:
4+
platform:
5+
description: 'Target platform (linux-amd64, linux-aarch64, darwin-aarch64)'
6+
required: true
7+
cmake-arch:
8+
description: 'CMake architecture (x86_64 or aarch64)'
9+
required: true
10+
use-homebrew:
11+
description: 'Whether to use Homebrew for dependencies (macOS)'
12+
required: false
13+
default: 'false'
14+
15+
runs:
16+
using: 'composite'
17+
steps:
18+
- name: Install native dependencies (Linux)
19+
if: inputs.use-homebrew == 'false'
20+
shell: bash
21+
run: |
22+
sudo apt-get update
23+
sudo apt-get install -y \
24+
build-essential \
25+
libopenblas-dev \
26+
liblapack-dev \
27+
patchelf \
28+
libgomp1 \
29+
wget
30+
31+
- name: Install GCC 9 (Linux)
32+
if: inputs.use-homebrew == 'false'
33+
shell: bash
34+
run: |
35+
sudo apt-get install -y gcc-9 g++-9 || sudo apt-get install -y gcc g++
36+
if command -v gcc-9 &>/dev/null; then
37+
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90
38+
sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 90
39+
fi
40+
gcc --version
41+
42+
- name: Install CMake 3.30.1 (Linux)
43+
if: inputs.use-homebrew == 'false'
44+
shell: bash
45+
run: |
46+
CMAKE_VERSION="3.30.1"
47+
wget -q https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${{ inputs.cmake-arch }}.tar.gz
48+
tar -xzf cmake-${CMAKE_VERSION}-linux-${{ inputs.cmake-arch }}.tar.gz
49+
sudo mv cmake-${CMAKE_VERSION}-linux-${{ inputs.cmake-arch }} /opt/cmake
50+
sudo ln -sf /opt/cmake/bin/cmake /usr/local/bin/cmake
51+
cmake --version
52+
53+
- name: Install FAISS (Linux)
54+
if: inputs.use-homebrew == 'false'
55+
shell: bash
56+
run: |
57+
git clone --depth 1 --branch v1.7.4 https://github.com/facebookresearch/faiss.git /tmp/faiss
58+
cd /tmp/faiss
59+
cmake -B build \
60+
-DFAISS_ENABLE_GPU=OFF \
61+
-DFAISS_ENABLE_PYTHON=OFF \
62+
-DBUILD_TESTING=OFF \
63+
-DCMAKE_BUILD_TYPE=Release
64+
cmake --build build -j $(nproc)
65+
sudo cmake --install build
66+
67+
- name: Install dependencies (macOS)
68+
if: inputs.use-homebrew == 'true'
69+
shell: bash
70+
run: |
71+
brew install cmake libomp openblas faiss
72+
73+
- name: Build native library
74+
shell: bash
75+
run: |
76+
cd paimon-faiss-jni
77+
./scripts/build-native.sh --clean --fat-lib
78+
79+
- name: List built libraries (Linux)
80+
if: inputs.use-homebrew == 'false' && inputs.platform == 'linux-amd64'
81+
shell: bash
82+
run: |
83+
echo "=== Built libraries ==="
84+
ls -la paimon-faiss-jni/src/main/resources/linux/amd64/
85+
echo ""
86+
echo "=== Library dependencies ==="
87+
ldd paimon-faiss-jni/src/main/resources/linux/amd64/libpaimon_faiss_jni.so || true
88+
89+
- name: List built libraries (Linux AARCH64)
90+
if: inputs.use-homebrew == 'false' && inputs.platform == 'linux-aarch64'
91+
shell: bash
92+
run: |
93+
echo "=== Built libraries ==="
94+
ls -la paimon-faiss-jni/src/main/resources/linux/aarch64/
95+
echo ""
96+
echo "=== Library dependencies ==="
97+
ldd paimon-faiss-jni/src/main/resources/linux/aarch64/libpaimon_faiss_jni.so || true
98+
99+
- name: List built libraries (macOS)
100+
if: inputs.use-homebrew == 'true'
101+
shell: bash
102+
run: |
103+
echo "=== Built libraries ==="
104+
ls -la paimon-faiss-jni/src/main/resources/darwin/aarch64/
105+
echo ""
106+
echo "=== Library dependencies ==="
107+
otool -L paimon-faiss-jni/src/main/resources/darwin/aarch64/libpaimon_faiss_jni.dylib || true

‎.github/actions/setup-faiss/action.yml‎

Lines changed: 0 additions & 86 deletions
This file was deleted.

‎.github/workflows/faiss-vector-index-tests.yml‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -52,16 +52,21 @@ jobs:
5252
java-version: ${{ env.JDK_VERSION }}
5353
distribution: 'temurin'
5454

55-
- name: Setup FAISS and build paimon-faiss
56-
uses: ./.github/actions/setup-faiss
55+
- name: Build FAISS native library
56+
uses: ./.github/actions/build-faiss-native
57+
with:
58+
platform: linux-amd64
59+
cmake-arch: x86_64
60+
61+
- name: Build paimon-faiss-jni
62+
shell: bash
63+
run: |
64+
mvn -B clean install -pl paimon-faiss-jni -am -DskipTests -Ppaimon-faiss-vector
5765
58-
- name: List bundled libraries
66+
- name: Build paimon-faiss
67+
shell: bash
5968
run: |
60-
echo "=== Bundled libraries ==="
61-
ls -la paimon-faiss-jni/src/main/resources/linux/amd64/
62-
echo ""
63-
echo "=== Library dependencies ==="
64-
ldd paimon-faiss-jni/src/main/resources/linux/amd64/libpaimon_faiss_jni.so || true
69+
mvn -B clean install -pl paimon-faiss -am -DskipTests -Ppaimon-faiss-vector
6570
6671
- name: Test paimon-faiss-jni
6772
timeout-minutes: 10

‎.github/workflows/publish-faiss-snapshot.yml‎

Lines changed: 16 additions & 120 deletions
Original file line numberDiff line numberDiff line change
@@ -52,57 +52,11 @@ jobs:
5252
java-version: ${{ env.JDK_VERSION }}
5353
distribution: 'temurin'
5454

55-
- name: Install native dependencies
56-
run: |
57-
sudo apt-get update
58-
sudo apt-get install -y \
59-
build-essential \
60-
libopenblas-dev \
61-
liblapack-dev \
62-
patchelf \
63-
libgomp1 \
64-
wget
65-
66-
- name: Install GCC 9
67-
run: |
68-
sudo apt-get install -y gcc-9 g++-9
69-
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90
70-
sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 90
71-
gcc --version
72-
73-
- name: Install CMake 3.30.1
74-
run: |
75-
CMAKE_VERSION="3.30.1"
76-
wget -q https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz
77-
tar -xzf cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz
78-
sudo mv cmake-${CMAKE_VERSION}-linux-x86_64 /opt/cmake
79-
sudo ln -sf /opt/cmake/bin/cmake /usr/local/bin/cmake
80-
cmake --version
81-
82-
- name: Install FAISS
83-
run: |
84-
git clone --depth 1 --branch v1.7.4 https://github.com/facebookresearch/faiss.git /tmp/faiss
85-
cd /tmp/faiss
86-
cmake -B build \
87-
-DFAISS_ENABLE_GPU=OFF \
88-
-DFAISS_ENABLE_PYTHON=OFF \
89-
-DBUILD_TESTING=OFF \
90-
-DCMAKE_BUILD_TYPE=Release
91-
cmake --build build -j $(nproc)
92-
sudo cmake --install build
93-
94-
- name: Build native library
95-
run: |
96-
cd paimon-faiss-jni
97-
./scripts/build-native.sh --clean --fat-lib
98-
99-
- name: List built libraries
100-
run: |
101-
echo "=== Built libraries ==="
102-
ls -la paimon-faiss-jni/src/main/resources/linux/amd64/
103-
echo ""
104-
echo "=== Library dependencies ==="
105-
ldd paimon-faiss-jni/src/main/resources/linux/amd64/libpaimon_faiss_jni.so || true
55+
- name: Build FAISS native library
56+
uses: ./.github/actions/build-faiss-native
57+
with:
58+
platform: linux-amd64
59+
cmake-arch: x86_64
10660

10761
- name: Upload native library
10862
uses: actions/upload-artifact@v4
@@ -124,59 +78,11 @@ jobs:
12478
java-version: ${{ env.JDK_VERSION }}
12579
distribution: 'temurin'
12680

127-
- name: Install native dependencies
128-
run: |
129-
sudo apt-get update
130-
sudo apt-get install -y \
131-
build-essential \
132-
libopenblas-dev \
133-
liblapack-dev \
134-
patchelf \
135-
libgomp1 \
136-
wget
137-
138-
- name: Install GCC 9
139-
run: |
140-
sudo apt-get install -y gcc-9 g++-9 || sudo apt-get install -y gcc g++
141-
if command -v gcc-9 &>/dev/null; then
142-
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90
143-
sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 90
144-
fi
145-
gcc --version
146-
147-
- name: Install CMake 3.30.1
148-
run: |
149-
CMAKE_VERSION="3.30.1"
150-
wget -q https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-aarch64.tar.gz
151-
tar -xzf cmake-${CMAKE_VERSION}-linux-aarch64.tar.gz
152-
sudo mv cmake-${CMAKE_VERSION}-linux-aarch64 /opt/cmake
153-
sudo ln -sf /opt/cmake/bin/cmake /usr/local/bin/cmake
154-
cmake --version
155-
156-
- name: Install FAISS
157-
run: |
158-
git clone --depth 1 --branch v1.7.4 https://github.com/facebookresearch/faiss.git /tmp/faiss
159-
cd /tmp/faiss
160-
cmake -B build \
161-
-DFAISS_ENABLE_GPU=OFF \
162-
-DFAISS_ENABLE_PYTHON=OFF \
163-
-DBUILD_TESTING=OFF \
164-
-DCMAKE_BUILD_TYPE=Release
165-
cmake --build build -j $(nproc)
166-
sudo cmake --install build
167-
168-
- name: Build native library
169-
run: |
170-
cd paimon-faiss-jni
171-
./scripts/build-native.sh --clean --fat-lib
172-
173-
- name: List built libraries
174-
run: |
175-
echo "=== Built libraries ==="
176-
ls -la paimon-faiss-jni/src/main/resources/linux/aarch64/
177-
echo ""
178-
echo "=== Library dependencies ==="
179-
ldd paimon-faiss-jni/src/main/resources/linux/aarch64/libpaimon_faiss_jni.so || true
81+
- name: Build FAISS native library
82+
uses: ./.github/actions/build-faiss-native
83+
with:
84+
platform: linux-aarch64
85+
cmake-arch: aarch64
18086

18187
- name: Upload native library
18288
uses: actions/upload-artifact@v4
@@ -198,22 +104,12 @@ jobs:
198104
java-version: ${{ env.JDK_VERSION }}
199105
distribution: 'temurin'
200106

201-
- name: Install dependencies
202-
run: |
203-
brew install cmake libomp openblas faiss
204-
205-
- name: Build native library
206-
run: |
207-
cd paimon-faiss-jni
208-
./scripts/build-native.sh --clean --fat-lib
209-
210-
- name: List built libraries
211-
run: |
212-
echo "=== Built libraries ==="
213-
ls -la paimon-faiss-jni/src/main/resources/darwin/aarch64/
214-
echo ""
215-
echo "=== Library dependencies ==="
216-
otool -L paimon-faiss-jni/src/main/resources/darwin/aarch64/libpaimon_faiss_jni.dylib || true
107+
- name: Build FAISS native library
108+
uses: ./.github/actions/build-faiss-native
109+
with:
110+
platform: darwin-aarch64
111+
cmake-arch: aarch64
112+
use-homebrew: 'true'
217113

218114
- name: Upload native library
219115
uses: actions/upload-artifact@v4

‎.github/workflows/utitcase.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ jobs:
6666
echo "JVM timezone is set to $jvm_timezone"
6767
6868
# Exclude modules that have their own dedicated test workflows
69-
TEST_MODULES="!paimon-e2e-tests,"
69+
TEST_MODULES="!paimon-e2e-tests,!paimon-vector-e2e-test,!paimon-faiss-jni,!paimon-faiss,"
7070
for suffix in ut 3.5 3.4 3.3 3.2; do
7171
TEST_MODULES+="!org.apache.paimon:paimon-spark-${suffix}_2.12,"
7272
done

0 commit comments

Comments
 (0)