Skip to content

Commit 03f7c27

Browse files
jerry-024jerry-024
authored
[vector] add faiss index and e2e test to support vector index (#6991)
1 parent 77ed1d7 commit 03f7c27

20 files changed

+3708
-69
lines changed

‎.github/workflows/faiss-vector-index-tests.yml‎

Lines changed: 0 additions & 66 deletions
This file was deleted.

‎.github/workflows/publish-faiss_snapshot.yml‎

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
################################################################################
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
################################################################################
18+
19+
name: Publish Faiss Snapshot
20+
21+
on:
22+
schedule:
23+
# At the end of every day
24+
- cron: '0 1 * * *'
25+
workflow_dispatch:
26+
push:
27+
paths:
28+
- 'paimon-faiss/**'
29+
branches:
30+
- master
31+
32+
env:
33+
JDK_VERSION: 8
34+
MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true
35+
36+
concurrency:
37+
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
38+
cancel-in-progress: true
39+
40+
jobs:
41+
# Build native library for Linux AMD64
42+
build-native-linux-amd64:
43+
runs-on: ubuntu-latest
44+
steps:
45+
- name: Checkout code
46+
uses: actions/checkout@v4
47+
48+
- name: Set up JDK ${{ env.JDK_VERSION }}
49+
uses: actions/setup-java@v4
50+
with:
51+
java-version: ${{ env.JDK_VERSION }}
52+
distribution: 'temurin'
53+
54+
- name: Build FAISS native library
55+
uses: ./.github/actions/build-faiss-native
56+
with:
57+
platform: linux-amd64
58+
59+
- name: Upload native library
60+
uses: actions/upload-artifact@v4
61+
with:
62+
name: native-linux-amd64
63+
path: paimon-faiss/paimon-faiss-jni/src/main/resources/linux/amd64/
64+
retention-days: 1
65+
66+
# Build native library for Linux AARCH64
67+
build-native-linux-aarch64:
68+
runs-on: ubuntu-24.04-arm
69+
steps:
70+
- name: Checkout code
71+
uses: actions/checkout@v4
72+
73+
- name: Set up JDK ${{ env.JDK_VERSION }}
74+
uses: actions/setup-java@v4
75+
with:
76+
java-version: ${{ env.JDK_VERSION }}
77+
distribution: 'temurin'
78+
79+
- name: Build FAISS native library
80+
uses: ./.github/actions/build-faiss-native
81+
with:
82+
platform: linux-aarch64
83+
84+
- name: Upload native library
85+
uses: actions/upload-artifact@v4
86+
with:
87+
name: native-linux-aarch64
88+
path: paimon-faiss/paimon-faiss-jni/src/main/resources/linux/aarch64/
89+
retention-days: 1
90+
91+
# Build native library for macOS ARM (Apple Silicon)
92+
build-native-macos-arm:
93+
runs-on: macos-14
94+
steps:
95+
- name: Checkout code
96+
uses: actions/checkout@v4
97+
98+
- name: Set up JDK ${{ env.JDK_VERSION }}
99+
uses: actions/setup-java@v4
100+
with:
101+
java-version: ${{ env.JDK_VERSION }}
102+
distribution: 'temurin'
103+
104+
- name: Build FAISS native library
105+
uses: ./.github/actions/build-faiss-native
106+
with:
107+
platform: darwin-aarch64
108+
use-homebrew: 'true'
109+
110+
- name: Upload native library
111+
uses: actions/upload-artifact@v4
112+
with:
113+
name: native-darwin-aarch64
114+
path: paimon-faiss/paimon-faiss-jni/src/main/resources/darwin/aarch64/
115+
retention-days: 1
116+
117+
# Package and publish
118+
package-and-publish:
119+
if: github.repository == 'apache/paimon'
120+
needs: [build-native-linux-amd64, build-native-linux-aarch64, build-native-macos-arm]
121+
runs-on: ubuntu-latest
122+
steps:
123+
- name: Checkout code
124+
uses: actions/checkout@v4
125+
126+
- name: Set up JDK ${{ env.JDK_VERSION }}
127+
uses: actions/setup-java@v4
128+
with:
129+
java-version: ${{ env.JDK_VERSION }}
130+
distribution: 'temurin'
131+
132+
- name: Download Linux AMD64 native library
133+
uses: actions/download-artifact@v4
134+
with:
135+
name: native-linux-amd64
136+
path: paimon-faiss/paimon-faiss-jni/src/main/resources/linux/amd64/
137+
138+
- name: Download Linux AARCH64 native library
139+
uses: actions/download-artifact@v4
140+
with:
141+
name: native-linux-aarch64
142+
path: paimon-faiss/paimon-faiss-jni/src/main/resources/linux/aarch64/
143+
144+
- name: Download macOS ARM native library
145+
uses: actions/download-artifact@v4
146+
with:
147+
name: native-darwin-aarch64
148+
path: paimon-faiss/paimon-faiss-jni/src/main/resources/darwin/aarch64/
149+
150+
- name: List all native libraries
151+
run: |
152+
echo "=== All native libraries ==="
153+
find paimon-faiss/paimon-faiss-jni/src/main/resources -type f \( -name "*.so" -o -name "*.so.*" -o -name "*.dylib" \) -exec ls -la {} \;
154+
155+
- name: Cache local Maven repository
156+
uses: actions/cache@v4
157+
with:
158+
path: ~/.m2/repository
159+
key: faiss-snapshot-maven-${{ hashFiles('**/pom.xml') }}
160+
restore-keys: |
161+
faiss-snapshot-maven-
162+
163+
- name: Build and package paimon-faiss-jni
164+
run: |
165+
mvn -B clean install -pl paimon-faiss/paimon-faiss-jni -am -DskipTests -Ppaimon-faiss -Drat.skip
166+
167+
- name: Build and package paimon-faiss-index
168+
run: |
169+
mvn -B clean install -pl paimon-faiss/paimon-faiss-index -am -DskipTests -Ppaimon-faiss -Drat.skip
170+
171+
- name: Publish snapshot
172+
env:
173+
ASF_USERNAME: ${{ secrets.NEXUS_USER }}
174+
ASF_PASSWORD: ${{ secrets.NEXUS_PW }}
175+
MAVEN_OPTS: -Xmx4096m
176+
run: |
177+
tmp_settings="tmp-settings.xml"
178+
echo "<settings><servers><server>" > $tmp_settings
179+
echo "<id>apache.snapshots.https</id><username>$ASF_USERNAME</username>" >> $tmp_settings
180+
echo "<password>$ASF_PASSWORD</password>" >> $tmp_settings
181+
echo "</server></servers></settings>" >> $tmp_settings
182+
183+
mvn --settings $tmp_settings deploy -pl paimon-faiss/paimon-faiss-jni,paimon-faiss/paimon-faiss-index -Dgpg.skip -Drat.skip -DskipTests -Ppaimon-faiss
184+
185+
rm $tmp_settings
186+

‎.github/workflows/utitcase.yml‎

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,33 @@ concurrency:
3939
cancel-in-progress: true
4040

4141
jobs:
42-
build:
42+
build_native:
4343
runs-on: ubuntu-latest
44+
steps:
45+
- name: Checkout code
46+
uses: actions/checkout@v4
47+
48+
- name: Set up JDK ${{ env.JDK_VERSION }}
49+
uses: actions/setup-java@v4
50+
with:
51+
java-version: ${{ env.JDK_VERSION }}
52+
distribution: 'temurin'
53+
54+
- name: Build FAISS native library
55+
uses: ./.github/actions/build-faiss-native
56+
with:
57+
platform: linux-amd64
58+
59+
- name: Upload native library artifact
60+
uses: actions/upload-artifact@v4
61+
with:
62+
name: faiss-native-linux-amd64
63+
path: paimon-faiss/paimon-faiss-jni/src/main/resources/linux/amd64/
64+
retention-days: 1
65+
66+
build_test:
67+
runs-on: ubuntu-latest
68+
needs: build_native
4469

4570
steps:
4671
- name: Checkout code
@@ -52,10 +77,21 @@ jobs:
5277
java-version: ${{ env.JDK_VERSION }}
5378
distribution: 'temurin'
5479

80+
- name: Download native library artifact
81+
uses: actions/download-artifact@v4
82+
with:
83+
name: faiss-native-linux-amd64
84+
path: paimon-faiss/paimon-faiss-jni/src/main/resources/linux/amd64/
85+
86+
- name: List downloaded native library
87+
run: |
88+
echo "=== Downloaded native libraries ==="
89+
ls -la paimon-faiss/paimon-faiss-jni/src/main/resources/linux/amd64/
90+
5591
- name: Build Others
5692
run: |
5793
echo "Start compiling modules"
58-
mvn -T 2C -B clean install -DskipTests -Pflink1,spark3
94+
mvn -T 2C -B clean install -DskipTests -Pflink1,spark3,paimon-faiss
5995
6096
- name: Test Others
6197
timeout-minutes: 60
@@ -64,7 +100,7 @@ jobs:
64100
jvm_timezone=$(random_timezone)
65101
echo "JVM timezone is set to $jvm_timezone"
66102
67-
TEST_MODULES="!paimon-e2e-tests,!paimon-faiss/paimon-faiss-jni,"
103+
TEST_MODULES="!paimon-e2e-tests,"
68104
for suffix in ut 3.5 3.4 3.3 3.2; do
69105
TEST_MODULES+="!org.apache.paimon:paimon-spark-${suffix}_2.12,"
70106
done

0 commit comments

Comments
 (0)