Skip to content

Commit 75a5561

Browse files
jerry-024jerry-024
committed
[vector] add faiss jni to support faiss in java
1 parent a426a42 commit 75a5561

File tree

23 files changed

+3969
-1
lines changed

23 files changed

+3969
-1
lines changed

‎.github/actions/build-faiss-native/action.yml‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
name: 'Build FAISS Native Library'
2+
description: 'Build FAISS native library for specified platform'
3+
inputs:
4+
platform:
5+
description: 'Target platform (linux-amd64, linux-aarch64, darwin-aarch64)'
6+
required: true
7+
faiss-version:
8+
description: 'FAISS version to build (e.g., 1.7.4)'
9+
required: false
10+
default: '1.7.4'
11+
use-homebrew:
12+
description: 'Whether to use Homebrew for dependencies (macOS)'
13+
required: false
14+
default: 'false'
15+
16+
runs:
17+
using: 'composite'
18+
steps:
19+
- name: Install native dependencies (Linux)
20+
if: inputs.use-homebrew == 'false'
21+
shell: bash
22+
run: |
23+
sudo apt-get update
24+
sudo apt-get install -y \
25+
build-essential \
26+
libopenblas-dev \
27+
liblapack-dev \
28+
patchelf \
29+
libgomp1 \
30+
wget
31+
32+
- name: Install GCC 9 (Linux)
33+
if: inputs.use-homebrew == 'false'
34+
shell: bash
35+
run: |
36+
sudo apt-get install -y gcc-9 g++-9 || sudo apt-get install -y gcc g++
37+
if command -v gcc-9 &>/dev/null; then
38+
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90
39+
sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 90
40+
fi
41+
gcc --version
42+
43+
- name: Install CMake 3.30.1 (Linux)
44+
if: inputs.use-homebrew == 'false'
45+
shell: bash
46+
run: |
47+
CMAKE_VERSION="3.30.1"
48+
if [[ "${{ inputs.platform }}" == *"amd64"* ]]; then
49+
CMAKE_ARCH="x86_64"
50+
else
51+
CMAKE_ARCH="aarch64"
52+
fi
53+
wget -q https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz
54+
tar -xzf cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz
55+
sudo mv cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH} /opt/cmake
56+
sudo ln -sf /opt/cmake/bin/cmake /usr/local/bin/cmake
57+
cmake --version
58+
59+
- name: Install FAISS (Linux)
60+
if: inputs.use-homebrew == 'false'
61+
shell: bash
62+
run: |
63+
git clone --depth 1 --branch v1.7.4 https://github.com/facebookresearch/faiss.git /tmp/faiss
64+
cd /tmp/faiss
65+
cmake -B build \
66+
-DFAISS_ENABLE_GPU=OFF \
67+
-DFAISS_ENABLE_PYTHON=OFF \
68+
-DBUILD_TESTING=OFF \
69+
-DCMAKE_BUILD_TYPE=Release
70+
cmake --build build -j $(nproc)
71+
sudo cmake --install build
72+
73+
- name: Install dependencies (macOS)
74+
if: inputs.use-homebrew == 'true'
75+
shell: bash
76+
run: |
77+
brew install cmake libomp openblas faiss
78+
79+
- name: Build native library
80+
shell: bash
81+
run: |
82+
./paimon-faiss/paimon-faiss-jni/scripts/build-native.sh --clean --fat-lib --faiss-version ${{ inputs.faiss-version }}
83+
84+
- name: List built libraries (Linux)
85+
if: inputs.use-homebrew == 'false' && inputs.platform == 'linux-amd64'
86+
shell: bash
87+
run: |
88+
echo "=== Built libraries ==="
89+
ls -la paimon-faiss-jni/src/main/resources/linux/amd64/
90+
echo ""
91+
echo "=== Library dependencies ==="
92+
ldd paimon-faiss-jni/src/main/resources/linux/amd64/libpaimon_faiss_jni.so || true
93+
94+
- name: List built libraries (Linux AARCH64)
95+
if: inputs.use-homebrew == 'false' && inputs.platform == 'linux-aarch64'
96+
shell: bash
97+
run: |
98+
echo "=== Built libraries ==="
99+
ls -la paimon-faiss-jni/src/main/resources/linux/aarch64/
100+
echo ""
101+
echo "=== Library dependencies ==="
102+
ldd paimon-faiss-jni/src/main/resources/linux/aarch64/libpaimon_faiss_jni.so || true
103+
104+
- name: List built libraries (macOS)
105+
if: inputs.use-homebrew == 'true'
106+
shell: bash
107+
run: |
108+
echo "=== Built libraries ==="
109+
ls -la paimon-faiss-jni/src/main/resources/darwin/aarch64/
110+
echo ""
111+
echo "=== Library dependencies ==="
112+
otool -L paimon-faiss-jni/src/main/resources/darwin/aarch64/libpaimon_faiss_jni.dylib || true

‎.github/workflows/faiss-vector-index-tests.yml‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
################################################################################
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
################################################################################
18+
19+
name: Faiss Vector Index Tests
20+
21+
on:
22+
push:
23+
paths:
24+
- 'paimon-faiss/**'
25+
- '.github/workflows/faiss-vector-index-tests.yml'
26+
pull_request:
27+
paths:
28+
- 'paimon-faiss/**'
29+
- '.github/workflows/faiss-vector-index-tests.yml'
30+
31+
env:
32+
JDK_VERSION: 8
33+
MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true
34+
35+
concurrency:
36+
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
37+
cancel-in-progress: true
38+
39+
jobs:
40+
build_test:
41+
runs-on: ubuntu-latest
42+
timeout-minutes: 90
43+
steps:
44+
- name: Checkout code
45+
uses: actions/checkout@v4
46+
47+
- name: Set up JDK ${{ env.JDK_VERSION }}
48+
uses: actions/setup-java@v4
49+
with:
50+
java-version: ${{ env.JDK_VERSION }}
51+
distribution: 'temurin'
52+
53+
- name: Build FAISS native library
54+
uses: ./.github/actions/build-faiss-native
55+
with:
56+
platform: linux-amd64
57+
58+
- name: Build paimon-faiss-jni
59+
shell: bash
60+
run: mvn -B clean install -pl paimon-faiss/paimon-faiss-jni -am -DskipTests -Ppaimon-faiss
61+
62+
- name: Test paimon-faiss-jni
63+
timeout-minutes: 10
64+
run: mvn -T 1C -B test -pl paimon-faiss/paimon-faiss-jni -DskipFaissTests=false -Ppaimon-faiss
65+
env:
66+
MAVEN_OPTS: -Xmx2048m

‎.github/workflows/utitcase.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ on:
2727
- 'paimon-python/**'
2828
- '.github/workflows/paimon-python-checks.yml'
2929
- 'paimon-lucene/**'
30+
- 'paimon-faiss/**'
31+
- '.github/workflows/faiss-vector-index-tests.yml'
3032

3133
env:
3234
JDK_VERSION: 8
@@ -62,7 +64,7 @@ jobs:
6264
jvm_timezone=$(random_timezone)
6365
echo "JVM timezone is set to $jvm_timezone"
6466
65-
TEST_MODULES="!paimon-e2e-tests,"
67+
TEST_MODULES="!paimon-e2e-tests,!paimon-faiss/paimon-faiss-jni"
6668
for suffix in ut 3.5 3.4 3.3 3.2; do
6769
TEST_MODULES+="!org.apache.paimon:paimon-spark-${suffix}_2.12,"
6870
done

‎.gitignore‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,9 @@ paimon-lucene/.idea/
3131

3232
### Mac OS ###
3333
.DS_Store
34+
35+
### Native build artifacts ###
36+
paimon-faiss/paimon-faiss-jni/build/
37+
paimon-faiss/paimon-faiss-jni/src/main/resources/darwin*
38+
paimon-faiss/paimon-faiss-jni/src/main/resources/linux*
39+
paimon-faiss/paimon-faiss-jni/src/main/native/cmake-build-debug/

‎paimon-faiss/paimon-faiss-jni/NOTICE‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
Apache Paimon Faiss JNI
2+
Copyright 2024-2026 The Apache Software Foundation
3+
4+
This product includes software developed at
5+
The Apache Software Foundation (https://www.apache.org/).
6+
7+
This product includes Faiss (https://github.com/facebookresearch/faiss)
8+
developed by Facebook AI Research under the MIT License.
9+

‎paimon-faiss/paimon-faiss-jni/pom.xml‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one
4+
or more contributor license agreements. See the NOTICE file
5+
distributed with this work for additional information
6+
regarding copyright ownership. The ASF licenses this file
7+
to you under the Apache License, Version 2.0 (the
8+
"License"); you may not use this file except in compliance
9+
with the License. You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing,
14+
software distributed under the License is distributed on an
15+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
KIND, either express or implied. See the License for the
17+
specific language governing permissions and limitations
18+
under the License.
19+
-->
20+
<project xmlns="http://maven.apache.org/POM/4.0.0"
21+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
22+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
23+
<modelVersion>4.0.0</modelVersion>
24+
25+
<parent>
26+
<artifactId>paimon-faiss</artifactId>
27+
<groupId>org.apache.paimon</groupId>
28+
<version>1.4-SNAPSHOT</version>
29+
</parent>
30+
31+
<artifactId>paimon-faiss-jni</artifactId>
32+
<name>Paimon : Faiss JNI</name>
33+
34+
<properties>
35+
<faiss.version>1.7.4</faiss.version>
36+
<skipFaissTests>true</skipFaissTests>
37+
</properties>
38+
39+
<dependencies>
40+
<dependency>
41+
<groupId>org.slf4j</groupId>
42+
<artifactId>slf4j-api</artifactId>
43+
</dependency>
44+
45+
<!-- Test dependencies -->
46+
<dependency>
47+
<groupId>org.junit.jupiter</groupId>
48+
<artifactId>junit-jupiter</artifactId>
49+
<scope>test</scope>
50+
</dependency>
51+
</dependencies>
52+
53+
<build>
54+
<plugins>
55+
<!-- Generate JNI headers -->
56+
<plugin>
57+
<groupId>org.apache.maven.plugins</groupId>
58+
<artifactId>maven-antrun-plugin</artifactId>
59+
<executions>
60+
<execution>
61+
<id>generate-jni-headers</id>
62+
<phase>compile</phase>
63+
<goals>
64+
<goal>run</goal>
65+
</goals>
66+
<configuration>
67+
<target>
68+
<mkdir dir="${project.basedir}/src/main/native/include"/>
69+
</target>
70+
</configuration>
71+
</execution>
72+
</executions>
73+
</plugin>
74+
75+
<!-- Resources plugin to include native libraries -->
76+
<plugin>
77+
<groupId>org.apache.maven.plugins</groupId>
78+
<artifactId>maven-resources-plugin</artifactId>
79+
<configuration>
80+
<nonFilteredFileExtensions>
81+
<nonFilteredFileExtension>so</nonFilteredFileExtension>
82+
<nonFilteredFileExtension>dylib</nonFilteredFileExtension>
83+
</nonFilteredFileExtensions>
84+
</configuration>
85+
</plugin>
86+
87+
<!-- Surefire for tests -->
88+
<plugin>
89+
<groupId>org.apache.maven.plugins</groupId>
90+
<artifactId>maven-surefire-plugin</artifactId>
91+
<configuration>
92+
<skipTests>${skipFaissTests}</skipTests>
93+
</configuration>
94+
</plugin>
95+
</plugins>
96+
</build>
97+
98+
<profiles>
99+
<!-- Profile for deploying to Maven Central with all platform native libraries -->
100+
<profile>
101+
<id>release</id>
102+
<build>
103+
<plugins>
104+
<!-- Include all platform native libraries -->
105+
<plugin>
106+
<groupId>org.apache.maven.plugins</groupId>
107+
<artifactId>maven-resources-plugin</artifactId>
108+
<executions>
109+
<execution>
110+
<id>copy-native-libs</id>
111+
<phase>prepare-package</phase>
112+
<goals>
113+
<goal>copy-resources</goal>
114+
</goals>
115+
<configuration>
116+
<outputDirectory>${project.build.outputDirectory}</outputDirectory>
117+
<resources>
118+
<resource>
119+
<directory>${project.basedir}/src/main/resources</directory>
120+
<includes>
121+
<include>**/*.so</include>
122+
<include>**/*.dylib</include>
123+
</includes>
124+
</resource>
125+
</resources>
126+
</configuration>
127+
</execution>
128+
</executions>
129+
</plugin>
130+
</plugins>
131+
</build>
132+
</profile>
133+
</profiles>
134+
</project>
135+

0 commit comments

Comments
 (0)