Skip to content

Commit f2e3e85

Browse files
jerry-024jerry-024
committed
fix
1 parent 998ff51 commit f2e3e85

File tree

5 files changed

+590
-126
lines changed

5 files changed

+590
-126
lines changed

‎paimon-faiss/src/main/native/CMakeLists.txt‎

Lines changed: 260 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,9 @@ endif()
4040
set(CMAKE_CXX_STANDARD_REQUIRED ON)
4141
set(CMAKE_CXX_EXTENSIONS OFF)
4242

43+
# Option to build fat library with all dependencies bundled
44+
option(FAT_LIBRARY "Build fat library with all dependencies statically linked" ON)
45+
4346
# macOS specific: Find OpenMP from Homebrew
4447
if(APPLE)
4548
# Find libomp installed via Homebrew
@@ -64,7 +67,7 @@ if(APPLE)
6467
endif()
6568
endif()
6669

67-
# Find OpenMP (optional on some systems)
70+
# Find OpenMP
6871
find_package(OpenMP QUIET)
6972
if(OpenMP_CXX_FOUND)
7073
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
@@ -77,62 +80,193 @@ endif()
7780
find_package(JNI REQUIRED)
7881
include_directories(${JNI_INCLUDE_DIRS})
7982

80-
# Find FAISS
81-
# FAISS can be installed via Homebrew: brew install faiss
82-
# Or via conda: conda install -c pytorch faiss-cpu
83-
# Or built from source: https://github.com/facebookresearch/faiss
84-
find_package(faiss QUIET)
85-
if(NOT faiss_FOUND)
86-
# Try to find FAISS using pkg-config
87-
find_package(PkgConfig)
88-
if(PkgConfig_FOUND)
89-
pkg_check_modules(FAISS faiss)
90-
endif()
91-
92-
if(NOT FAISS_FOUND)
93-
# Manual search for FAISS - check build directory structure first
94-
# FAISS build directory has: build/faiss/libfaiss.so and headers in source dir
95-
find_path(FAISS_INCLUDE_DIR faiss/Index.h
96-
HINTS
97-
$ENV{FAISS_HOME}
98-
$ENV{FAISS_HOME}/include
99-
$ENV{FAISS_BUILD_DIR}/../
100-
/root/faiss/faiss
101-
/usr/local/include
102-
/opt/homebrew/include
103-
)
104-
find_library(FAISS_LIBRARY faiss
105-
HINTS
106-
$ENV{FAISS_BUILD_DIR}/faiss
107-
$ENV{FAISS_HOME}/build/faiss
108-
$ENV{FAISS_HOME}/lib
109-
/root/faiss/faiss/build/faiss
110-
/usr/local/lib
111-
/usr/lib
112-
/usr/lib64
113-
/opt/homebrew/lib
114-
)
115-
116-
if(FAISS_INCLUDE_DIR AND FAISS_LIBRARY)
117-
set(FAISS_FOUND TRUE)
118-
set(FAISS_INCLUDE_DIRS ${FAISS_INCLUDE_DIR})
119-
set(FAISS_LIBRARIES ${FAISS_LIBRARY})
120-
message(STATUS "Found FAISS include dir: ${FAISS_INCLUDE_DIR}")
121-
message(STATUS "Found FAISS library: ${FAISS_LIBRARY}")
122-
else()
123-
message(FATAL_ERROR "FAISS not found. Please install FAISS:\n"
124-
" brew install faiss (macOS)\n"
125-
" conda install -c pytorch faiss-cpu\n"
126-
"Or set FAISS_HOME environment variable to the FAISS source/install directory.")
127-
endif()
128-
endif()
83+
# Find FAISS include directory
84+
find_path(FAISS_INCLUDE_DIR faiss/Index.h
85+
HINTS
86+
$ENV{FAISS_HOME}
87+
$ENV{FAISS_HOME}/include
88+
$ENV{FAISS_BUILD_DIR}/../
89+
/root/faiss/faiss
90+
/usr/local/include
91+
/opt/homebrew/include
92+
)
93+
94+
if(FAISS_INCLUDE_DIR)
95+
include_directories(${FAISS_INCLUDE_DIR})
96+
message(STATUS "Found FAISS include dir: ${FAISS_INCLUDE_DIR}")
97+
else()
98+
message(FATAL_ERROR "FAISS headers not found. Set FAISS_HOME environment variable.")
12999
endif()
130100

131-
if(faiss_FOUND)
132-
# Using CMake config from installed FAISS
133-
set(FAISS_LIBRARIES faiss)
101+
# Collect all static libraries for fat library build
102+
set(STATIC_LIBS_TO_BUNDLE "")
103+
104+
if(FAT_LIBRARY AND NOT APPLE)
105+
message(STATUS "")
106+
message(STATUS "========== Building FAT library with all dependencies ==========")
107+
message(STATUS "")
108+
109+
# Find static FAISS library
110+
find_library(FAISS_STATIC_LIB
111+
NAMES libfaiss.a faiss.a
112+
HINTS
113+
$ENV{FAISS_BUILD_DIR}/faiss
114+
$ENV{FAISS_HOME}/build/faiss
115+
$ENV{FAISS_HOME}/lib
116+
/root/faiss/faiss/build/faiss
117+
${HOME}/faiss/build/faiss
118+
/usr/local/lib
119+
/usr/lib
120+
/usr/lib64
121+
/usr/lib/x86_64-linux-gnu
122+
/usr/lib/aarch64-linux-gnu
123+
NO_DEFAULT_PATH
124+
)
125+
if(NOT FAISS_STATIC_LIB)
126+
find_library(FAISS_STATIC_LIB NAMES libfaiss.a)
127+
endif()
128+
129+
if(FAISS_STATIC_LIB)
130+
message(STATUS "Found static FAISS: ${FAISS_STATIC_LIB}")
131+
list(APPEND STATIC_LIBS_TO_BUNDLE ${FAISS_STATIC_LIB})
132+
else()
133+
message(FATAL_ERROR "Static FAISS library (libfaiss.a) not found!\n"
134+
"Build FAISS from source with:\n"
135+
" git clone https://github.com/facebookresearch/faiss.git\n"
136+
" cd faiss\n"
137+
" cmake -B build -DBUILD_SHARED_LIBS=OFF -DFAISS_ENABLE_GPU=OFF -DFAISS_ENABLE_PYTHON=OFF .\n"
138+
" cmake --build build -j$(nproc)\n"
139+
"Then set: export FAISS_HOME=$(pwd) FAISS_BUILD_DIR=$(pwd)/build")
140+
endif()
141+
142+
# Common library search paths for both x86_64 and aarch64
143+
set(LIB_SEARCH_PATHS
144+
/usr/local/lib
145+
/usr/lib
146+
/usr/lib64
147+
/usr/lib/x86_64-linux-gnu
148+
/usr/lib/aarch64-linux-gnu
149+
/opt/OpenBLAS/lib
150+
)
151+
152+
# GCC library search paths
153+
set(GCC_SEARCH_PATHS
154+
/usr/lib/gcc/x86_64-linux-gnu
155+
/usr/lib/gcc/aarch64-linux-gnu
156+
/usr/lib64/gcc/x86_64-linux-gnu
157+
/usr/lib64/gcc/aarch64-linux-gnu
158+
/usr/lib/gcc
159+
)
160+
161+
# Find static OpenBLAS
162+
find_library(OPENBLAS_STATIC_LIB
163+
NAMES libopenblas.a openblas.a
164+
HINTS ${LIB_SEARCH_PATHS}
165+
NO_DEFAULT_PATH
166+
)
167+
if(NOT OPENBLAS_STATIC_LIB)
168+
find_library(OPENBLAS_STATIC_LIB NAMES libopenblas.a)
169+
endif()
170+
171+
if(OPENBLAS_STATIC_LIB)
172+
message(STATUS "Found static OpenBLAS: ${OPENBLAS_STATIC_LIB}")
173+
list(APPEND STATIC_LIBS_TO_BUNDLE ${OPENBLAS_STATIC_LIB})
174+
else()
175+
message(STATUS "Static OpenBLAS not found, will try dynamic linking")
176+
endif()
177+
178+
# Find static LAPACK (optional)
179+
find_library(LAPACK_STATIC_LIB
180+
NAMES liblapack.a lapack.a
181+
HINTS ${LIB_SEARCH_PATHS}
182+
NO_DEFAULT_PATH
183+
)
184+
if(NOT LAPACK_STATIC_LIB)
185+
find_library(LAPACK_STATIC_LIB NAMES liblapack.a)
186+
endif()
187+
188+
if(LAPACK_STATIC_LIB)
189+
message(STATUS "Found static LAPACK: ${LAPACK_STATIC_LIB}")
190+
list(APPEND STATIC_LIBS_TO_BUNDLE ${LAPACK_STATIC_LIB})
191+
endif()
192+
193+
# Find static BLAS (sometimes needed separately)
194+
find_library(BLAS_STATIC_LIB
195+
NAMES libblas.a blas.a
196+
HINTS ${LIB_SEARCH_PATHS}
197+
)
198+
if(BLAS_STATIC_LIB)
199+
message(STATUS "Found static BLAS: ${BLAS_STATIC_LIB}")
200+
list(APPEND STATIC_LIBS_TO_BUNDLE ${BLAS_STATIC_LIB})
201+
endif()
202+
203+
# Find static gfortran runtime (needed by LAPACK/BLAS)
204+
find_library(GFORTRAN_STATIC_LIB
205+
NAMES libgfortran.a
206+
HINTS ${GCC_SEARCH_PATHS}
207+
PATH_SUFFIXES 13 12 11 10 9 8 7
208+
)
209+
if(GFORTRAN_STATIC_LIB)
210+
message(STATUS "Found static gfortran: ${GFORTRAN_STATIC_LIB}")
211+
list(APPEND STATIC_LIBS_TO_BUNDLE ${GFORTRAN_STATIC_LIB})
212+
endif()
213+
214+
# Find static OpenMP/GOMP
215+
find_library(GOMP_STATIC_LIB
216+
NAMES libgomp.a
217+
HINTS ${GCC_SEARCH_PATHS}
218+
PATH_SUFFIXES 13 12 11 10 9 8 7
219+
)
220+
if(GOMP_STATIC_LIB)
221+
message(STATUS "Found static GOMP: ${GOMP_STATIC_LIB}")
222+
list(APPEND STATIC_LIBS_TO_BUNDLE ${GOMP_STATIC_LIB})
223+
endif()
224+
225+
# Find static quadmath (needed by gfortran on x86_64)
226+
find_library(QUADMATH_STATIC_LIB
227+
NAMES libquadmath.a
228+
HINTS ${GCC_SEARCH_PATHS}
229+
PATH_SUFFIXES 13 12 11 10 9 8 7
230+
)
231+
if(QUADMATH_STATIC_LIB)
232+
message(STATUS "Found static quadmath: ${QUADMATH_STATIC_LIB}")
233+
list(APPEND STATIC_LIBS_TO_BUNDLE ${QUADMATH_STATIC_LIB})
234+
endif()
235+
236+
message(STATUS "")
237+
message(STATUS "Static libraries to bundle: ${STATIC_LIBS_TO_BUNDLE}")
238+
message(STATUS "")
239+
240+
set(FAT_BUILD TRUE)
134241
else()
135-
include_directories(${FAISS_INCLUDE_DIRS})
242+
# Find shared FAISS library for dynamic linking
243+
find_library(FAISS_SHARED_LIB
244+
NAMES faiss libfaiss.so
245+
HINTS
246+
$ENV{FAISS_BUILD_DIR}/faiss
247+
$ENV{FAISS_HOME}/build/faiss
248+
$ENV{FAISS_HOME}/lib
249+
/root/faiss/faiss/build/faiss
250+
/usr/local/lib
251+
/usr/lib
252+
/usr/lib64
253+
/opt/homebrew/lib
254+
)
255+
256+
if(FAISS_SHARED_LIB)
257+
message(STATUS "Found shared FAISS: ${FAISS_SHARED_LIB}")
258+
else()
259+
# Try CMake config
260+
find_package(faiss QUIET)
261+
if(faiss_FOUND)
262+
set(FAISS_SHARED_LIB faiss)
263+
message(STATUS "Found FAISS via CMake config")
264+
else()
265+
message(FATAL_ERROR "FAISS library not found. Install or build FAISS first.")
266+
endif()
267+
endif()
268+
269+
set(FAT_BUILD FALSE)
136270
endif()
137271

138272
# Create the shared library
@@ -141,16 +275,76 @@ add_library(paimon_faiss_jni SHARED
141275
)
142276

143277
# Link libraries
144-
target_link_libraries(paimon_faiss_jni
145-
${FAISS_LIBRARIES}
146-
${JNI_LIBRARIES}
147-
)
148-
149-
# Link OpenMP if found
150-
if(OpenMP_CXX_FOUND)
151-
target_link_libraries(paimon_faiss_jni OpenMP::OpenMP_CXX)
152-
elseif(APPLE AND LIBOMP_PREFIX)
153-
target_link_libraries(paimon_faiss_jni "${LIBOMP_PREFIX}/lib/libomp.dylib")
278+
if(FAT_BUILD)
279+
message(STATUS "Linking all dependencies into fat library...")
280+
281+
# For proper fat library linking, we need to:
282+
# 1. Use -Wl,--whole-archive to include ALL symbols from static libs
283+
# 2. Use -Wl,--no-whole-archive after the static libs
284+
# 3. Link system libraries normally
285+
286+
# Build the whole-archive link line
287+
set(WHOLE_ARCHIVE_LIBS "")
288+
foreach(static_lib ${STATIC_LIBS_TO_BUNDLE})
289+
list(APPEND WHOLE_ARCHIVE_LIBS ${static_lib})
290+
endforeach()
291+
292+
# Use generator expressions for proper whole-archive linking
293+
target_link_libraries(paimon_faiss_jni PRIVATE
294+
-Wl,--whole-archive
295+
${WHOLE_ARCHIVE_LIBS}
296+
-Wl,--no-whole-archive
297+
${JNI_LIBRARIES}
298+
pthread
299+
m
300+
dl
301+
)
302+
303+
# If static OpenBLAS wasn't found, link dynamically
304+
if(NOT OPENBLAS_STATIC_LIB)
305+
target_link_libraries(paimon_faiss_jni PRIVATE openblas)
306+
endif()
307+
308+
# If static GOMP wasn't found, link dynamically
309+
if(NOT GOMP_STATIC_LIB AND OpenMP_CXX_FOUND)
310+
target_link_libraries(paimon_faiss_jni PRIVATE gomp)
311+
endif()
312+
313+
# If static gfortran wasn't found but LAPACK is used, link dynamically
314+
if(NOT GFORTRAN_STATIC_LIB AND LAPACK_STATIC_LIB)
315+
target_link_libraries(paimon_faiss_jni PRIVATE gfortran)
316+
endif()
317+
318+
# Set RPATH to $ORIGIN for any remaining shared deps
319+
set_target_properties(paimon_faiss_jni PROPERTIES
320+
BUILD_WITH_INSTALL_RPATH TRUE
321+
INSTALL_RPATH "$ORIGIN"
322+
)
323+
324+
# Hide internal symbols to reduce size and avoid symbol conflicts
325+
set_target_properties(paimon_faiss_jni PROPERTIES
326+
C_VISIBILITY_PRESET hidden
327+
CXX_VISIBILITY_PRESET hidden
328+
VISIBILITY_INLINES_HIDDEN ON
329+
)
330+
331+
# Strip unnecessary symbols to reduce size
332+
add_custom_command(TARGET paimon_faiss_jni POST_BUILD
333+
COMMAND ${CMAKE_STRIP} --strip-unneeded $<TARGET_FILE:paimon_faiss_jni> 2>/dev/null || true
334+
COMMENT "Stripping unnecessary symbols from fat library"
335+
)
336+
else()
337+
# Dynamic linking
338+
target_link_libraries(paimon_faiss_jni
339+
${FAISS_SHARED_LIB}
340+
${JNI_LIBRARIES}
341+
)
342+
343+
if(OpenMP_CXX_FOUND)
344+
target_link_libraries(paimon_faiss_jni OpenMP::OpenMP_CXX)
345+
elseif(APPLE AND LIBOMP_PREFIX)
346+
target_link_libraries(paimon_faiss_jni "${LIBOMP_PREFIX}/lib/libomp.dylib")
347+
endif()
154348
endif()
155349

156350
# Set output directory
@@ -162,6 +356,8 @@ set_target_properties(paimon_faiss_jni PROPERTIES
162356
if(APPLE)
163357
set_target_properties(paimon_faiss_jni PROPERTIES
164358
SUFFIX ".dylib"
359+
BUILD_WITH_INSTALL_RPATH TRUE
360+
INSTALL_RPATH "@loader_path;/opt/homebrew/lib;/usr/local/lib"
165361
)
166362
elseif(UNIX)
167363
set_target_properties(paimon_faiss_jni PROPERTIES

0 commit comments

Comments
 (0)