[feat][store] Makes diskann independent of the system, of course it can be configured. By default it is independent.

Author: yuhaijun999

CMakeLists.txt

@@ -35,12 +35,13 @@ option(DINGO_BUILD_STATIC "Link libraries statically to generate the dingodb bin
 option(ENABLE_FAILPOINT "Enable failpoint" OFF)
 option(WITH_DISKANN "Build with diskann index" OFF)
 option(WITH_MKL "Build with intel mkl" OFF)
-option(BOOST_SEARCH_PATH "")
+option(BOOST_SEARCH_PATH "" OFF)
 option(BUILD_GOOGLE_SANITIZE "Enable google sanitize" OFF)
 option(BRPC_ENABLE_CPU_PROFILER "Enable brpc cpu profiler" OFF)
 option(XDPROCKS_PATH "Enable xdprocks raw engine")
 option(VECTORIZATION_INSTRUCTION_SET "vectorization instruction set")
 option(WITH_LIBURING "Build with liburing" ON)
+option(DISKANN_DEPEND_ON_SYSTEM "diskann require system dependencies for boost and aio " OFF)
 
 message(STATUS CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE})
 message(STATUS THIRD_PARTY_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE})
@@ -286,7 +287,11 @@ include(ftxui)
 include(hnswlib)
 include(bdb)
 include(tantivy-search)
-include(libaio)
+if(WITH_DISKANN)
+  if(NOT DISKANN_DEPEND_ON_SYSTEM)
+    include(libaio)
+  endif()
+endif()
 
 set(MESSAGE_SOURCE_DIR ${CMAKE_SOURCE_DIR}/dingo-store-proto/proto)
 set(MESSAGE_DIR ${CMAKE_CURRENT_BINARY_DIR}/proto)
@@ -364,8 +369,13 @@ set(DYNAMIC_LIB
     ${TANTIVY_SEARCH_LIBRARIES}
     ${FTXUI_COMPONENT_LIBRARY}
     ${FTXUI_DOM_LIBRARY}
-    ${FTXUI_SCREEN_LIBRARY}
-    ${LIBAIO_LIBRARIES})
+    ${FTXUI_SCREEN_LIBRARY})
+
+if(WITH_DISKANN)
+  if(NOT DISKANN_DEPEND_ON_SYSTEM)
+    set(DYNAMIC_LIB ${DYNAMIC_LIB} ${LIBAIO_LIBRARIES})
+  endif()
+endif()
 
 if(ENABLE_XDPROCKS)
   set(DYNAMIC_LIB ${DYNAMIC_LIB} "${XDPROCKS_PATH}/lib/libxdprocks.a" "${XDPROCKS_PATH}/lib/libstorelib.a"
@@ -452,7 +462,13 @@ include_directories(${FAISS_INCLUDE_DIR})
 set(VECTOR_LIB ${FAISS_LIBRARIES} ${OPENMP_LIBRARY})
 
 if(WITH_DISKANN)
-  add_definitions(-DENABLE_DISKANN_MODULE=ON)
+  add_definitions(-DENABLE_DISKANN_MODULE=1)
+  if(DISKANN_DEPEND_ON_SYSTEM)
+    add_definitions(-DDISKANN_DEPEND_ON_SYSTEM=1)
+  else()
+    add_definitions(-DDISKANN_DEPEND_ON_SYSTEM=0)
+  endif()
+
   if(NOT WITH_MKL)
     message(FATAL_ERROR "The WITH_MKL is not ON, please install enable WITH_MKL to build diskann.")
   endif()
@@ -462,15 +478,34 @@ if(WITH_DISKANN)
   find_package(MKL CONFIG REQUIRED)
 
   if(NOT BOOST_SEARCH_PATH)
+    # cmake_policy(SET CMP0167 OLD)
     find_package(Boost REQUIRED COMPONENTS program_options)
+    if(Boost_FOUND)
+      set(BOOST_SUMMARY "")
+      string(APPEND BOOST_SUMMARY "Boost version: ${Boost_VERSION} ")
+      string(APPEND BOOST_SUMMARY "Boost include dirs: ${Boost_INCLUDE_DIRS} ")
+      string(APPEND BOOST_SUMMARY "Boost libraries: ${Boost_LIBRARIES}")
+
+      message(STATUS "${BOOST_SUMMARY}")
+      add_definitions(-DBOOST_SUMMARY="${BOOST_SUMMARY}")
+    else()
+      message(FATAL_ERROR "Boost not found!")
+    endif()
   else()
     message(STATUS "BOOST_SEARCH_PATH=${BOOST_SEARCH_PATH}, use user-defined boost version")
   endif()
 
   include(diskann)
   include_directories(${DISKANN_INCLUDE_DIR})
   set(DEPEND_LIBS ${DEPEND_LIBS} diskann)
-  set(VECTOR_LIB ${VECTOR_LIB} ${DISKANN_LIBRARIES} ${LIBAIO_LIBRARIES})
+  if(DISKANN_DEPEND_ON_SYSTEM)
+    set(VECTOR_LIB ${VECTOR_LIB} ${DISKANN_LIBRARIES} aio)
+    message(STATUS "libaio use system")
+  else()
+    set(VECTOR_LIB ${VECTOR_LIB} ${DISKANN_LIBRARIES} ${LIBAIO_LIBRARIES})
+    message(STATUS "libaio use third-party")
+  endif()
+
 endif()
 
 if(LINK_TCMALLOC)
@@ -671,6 +706,15 @@ add_definitions(-DGIT_COMMIT_TIME="${GIT_COMMIT_TIME}")
 add_definitions(-DDINGO_BUILD_TYPE="${CMAKE_BUILD_TYPE}")
 add_definitions(-DDINGO_CONTRIB_BUILD_TYPE="${THIRD_PARTY_BUILD_TYPE}")
 
+if(WITH_DISKANN)
+  execute_process(
+    COMMAND make -C ${CMAKE_SOURCE_DIR}/contrib/libaio clean
+    OUTPUT_VARIABLE output
+    ERROR_VARIABLE error
+    RESULT_VARIABLE result
+    OUTPUT_STRIP_TRAILING_WHITESPACE)
+endif()
+
 if(ENABLE_FAILPOINT)
   message(STATUS "Enable failpoint")
   add_definitions(-DENABLE_FAILPOINT="ON")

cmake/libaio.cmake

@@ -22,24 +22,8 @@ set(LIBAIO_LIBRARIES
     "${LIBAIO_INSTALL_DIR}/lib/libaio.a"
     CACHE FILEPATH "libaio library." FORCE)
 
-if(THIRD_PARTY_BUILD_TYPE MATCHES "Debug")
-  set(LIBAIO_DEBUG_FLAG "--enable-debug")
-elseif(THIRD_PARTY_BUILD_TYPE MATCHES "RelWithDebInfo")
-  set(LIBAIO_DEBUG_FLAG "--enable-debug")
-endif()
-
-add_custom_target(
-  clean_libaio
-  COMMAND $(MAKE) -C ${LIBAIO_SOURCE_DIR} clean
-  COMMENT
-    "Cleaning libaio.............................................................................................................................................."
-  message
-  (
-  STATUS
-  "Cleaning libaio..............................................................................................................................................."
-  ))
-
-# option(FORCE_REBUILD_LIBAIO "Force rebuild libaio" ON)
+# if(THIRD_PARTY_BUILD_TYPE MATCHES "Debug") set(LIBAIO_DEBUG_FLAG "--enable-debug") elseif(THIRD_PARTY_BUILD_TYPE
+# MATCHES "RelWithDebInfo") set(LIBAIO_DEBUG_FLAG "--enable-debug") endif()
 
 ExternalProject_Add(
   extern_libaio
@@ -48,13 +32,10 @@ ExternalProject_Add(
   # BINARY_DIR ${LIBAIO_BINARY_DIR}
   PREFIX ${LIBAIO_BINARY_DIR}
   BUILD_IN_SOURCE ON
-  CONFIGURE_COMMAND
-    ""
-    # BUILD_COMMAND $(MAKE) install prefix=${LIBAIO_BINARY_DIR} includedir=${LIBAIO_BINARY_DIR}/include
-    # libdir=${LIBAIO_BINARY_DIR}/lib BUILD_COMMAND $(MAKE) -C src lib BUILD_COMMAND $(MAKE)
-    # --file=${LIBAIO_SOURCES_DIR}/Makefile CLEAN_COMMAND $(MAKE) -C ${LIBAIO_SOURCES_DIR} clean BUILD_COMMAND $(MAKE)
-    # -C ${LIBAIO_SOURCES_DIR} clean # BUILD_PRE_COMMAND ${CMAKE_COMMAND} -E env $(MAKE) -C ${LIBAIO_SOURCES_DIR} clean
-  BUILD_COMMAND $(MAKE) -C ${LIBAIO_SOURCES_DIR} clean && $(MAKE) --file=${LIBAIO_SOURCES_DIR}/Makefile install prefix=${LIBAIO_BINARY_DIR} includedir=${LIBAIO_BINARY_DIR}/include libdir=${LIBAIO_BINARY_DIR}/lib
+  CONFIGURE_COMMAND ""
+  BUILD_ALWAYS ON
+  BUILD_COMMAND $(MAKE) --file=${LIBAIO_SOURCES_DIR}/Makefile install prefix=${LIBAIO_BINARY_DIR}
+                includedir=${LIBAIO_BINARY_DIR}/include libdir=${LIBAIO_BINARY_DIR}/lib
   INSTALL_COMMAND mkdir -p ${LIBAIO_INSTALL_DIR}/lib ${LIBAIO_INSTALL_DIR}/include
   COMMAND cp ${LIBAIO_BINARY_DIR}/include/libaio.h ${LIBAIO_INSTALL_DIR}/include/
   COMMAND cp ${LIBAIO_BINARY_DIR}/lib/libaio.a ${LIBAIO_INSTALL_DIR}/lib/
@@ -65,5 +46,4 @@ ExternalProject_Add(
 
 add_library(libaio STATIC IMPORTED GLOBAL)
 set_property(TARGET libaio PROPERTY IMPORTED_LOCATION ${LIBAIO_LIBRARIES})
-add_dependencies(libaio clean_libaio extern_libaio)
-# add_dependencies(libaio extern_libaio)
+add_dependencies(libaio extern_libaio)

dingo-store-proto

@@ -19,6 +19,30 @@
 
 namespace dingodb {
 
+#if defined(ENABLE_DISKANN_MODULE)
+#if ENABLE_DISKANN_MODULE == 1
+#define USE_DISKANN true
+#else
+#define USE_DISKANN false
+#endif
+#else  // #if defined(ENABLE_DISKANN_MODULE)
+#define USE_DISKANN false
+#endif
+
+#if defined(DISKANN_DEPEND_ON_SYSTEM)
+#if DISKANN_DEPEND_ON_SYSTEM == 1
+#define ENABLE_DISKANN_DEPEND_ON_SYSTEM true
+#else
+#define ENABLE_DISKANN_DEPEND_ON_SYSTEM false
+#endif
+#else  // #if defined(DISKANN_DEPEND_ON_SYSTEM)
+#define ENABLE_DISKANN_DEPEND_ON_SYSTEM false
+#endif
+
+#if !defined(BOOST_SUMMARY)
+#define BOOST_SUMMARY ""
+#endif
+
 DEFINE_string(git_commit_hash, GIT_VERSION, "current git commit version");
 DEFINE_string(git_tag_name, GIT_TAG_NAME, "current dingo git tag version");
 DEFINE_string(git_commit_user, GIT_COMMIT_USER, "current dingo git commit user");
@@ -33,6 +57,10 @@ DEFINE_bool(use_openblas, false, "use openblas");
 DEFINE_bool(use_tcmalloc, false, "use tcmalloc");
 DEFINE_bool(use_profiler, false, "use profiler");
 DEFINE_bool(use_sanitizer, false, "use sanitizer");
+DEFINE_bool(use_diskann, USE_DISKANN, "use diskann");
+DEFINE_bool(diskann_depend_on_system, ENABLE_DISKANN_DEPEND_ON_SYSTEM,
+            "if true, diskann depend on system, else use third_party.");
+DEFINE_string(boost_summary, BOOST_SUMMARY, "boost summary");
 
 std::string GetBuildFlag() {
 #ifdef USE_MKL
@@ -67,9 +95,11 @@ std::string GetBuildFlag() {
 
   return butil::string_printf(
       "DINGO_STORE USE_MKL:[%s] USE_OPENBLAS:[%s] LINK_TCMALLOC:[%s] BRPC_ENABLE_CPU_PROFILER:[%s] "
-      "USE_SANITIZE:[%s]\n",
+      "USE_SANITIZE:[%s]\nDINGO_STORE USE_DISKANN:[%s]  DISKANN_DEPEND_ON_SYSTEM:[%s]\nDINGO_STORE "
+      "BOOST_SUMMARY:[%s]\n",
       FLAGS_use_mkl ? "ON" : "OFF", FLAGS_use_openblas ? "ON" : "OFF", FLAGS_use_tcmalloc ? "ON" : "OFF",
-      FLAGS_use_profiler ? "ON" : "OFF", FLAGS_use_sanitizer ? "ON" : "OFF");
+      FLAGS_use_profiler ? "ON" : "OFF", FLAGS_use_sanitizer ? "ON" : "OFF", FLAGS_use_diskann ? "ON" : "OFF",
+      FLAGS_diskann_depend_on_system ? "ON" : "OFF", FLAGS_boost_summary.c_str());
 }
 
 void DingoShowVerion() {
@@ -107,6 +137,9 @@ pb::common::VersionInfo GetVersionInfo() {
   version_info.set_use_tcmalloc(FLAGS_use_tcmalloc);
   version_info.set_use_profiler(FLAGS_use_profiler);
   version_info.set_use_sanitizer(FLAGS_use_sanitizer);
+  version_info.set_use_diskann(FLAGS_use_diskann);
+  version_info.set_diskann_depend_on_system(FLAGS_diskann_depend_on_system);
+  version_info.set_boost_summary(FLAGS_boost_summary);
   return version_info;
 }