feat(rocksdb): support building RocksDB with the HDFS plugin in the Pegasus server (#2362)

#2361

Enable building RocksDB with HDFS plugin through the following steps:
1. Introduce [rocksdb-hdfs-env](https://github.com/riversand963/rocksdb-hdfs-env)
plugin when building RocksDB as a third-party dependency.
2. Configure Java and Hadoop environment variables, as well as the dynamic library
search paths, to support compiling and linking the plugin.
3. Apply patches to fix issues encountered during compilation.

Author: empiredan

.github/actions/rebuild_thirdparty_if_needed/action.yaml

@@ -37,15 +37,18 @@ runs:
       # Build third-parties and leave some necessary libraries and source.
       run: |
         rm -f /root/thirdparties-src.zip
+        ../admin_tools/download_hadoop.sh hadoop-bin
+        rm -rf hadoop-bin/share/doc
+        mv hadoop-bin ..
+        # The RocksDB HDFS plugin (rocksdb-hdfs-env) in thirdparty relies on ${HADOOP_HOME}
+        # environment variable to locate the libraries to link against.
+        export HADOOP_HOME="$(dirname "$(pwd)")"/hadoop-bin
         mkdir build
         cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -DUSE_JEMALLOC=${USE_JEMALLOC} -DENABLE_ASAN=${ENABLE_ASAN} -B build/
         cmake --build build/ -j $(nproc)
         rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
         find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
-        ../admin_tools/download_hadoop.sh hadoop-bin
         ../admin_tools/download_zk.sh zookeeper-bin
-        rm -rf hadoop-bin/share/doc
         rm -rf zookeeper-bin/docs
-        mv hadoop-bin ..
         mv zookeeper-bin ..
       shell: bash

cmake_modules/BaseFunctions.cmake

@@ -325,6 +325,42 @@ function(dsn_setup_include_path)#TODO(huangwei5): remove this
   include_directories(${THIRDPARTY_INSTALL_DIR}/include)
 endfunction(dsn_setup_include_path)
 
+function(dsn_setup_java_libs)
+  if (NOT DEFINED ARCH_TYPE)
+    message(FATAL_ERROR "ARCH_TYPE is not defined. Please configure with -DARCH_TYPE=...")
+  endif()
+
+  if (NOT DEFINED JAVA_HOME)
+    message(FATAL_ERROR "JAVA_HOME is not defined. Please configure with -DJAVA_HOME=...")
+  endif()
+
+  if (NOT EXISTS "${JAVA_HOME}")
+    message(FATAL_ERROR "JAVA_HOME does not exist: ${JAVA_HOME}")
+  endif()
+
+  message(STATUS "JAVA_HOME = ${JAVA_HOME}")
+
+  if (APPLE)
+    if (NOT EXISTS "${JAVA_HOME}/lib/server/libjvm.dylib"
+        AND NOT EXISTS "${JAVA_HOME}/jre/lib/server/libjvm.dylib")
+      message(FATAL_ERROR "libjvm.dylib not found under JAVA_HOME: ${JAVA_HOME}")
+    endif()
+  else()
+    if (NOT EXISTS "${JAVA_HOME}/lib/server/libjvm.so"
+        AND NOT EXISTS "${JAVA_HOME}/jre/lib/${ARCH_TYPE}/server/libjvm.so")
+      message(FATAL_ERROR "libjvm.so not found under JAVA_HOME: ${JAVA_HOME}")
+    endif()
+  endif()
+
+  # Provide directories to be searched for JVM libraries such as libjvm.so, libjava.so
+  # and libverify.so.
+  #
+  # Currently these directories are used by the RocksDB HDFS plugin (rocksdb-hdfs-env)
+  # in thirdparty to be searched while linking against JVM libraries for JNI.
+  link_directories(${JAVA_HOME}/jre/lib/${ARCH_TYPE}/server)
+  link_directories(${JAVA_HOME}/jre/lib/${ARCH_TYPE})
+endfunction(dsn_setup_java_libs)
+
 function(dsn_setup_thirdparty_libs)
   set(BOOST_ROOT ${THIRDPARTY_INSTALL_DIR})
   set(Boost_USE_MULTITHREADED ON)
@@ -360,10 +396,7 @@ function(dsn_setup_thirdparty_libs)
   endif()
   find_package(RocksDB REQUIRED)
 
-  # libhdfs
-  find_package(JNI REQUIRED)
-  message (STATUS "JAVA_JVM_LIBRARY=${JAVA_JVM_LIBRARY}")
-  link_libraries(${JAVA_JVM_LIBRARY})
+  dsn_setup_java_libs()
 
   find_package(OpenSSL REQUIRED)
   include_directories(${OPENSSL_INCLUDE_DIR})

run.sh

@@ -276,6 +276,17 @@ function run_build()
     echo "Build start time: `date`"
     start_time=`date +%s`
 
+    case "$(uname)" in
+        Darwin)
+            echo "Currently, macOS does not support ${ROOT}/admin_tools/config_hdfs.sh"
+            ;;
+        *)
+            # The RocksDB HDFS plugin (rocksdb-hdfs-env) in thirdparty relies on ${HADOOP_HOME}
+            # environment variable to locate the libraries to link against.
+            source "${ROOT}"/admin_tools/config_hdfs.sh
+            ;;
+    esac
+
     if [[ ${SKIP_THIRDPARTY} == "YES" ]]; then
         echo "Skip building third-parties..."
     else
@@ -301,6 +312,8 @@ function run_build()
     fi
 
     CMAKE_OPTIONS="${CMAKE_OPTIONS}
+                   -DARCH_TYPE=${ARCH_TYPE}
+                   -DJAVA_HOME=${JAVA_HOME}
                    -DENABLE_GCOV=${ENABLE_GCOV}
                    -DENABLE_GPERF=${ENABLE_GPERF}
                    -DBoost_NO_BOOST_CMAKE=ON

thirdparty/CMakeLists.txt

@@ -502,7 +502,7 @@ set(ROCKSDB_OPTIONS
 if (NOT APPLE)
     set(ROCKSDB_OPTIONS
             ${ROCKSDB_OPTIONS}
-            -DROCKSDB_PLUGINS=encfs)
+            "-DROCKSDB_PLUGINS=encfs hdfs")
 endif ()
 ExternalProject_Add(rocksdb
         URL ${OSS_URL_PREFIX}/rocksdb-v8.5.3.tar.gz
@@ -511,6 +511,9 @@ ExternalProject_Add(rocksdb
         PATCH_COMMAND patch -p1 < ${TP_DIR}/fix_rocksdb-cmake-PORTABLE-option.patch
         COMMAND rm -rf ${TP_DIR}/build/Source/rocksdb/plugin/encfs
         COMMAND git clone -b main --depth=1 https://github.com/pegasus-kv/encfs.git ${TP_DIR}/build/Source/rocksdb/plugin/encfs
+        COMMAND rm -rf ${TP_DIR}/build/Source/rocksdb/plugin/hdfs
+        COMMAND git clone -b master --depth=1 https://github.com/riversand963/rocksdb-hdfs-env.git ${TP_DIR}/build/Source/rocksdb/plugin/hdfs
+        COMMAND cd ${TP_DIR}/build/Source/rocksdb/plugin/hdfs && patch -p1 < ${TP_DIR}/fix_rocksdb-plugin-hdfs.patch
         DEPENDS googletest jemalloc lz4 snappy zstd
         CMAKE_ARGS ${ROCKSDB_OPTIONS}
         DOWNLOAD_EXTRACT_TIMESTAMP true

thirdparty/fix_rocksdb-plugin-hdfs.patch

@@ -0,0 +1,33 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index c660747..5fef10e 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -5,6 +5,7 @@ cmake_minimum_required(VERSION 3.4)
+ # Windows is not supported.
+ 
+ set(hdfs_SOURCES "env_hdfs.cc" "env_hdfs_impl.cc" PARENT_SCOPE)
++set(hdfs_HEADERS "env_hdfs.h" PARENT_SCOPE)
+ set(hdfs_LIBS "hdfs" "dl" "verify" "java" "jvm" PARENT_SCOPE)
+ set(hdfs_INCLUDE_PATHS "$ENV{JAVA_HOME}/include" "$ENV{JAVA_HOME}/include/linux" "$ENV{HADOOP_HOME}/include" PARENT_SCOPE)
+ set(hdfs_LINK_PATHS "$ENV{JAVA_HOME}/jre/lib/amd64/server" "$ENV{JAVA_HOME}/jre/lib/amd64" "$ENV{HADOOP_HOME}/lib/native" PARENT_SCOPE)
+diff --git a/env_hdfs_impl.cc b/env_hdfs_impl.cc
+index 01574bc..3927e5d 100644
+--- a/env_hdfs_impl.cc
++++ b/env_hdfs_impl.cc
+@@ -14,6 +14,7 @@
+ #include <iostream>
+ #include <sstream>
+ #include "logging/logging.h"
++#include "port/sys_time.h"
+ #include "rocksdb/status.h"
+ #include "util/string_util.h"
+ 
+@@ -524,7 +525,7 @@ IOStatus HdfsFileSystem::FileExists(const std::string& fname,
+     default:  // anything else should be an error
+       ROCKS_LOG_FATAL(mylog, "FileExists hdfsExists call failed");
+       return IOStatus::IOError("hdfsExists call failed with error " +
+-                               ROCKSDB_NAMESPACE::ToString(value) + " on path " + fname + ".\n");
++                               std::to_string(value) + " on path " + fname + ".\n");
+   }
+ }
+