hdfs support keberos auth (#107)

* hdfs support krb5

* update libhdfs3 to support krb5

* hdfs support keberos auth

* update ppc_builder

* fix centos ci

* add more logs

* fix hdfs auth bug

* support specify the krb5.conf

* fix centos ci

Author: cyjseagull

.github/workflows/cpp_full_node_workflow.yml

@@ -22,6 +22,8 @@ concurrency:
 env:
   RUST_BACKTRACE: 1
   ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+  ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION: node16
+  ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
 
 jobs:
   build:
@@ -99,23 +101,37 @@ jobs:
         if: runner.os != 'Windows'
         run: |
           cd cpp/build && CTEST_OUTPUT_ON_FAILURE=TRUE make test ARGS="--output-on-failure"
+      - name: Publish Error
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: build-x64-linux-dbg-err.log
+          path: /home/runner/work/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/gsasl/build-x64-linux-dbg-err.log
+        
 
   build_centos:
     name: build_centos full node
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
     continue-on-error: true
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest]
+          container:
+            - centos-7.9
     container:
       image: docker.io/centos:7
       volumes:
         - /usr/local/share/vcpkg:/usr/local/share/vcpkg
+        - /node20217:/node20217:rw,rshared
+        - ${{ matrix.container == 'centos-7.9' && '/node20217:/__e/node20:ro,rshared' || ' ' }}
     steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 5
+      - name: install nodejs20glibc2.17
+        if: ${{ matrix.container == 'centos-7.9' }}
+        run: |
+          curl -LO https://unofficial-builds.nodejs.org/download/release/v20.9.0/node-v20.9.0-linux-x64-glibc-217.tar.xz
+          tar -xf node-v20.9.0-linux-x64-glibc-217.tar.xz --strip-components 1 -C /node20217
+      - name: Check out the repo
+        uses: actions/checkout@v4
       - uses: actions/cache@v3
         id: deps_cache
         with:
@@ -148,6 +164,10 @@ jobs:
         with:
           toolchain: nightly-2022-07-28
           override: true
+      - name: Prepare vcpkg
+        if: runner.os != 'Windows'
+        uses: friendlyanon/setup-vcpkg@v1
+        with: { committish: 51b14cd4e1230dd51c11ffeff6f7d53c61cc5297 }
       - name: Check disk space
         run: df . -h
       - name: Free disk space
@@ -169,7 +189,7 @@ jobs:
           rm -rf python 
           mkdir -p cpp/build
           cd cpp/build
-          cmake3 -DBUILD_STATIC=ON -DCMAKE_BUILD_TYPE=Release -DTESTS=ON -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake ../
+          cmake3 -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake -DBUILD_STATIC=ON -DCMAKE_BUILD_TYPE=Release -DTESTS=ON -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake ../
       - name: FreeDiskSpace
         run: |
           df -lh

.github/workflows/cpp_sdk_workflow.yml

@@ -22,6 +22,8 @@ concurrency:
 env:
   RUST_BACKTRACE: 1
   ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+  ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION: node16
+  ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
 
 jobs:
   build_sdk:
@@ -105,20 +107,27 @@ jobs:
 
   build_centos_for_sdk:
       name: build_centos_for_sdk
-      runs-on: ${{ matrix.os }}
+      runs-on: ubuntu-latest
       continue-on-error: true
       strategy:
         fail-fast: false
         matrix:
-          os: [ubuntu-latest]
+          container:
+            - centos-7.9
       container:
         image: docker.io/centos:7
         volumes:
           - /usr/local/share/vcpkg:/usr/local/share/vcpkg
+          - /node20217:/node20217:rw,rshared
+          - ${{ matrix.container == 'centos-7.9' && '/node20217:/__e/node20:ro,rshared' || ' ' }}
       steps:
-        - uses: actions/checkout@v3
-          with:
-            fetch-depth: 5
+        - name: install nodejs20glibc2.17
+          if: ${{ matrix.container == 'centos-7.9' }}
+          run: |
+            curl -LO https://unofficial-builds.nodejs.org/download/release/v20.9.0/node-v20.9.0-linux-x64-glibc-217.tar.xz
+            tar -xf node-v20.9.0-linux-x64-glibc-217.tar.xz --strip-components 1 -C /node20217
+        - name: Check out the repo
+          uses: actions/checkout@v4
         - uses: actions/cache@v3
           id: deps_cache
           with:
@@ -149,11 +158,15 @@ jobs:
             yum install -y java-11-openjdk-devel git make gcc gcc-c++ glibc-static glibc-devel openssl cmake3 ccache devtoolset-11 llvm-toolset-7.0 rh-perl530-perl libzstd-devel zlib-devel flex bison python-devel python3-devel
             yum install -y rh-perl530-perl cmake3 zlib-devel ccache lcov python-devel python3-devel
             yum install -y git
+        - name: Prepare vcpkg
+          if: runner.os != 'Windows'
+          uses: friendlyanon/setup-vcpkg@v1
+          with: { committish: 51b14cd4e1230dd51c11ffeff6f7d53c61cc5297 }
         - name: Build
           run: |
             alias cmake='cmake3'
             . /opt/rh/devtoolset-11/enable
             mkdir -p cpp/build
             cd cpp/build
-            cmake3 -DCMAKE_BUILD_TYPE=Release -DBUILD_SDK=ON -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake ../
+            cmake3 -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SDK=ON -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake ../
             cmake3 --build . --parallel 3

.github/workflows/cpp_toolkit_workflow.yml

@@ -22,6 +22,8 @@ concurrency:
 env:
   RUST_BACKTRACE: 1
   ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+  ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION: node16
+  ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
 
 jobs:
   build_wedpr_toolkit:
@@ -111,20 +113,27 @@ jobs:
 
   build_centos_for_toolkit:
       name: build_centos_for_toolkit
-      runs-on: ${{ matrix.os }}
+      runs-on: ubuntu-latest
       continue-on-error: true
       strategy:
         fail-fast: false
         matrix:
-          os: [ubuntu-latest]
+            container:
+              - centos-7.9
       container:
         image: docker.io/centos:7
         volumes:
           - /usr/local/share/vcpkg:/usr/local/share/vcpkg
+          - /node20217:/node20217:rw,rshared
+          - ${{ matrix.container == 'centos-7.9' && '/node20217:/__e/node20:ro,rshared' || ' ' }}
       steps:
-        - uses: actions/checkout@v3
-          with:
-            fetch-depth: 5
+        - name: install nodejs20glibc2.17
+          if: ${{ matrix.container == 'centos-7.9' }}
+          run: |
+            curl -LO https://unofficial-builds.nodejs.org/download/release/v20.9.0/node-v20.9.0-linux-x64-glibc-217.tar.xz
+            tar -xf node-v20.9.0-linux-x64-glibc-217.tar.xz --strip-components 1 -C /node20217
+        - name: Check out the repo
+          uses: actions/checkout@v4
         - uses: actions/cache@v3
           id: deps_cache
           with:
@@ -155,13 +164,17 @@ jobs:
             yum install -y java-11-openjdk-devel git make gcc gcc-c++ glibc-static glibc-devel openssl cmake3 ccache devtoolset-11 llvm-toolset-7.0 rh-perl530-perl libzstd-devel zlib-devel flex bison python-devel python3-devel
             yum install -y rh-perl530-perl cmake3 zlib-devel ccache lcov python-devel python3-devel
             yum install -y git
+        - name: Prepare vcpkg
+          if: runner.os != 'Windows'
+          uses: friendlyanon/setup-vcpkg@v1
+          with: { committish: 51b14cd4e1230dd51c11ffeff6f7d53c61cc5297 }
         - name: Build
           run: |
             alias cmake='cmake3'
             . /opt/rh/devtoolset-11/enable
             mkdir -p cpp/build
             cd cpp/build
-            cmake3 -DCMAKE_BUILD_TYPE=Release -DBUILD_WEDPR_TOOLKIT=ON -DBUILD_PYTHON=ON -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake ../
+            cmake3 -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_WEDPR_TOOLKIT=ON -DBUILD_PYTHON=ON -DCMAKE_TOOLCHAIN_FILE=/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake ../
             cmake3 --build . --parallel 3
         - uses: actions/upload-artifact@v3
           with:

cpp/cmake/CompilerSettings.cmake

@@ -197,7 +197,7 @@ endif()
 
 # rust static library linking requirements for macos
 if(APPLE)
-    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -framework Security")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -framework Security -framework Kerberos")
 else()
    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -ldl")
 endif()

cpp/cmake/TargetSettings.cmake

@@ -87,7 +87,7 @@ set(AIR_BINARY_NAME ppc-air-node)
 set(CEM_BINARY_NAME wedpr-cem)
 set(MPC_BINARY_NAME wedpr-mpc)
 
-set(HDFS_LIB libhdfs3-static)
+set(HDFS_LIB libhdfs3)
 
 # set cpu-info
 set(CPU_FEATURES_LIB "")

cpp/ppc-framework/protocol/Krb5AuthConfig.h

@@ -0,0 +1,68 @@
+/*
+ *  Copyright (C) 2022 WeDPR.
+ *  SPDX-License-Identifier: Apache-2.0
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ * @file Krb5AuthConfig.h
+ * @author: yujiechen
+ * @date 2024-12-2
+ */
+#pragma once
+#include "ppc-framework/Common.h"
+#include <memory>
+#include <sstream>
+#include <string>
+
+namespace ppc::protocol
+{
+struct Krb5AuthConfig
+{
+    using Ptr = std::shared_ptr<Krb5AuthConfig>;
+    std::string principal;
+    std::string password;
+    std::string ccachePath;
+    std::string authConfigFilePath = "./conf/krb5.conf";
+    void check() const
+    {
+        if (principal.size() == 0)
+        {
+            BOOST_THROW_EXCEPTION(WeDPRException() << bcos::errinfo_comment(
+                                      "Invalid krb5 auth config: Must set the principal!"));
+        }
+        if (password.size() == 0)
+        {
+            BOOST_THROW_EXCEPTION(WeDPRException() << bcos::errinfo_comment(
+                                      "Invalid krb5 auth config: Must set the password!"));
+        }
+        if (ccachePath.size() == 0)
+        {
+            BOOST_THROW_EXCEPTION(WeDPRException() << bcos::errinfo_comment(
+                                      "Invalid krb5 auth config: Must set the ccachePath!"));
+        }
+        if (authConfigFilePath.size() == 0)
+        {
+            BOOST_THROW_EXCEPTION(
+                WeDPRException() << bcos::errinfo_comment(
+                    "Invalid krb5 auth config: Must set the authConfigFilePath!"));
+        }
+    }
+
+    inline std::string desc() const
+    {
+        std::stringstream oss;
+        oss << LOG_KV("principal", principal) << LOG_KV("ccachePath", ccachePath)
+            << LOG_KV("authConfigFilePath", authConfigFilePath);
+        return oss.str();
+    }
+};
+}  // namespace ppc::protocol

cpp/ppc-framework/protocol/Protocol.h

@@ -23,6 +23,7 @@
 #define WIN32_LEAN_AND_MEAN
 #include <windows.h>
 #endif
+#include "Krb5AuthConfig.h"
 #include "ppc-framework/Common.h"
 #include <bcos-utilities/Log.h>
 #include <map>
@@ -452,6 +453,7 @@ struct FileStorageConnectionOption
     bool replaceDataNodeOnFailure = false;
     // the default connection-timeout for the hdfs is 1000ms
     uint16_t connectionTimeout = 1000;
+    Krb5AuthConfig::Ptr authConfig;
 
     void check() const
     {
@@ -470,14 +472,19 @@ struct FileStorageConnectionOption
             BOOST_THROW_EXCEPTION(WeDPRException() << bcos::errinfo_comment(
                                       "Invalid HDFS Option, Must set valid namenodeport!"));
         }
+        if (authConfig)
+        {
+            authConfig->check();
+        }
     }
     inline std::string desc() const
     {
         std::stringstream oss;
         oss << LOG_KV("nameNode", nameNode) << LOG_KV("nameNodePort", nameNodePort)
             << LOG_KV("user", userName) << LOG_KV("token", token)
             << LOG_KV("replace-datanode-on-failure", replaceDataNodeOnFailure)
-            << LOG_KV("connectionTimeout", connectionTimeout);
+            << LOG_KV("connectionTimeout", connectionTimeout)
+            << LOG_KV("authInfo", authConfig ? authConfig->desc() : "null");
         return oss.str();
     }
 };

cpp/tools/build_ppc.sh

@@ -476,14 +476,24 @@ generate_node_config_ini() {
 
 [hdfs_storage]
     ; the hdfs configuration
-    user = app
+    user = root
     name_node = 127.0.0.1
     name_node_port = 9000
     token =
     ; enable replace-datanode-on-failure or not
     replace-datanode-on-failure = false
     ; the connection-timeout, in ms, default is 1000ms
     connection-timeout = 1000
+    ; enable auth or not, default is false
+    ; enable_krb5_auth = false
+    ; the hdfs kerberos auth principal, used when enable_krb5_auth
+    ; auth_principal = [email protected]
+    ; the hdfs kerberos auth password, used when enable_krb5_auth
+    ; auth_password =
+    ; the ccache path, used when enable_krb5_auth
+    ; ccache_path = /tmp/krb5cc_ppc_node
+    ; the krb5.conf path
+    ; krb5_conf_path  = conf/krb5.conf
 
 
 [ra2018psi]
@@ -541,6 +551,31 @@ generate_node_config_ini() {
 EOF
 }
 
+generate_krb5_file_template()
+{
+  local filepath=$1
+  mkdir -p $(dirname $filepath)   
+   cat << EOF > "${filepath}"
+[libdefaults]
+ default_realm = NODE.DC1.CONSUL
+ dns_lookup_realm = false
+ dns_lookup_kdc = false
+ ticket_lifetime = 24h
+ renew_lifetime = 7d
+ forwardable = true
+
+[realms]
+ NODE.DC1.CONSUL = {
+  kdc = 
+  admin_server =
+ }
+
+[domain_realm]
+ .node.dc1.consul = NODE.DC1.CONSUL
+ node.dc1.consul = NODE.DC1.CONSUL
+EOF
+}
+
 generate_script_template()
 {
     local filepath=$1
@@ -915,6 +950,7 @@ deploy_nodes()
             private_key=$(generate_private_key "${node_dir}/conf")
             node_id=$(cat "${node_dir}/conf/node.nodeid")
             generate_node_config_ini "${node_dir}/config.ini" "${listen_ip}" "${gateway_port}" "${listen_ip}" "${rpc_port}" "${listen_ip}" "${grpc_port}" ${agency_id} "${count}" "${node_id}"
+            generate_krb5_file_template "${node_dir}/conf/krb5.conf"
             generate_p2p_connected_conf "${node_dir}/${p2p_connected_conf_name}" "${connected_nodes}" "false"
             set_value ${ip//./}_count $(($(get_value ${ip//./}_count) + 1))
             ((++count))