update toolkit version (#110)

* fix ci

* update toolkit version

* add Dockerfile

* add agencyID to psi/mpc serviceName

* support fetch log and fetch job result seperately

Author: cyjseagull

.github/workflows/cpp_full_node_workflow.yml

@@ -33,7 +33,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-12, ubuntu-20.04]
+        os: [macos-13, ubuntu-20.04]
     steps:
       - uses: actions/checkout@v2
         with:
@@ -74,6 +74,7 @@ jobs:
         if: runner.os == 'macOS'
         run: |
           bash -x cpp/tools/install_depends.sh -o macos
+          rm -rf /Users/runner/work/WeDPR-Component/WeDPR-Component/vcpkg/buildtrees/gsasl
           mkdir -p cpp/build && cd cpp/build && cmake -DBUILD_STATIC=ON -DTESTS=ON -DCOVERAGE=ON -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake ../
           make -j3
       - name: Publish Error

.github/workflows/cpp_sdk_workflow.yml

@@ -33,7 +33,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, windows-2019, macos-12]
+        os: [ubuntu-22.04, windows-2019, macos-13]
     steps:
       - uses: actions/checkout@v2
         with:

.github/workflows/cpp_toolkit_workflow.yml

@@ -33,7 +33,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, windows-2019, macos-12]
+        os: [ubuntu-22.04, windows-2019, macos-13]
     steps:
       - uses: actions/checkout@v2
         with:
@@ -72,7 +72,7 @@ jobs:
       - name: Build for windows
         if: runner.os == 'Windows'
         run: |
-          mkdir -p cpp/build && cd cpp/build && cmake -G "Visual Studio 16 2019" -A x64 -DBUILD_PYTHON=ON -DCMAKE_BUILD_TYPE=Release -DTESTS=OFF -DBUILD_WEDPR_TOOLKIT=ON -DVCPKG_TARGET_TRIPLET=x64-windows-static -DVCPKG_HOST_TRIPLET=x64-windows-static  -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake .. && MSBuild /version && MSBuild WeDPR-Component.sln /p:Configuration=Release /p:Platform=x64
+          mkdir -p cpp/build && cd cpp/build && cmake -G "Visual Studio 16 2019" -A x64 -DBUILD_PYTHON=OFF -DCMAKE_BUILD_TYPE=Release -DTESTS=OFF -DBUILD_WEDPR_TOOLKIT=ON -DVCPKG_TARGET_TRIPLET=x64-windows-static -DVCPKG_HOST_TRIPLET=x64-windows-static  -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake .. && MSBuild /version && MSBuild WeDPR-Component.sln /p:Configuration=Release /p:Platform=x64
       - name: Build for linux
         if: runner.os == 'Linux'
         run: |

cpp/tools/install_depends.sh

@@ -75,12 +75,12 @@ install_centos_depends()
 {
     LOG_INFO "install depends for centos ..."
     # install the basic package
-    sudo yum install -y lcov bison flex epel-release centos-release-scl flex bison patch devtoolset-11 rh-perl530-perl cmake3 zlib-devel ccache lcov python-devel python3-devel krb5-devel autoconf
+    sudo yum install -y lcov bison flex epel-release centos-release-scl flex bison patch devtoolset-11 rh-perl530-perl cmake3 zlib-devel ccache lcov python-devel python3-devel autoconf
     LOG_INFO "install basic-package-depends for centos success..."
 
-    LOG_INFO "install gsasl ... "
-    install_gsasl_depend "centos"
-    LOG_INFO "install gsasl success!"
+    #LOG_INFO "install gsasl ... "
+    #install_gsasl_depend "centos"
+    #LOG_INFO "install gsasl success!"
 
     LOG_INFO "install nasm ... "
     install_nasm_depend
@@ -93,12 +93,12 @@ install_ubuntu_depends()
 {
     LOG_INFO "install depends for ubuntu ..."
     # install the basic package
-    sudo apt install -y lcov bison flex g++ libssl-dev openssl cmake git build-essential autoconf texinfo flex patch bison libgmp-dev zlib1g-dev automake pkg-config libtool wget libkrb5-dev autoconf
+    sudo apt install -y lcov bison flex g++ libssl-dev openssl cmake git build-essential autoconf texinfo flex patch bison libgmp-dev zlib1g-dev automake pkg-config libtool wget autoconf
     LOG_INFO "install basic-package-depends for ubuntu success..."
 
-    LOG_INFO "install gsasl ... "
-    install_gsasl_depend "ubuntu"
-    LOG_INFO "install gsasl success!"
+    #LOG_INFO "install gsasl ... "
+    #install_gsasl_depend "ubuntu"
+    #LOG_INFO "install gsasl success!"
 
     LOG_INFO "install nasm ... "
     install_nasm_depend
@@ -125,9 +125,9 @@ install_iconv_depend()
 install_macos_depends()
 {
     LOG_INFO "install depends for macos ..."
-    brew install autoconf nasm lcov iconv
-    install_gsasl_depend "macos"
-    install_iconv_depend
+    brew install autoconf nasm lcov
+    #install_gsasl_depend "macos"
+    #install_iconv_depend
     LOG_INFO "install depends for macos success ..."
 }
 

cpp/wedpr-helper/ppc-utilities/Utilities.h

@@ -64,6 +64,11 @@ inline std::string generateUUID()
     static thread_local auto uuid_gen = boost::uuids::basic_random_generator<std::random_device>();
     return boost::uuids::to_string(uuid_gen());
 }
+
+inline std::string getServiceName(std::string const& agencyID, std::string const& serviceID)
+{
+    return agencyID + "_" + serviceID;
+}
 template <typename T>
 inline std::string printVector(T const& list)
 {

cpp/wedpr-initializer/Initializer.cpp

@@ -40,6 +40,7 @@
 #include "ppc-psi/src/labeled-psi/LabeledPSIFactory.h"
 #include "ppc-psi/src/ra2018-psi/RA2018PSIFactory.h"
 #include "ppc-tools/src/config/PPCConfig.h"
+#include "wedpr-helper/ppc-utilities/Utilities.h"
 #include <tbb/tbb.h>
 #include <thread>
 
@@ -86,8 +87,9 @@ void Initializer::init(ppc::gateway::IGateway::Ptr const& gateway)
     TransportBuilder transportBuilder;
     // register the serviceInfo
     auto serviceConfig = m_serviceConfigBuilder.buildServiceConfig();
-    auto entryPoint =
-        m_serviceConfigBuilder.buildEntryPoint(PSI_SERVICE_TYPE, m_config->accessEntrypoint());
+    auto entryPoint = m_serviceConfigBuilder.buildEntryPoint(
+        getServiceName(m_config->agencyID(), PSI_SERVICE_TYPE), m_config->accessEntrypoint());
+
     serviceConfig.addEntryPoint(entryPoint);
     auto serviceMeta = serviceConfig.encode();
     m_config->frontConfig()->setMeta(serviceMeta);

cpp/wedpr-main/mpc-node/MPCInitializer.cpp

@@ -22,6 +22,7 @@
 #include "ppc-framework/protocol/Constant.h"
 #include "ppc-mpc/src/MPCService.h"
 #include "ppc-tools/src/config/PPCConfig.h"
+#include "wedpr-helper/ppc-utilities/Utilities.h"
 #include "wedpr-protocol/protocol/src/ServiceConfig.h"
 #include "wedpr-transport/sdk/src/TransportBuilder.h"
 
@@ -61,7 +62,7 @@ void MPCInitializer::init(std::string const& _configPath)
     auto threadPool = std::make_shared<bcos::ThreadPool>("mpc-pool", threadPoolSize);
 
     INIT_LOG(INFO) << LOG_DESC("init the mpc threadpool")
-        << LOG_KV("threadPoolSize", threadPoolSize);
+                   << LOG_KV("threadPoolSize", threadPoolSize);
 
     auto mpcService = std::make_shared<MPCService>();
     mpcService->setMPCConfig(mpcConfig);
@@ -71,11 +72,11 @@ void MPCInitializer::init(std::string const& _configPath)
     m_rpc->registerHandler("run", std::bind(&MPCService::runMpcRpc, mpcService,
                                       std::placeholders::_1, std::placeholders::_2));
     m_rpc->registerHandler("asyncRun", std::bind(&MPCService::asyncRunMpcRpc, mpcService,
-                                      std::placeholders::_1, std::placeholders::_2));
+                                           std::placeholders::_1, std::placeholders::_2));
     m_rpc->registerHandler("kill", std::bind(&MPCService::killMpcRpc, mpcService,
                                        std::placeholders::_1, std::placeholders::_2));
     m_rpc->registerHandler("query", std::bind(&MPCService::queryMpcRpc, mpcService,
-                                       std::placeholders::_1, std::placeholders::_2));
+                                        std::placeholders::_1, std::placeholders::_2));
     INIT_LOG(INFO) << LOG_DESC("init the mpc rpc success");
     // init the transport
     initTransport(pt);
@@ -89,8 +90,8 @@ void MPCInitializer::initTransport(boost::property_tree::ptree const& property)
 
     // add the service meta
     ServiceConfigBuilder serviceConfigBuilder;
-    auto entryPoint =
-        serviceConfigBuilder.buildEntryPoint(MPC_SERVICE_TYPE, m_config->accessEntrypoint());
+    auto entryPoint = serviceConfigBuilder.buildEntryPoint(
+        getServiceName(m_config->agencyID(), MPC_SERVICE_TYPE), m_config->accessEntrypoint());
     auto serviceConfig = serviceConfigBuilder.buildServiceConfig();
     serviceConfig.addEntryPoint(entryPoint);
     auto serviceMeta = serviceConfig.encode();

docker-files/cpp/Dockerfile

@@ -0,0 +1,44 @@
+FROM ubuntu:18.04 as builder
+
+LABEL maintainer [email protected]
+
+WORKDIR /
+
+ARG SOURCE_BRANCH
+ENV DEBIAN_FRONTEND=noninteractive \
+    SOURCE=${SOURCE_BRANCH:-master}
+
+RUN apt-get -q update && apt-get install -qy --no-install-recommends \
+    vim curl git make build-essential cmake  \
+    libgmp-dev flex bison patch libzstd-dev unzip ninja-build pkg-config curl zip tar ccache uuid-runtime automake autoconf \
+    m4 tcpdump net-tools libkrb5-dev krb5-user pkg-config default-libmysqlclient-dev gcc g++ \
+    && ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
+    && apt-get install -qy --no-install-recommends tzdata \
+    && dpkg-reconfigure --frontend noninteractive tzdata \
+    && rm -rf /var/lib/apt/lists/*
+
+# install rust
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
+
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+ENV VCPKG_FORCE_SYSTEM_BINARIES=1
+
+RUN git clone https://github.com/WeBankBlockchain/WeDPR-Component.git --recursive --depth=1 -b ${SOURCE} \
+    && mkdir -p WeDPR-Component/cpp/build && cd WeDPR-Component/cpp/build \
+    && cmake .. -DBUILD_STATIC=ON && make -j8 && cat /WeDPR-Component/cpp/build/*.log
+
+
+FROM ubuntu:18.04
+LABEL maintainer [email protected]
+    
+RUN apt-get -q update && apt-get install -qy --no-install-recommends vim libkrb5-dev krb5-user pkg-config default-libmysqlclient-dev \
+        && ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
+        && apt-get install -qy --no-install-recommends tzdata \
+        && dpkg-reconfigure --frontend noninteractive tzdata \
+        && rm -rf /var/lib/apt/lists/*
+    
+COPY --from=builder /WeDPR-Component/cpp/build/bin/ppc-gateway-service /usr/local/bin/ppc-gateway-service
+COPY --from=builder /WeDPR-Component/cpp/build/bin/ppc-pro-node /usr/local/bin/ppc-pro-node
+COPY --from=builder /WeDPR-Component/cpp/build/bin/ppc-air-node /usr/local/bin/ppc-air-node
+COPY --from=builder /WeDPR-Component/cpp/build/bin/wedpr-mpc /usr/local/bin/wedpr-mpc

python/ppc_model/model_result/task_result_handler.py

@@ -11,10 +11,13 @@
 
 
 class TaskResultRequest:
-    def __init__(self, job_id, task_type, only_fetch_log, user):
+    def __init__(self, job_id, task_type,
+                 fetch_log: bool,
+                 fetch_result: bool, user):
         self.job_id = job_id
         self.task_type = task_type
-        self.only_fetch_log = only_fetch_log
+        self.fetch_log = fetch_log
+        self.fetch_result = fetch_result
         self.user = user
 
 
@@ -346,7 +349,7 @@ def __init__(self, task_result_request: TaskResultRequest, components: Initializ
 
     def get_response(self):
         response = dict()
-        if not self.task_result_request.only_fetch_log:
+        if self.task_result_request.fetch_result is True:
             merged_result = dict()
             for result in self.result_list:
                 merged_result.update(result.to_dict())
@@ -356,13 +359,14 @@ def get_response(self):
                 response = {"jobPlanetResult":  merged_result,
                             "modelData": self.model_data}
         # record the log
-        log_size, log_path, log_content = self.components.log_retriever.retrieve_log(
-            self.task_result_request.job_id, self.task_result_request.user)
-        log_result = {}
-        log_result.update({"logSize": log_size})
-        log_result.update({"logPath": log_path})
-        log_result.update({"logContent": log_content})
-        response.update({"logDetail": log_result})
+        if self.task_result_request.fetch_log is True:
+            log_size, log_path, log_content = self.components.log_retriever.retrieve_log(
+                self.task_result_request.job_id, self.task_result_request.user)
+            log_result = {}
+            log_result.update({"logSize": log_size})
+            log_result.update({"logPath": log_path})
+            log_result.update({"logContent": log_content})
+            response.update({"logDetail": log_result})
         return utils.make_response(PpcErrorCode.SUCCESS.get_code(), PpcErrorCode.SUCCESS.get_msg(), response)
 
     def _get_evaluation_result(self):

python/ppc_model/network/http/model_controller.py

@@ -105,12 +105,12 @@ def post(self, task_id):
                 f"get task result, task_id: {task_id}, args: {args}")
             user_name = args['user']
             task_type = args['jobType']
-            only_fetch_log = {'True': True, 'False': False}.get(
-                args['onlyFetchLog'])
+            fetch_log = args['fetchLog']
+            fetch_job_result = args['fetchJobResult']
             components.logger().info(
-                f"get_job_direct_result_response, job: {task_id}")
+                f"get_job_direct_result_response, job: {task_id}, fetch_log: {fetch_log}, fetch_job_result: {fetch_job_result}")
             task_result_request = TaskResultRequest(
-                task_id, task_type, only_fetch_log, user_name)
+                task_id, task_type, fetch_log, fetch_job_result, user_name)
             job_result_handler = TaskResultHandler(
                 task_result_request=task_result_request, components=components)
             response = job_result_handler.get_response()