Merge remote-tracking branch 'origin/candidate-10.0.x' into candidate-10.2.x

Author: GordonSmith

.github/workflows/build-clienttools-windows-2022.yml

@@ -30,7 +30,7 @@ on:
         type: string
         description: 'CMake config options'
         required: false
-        default: '-T host=x64 -A x64 -DUSE_OPTIONAL=OFF -DCLIENTTOOLS_ONLY=ON -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DUSE_PARQUET=OFF -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF'
+        default: '-T host=x64 -A x64 -DUSE_OPTIONAL=OFF -DCLIENTTOOLS_ONLY=ON -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DUSE_PARQUET=OFF -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF -DDIGICERT_KEYPAIR_ALIAS=RIS_Data_Services_HPCC_Systems'
       cmake-build-options:
         type: string
         description: 'CMake build options'
@@ -66,19 +66,32 @@ jobs:
           submodules: recursive
           path: ./LN
 
-      - name: Setup Code Signing
+      - name: Setup Certificate File
+      - name: Setup Certificate File
         shell: "pwsh"
         run: |
-          mkdir -p ./sign
-          cd sign
-          $pfxPath = "hpcc_code_signing.pfx"
-          $encodedBytes = [System.Convert]::FromBase64String("${{ secrets.SIGNING_CERTIFICATE }}")
-          $currentDirectory = Get-Location
-          $certificatePath = Join-Path -Path $currentDirectory -ChildPath $pfxPath
-          [IO.File]::WriteAllBytes("$certificatePath", $encodedBytes)
-          $passphrasePath = Join-Path -Path $currentDirectory -ChildPath "passphrase.txt"
-          Set-Content -Path $passphrasePath -Value "${{ secrets.SIGNING_CERTIFICATE_PASSPHRASE }}"
-          CertUtil -p ${{ secrets.SIGNING_CERTIFICATE_PASSPHRASE }} hpcc_code_signing.pfx           
+          $base64 = "${{ secrets.SM_CLIENT_CERT_FILE_B64 }}"
+          if ([string]::IsNullOrEmpty($base64)) {
+            Write-Error "Secret SM_CLIENT_CERT_FILE_B64 is empty or not set."
+            exit 1
+          }
+          $bytes = [Convert]::FromBase64String($base64)
+          $outDir = "$env:RUNNER_TEMP"
+          $p12Path = Join-Path $outDir "sm_client_cert.p12"
+          [IO.File]::WriteAllBytes($p12Path, $bytes)
+          # Make sure the path is visible to subsequent steps
+          "SM_CLIENT_CERT_FILE=$p12Path" | Out-File -FilePath $env:GITHUB_ENV -Append
+          
+      - name: Setup Software Trust Manager
+        uses: digicert/code-signing-software-trust-action@v1.0.1
+        with:
+          simple-signing-mode: true
+          keypair-alias: "RIS_Data_Services_HPCC_Systems"
+        env:
+          SM_HOST: https://clientauth.one.digicert.com
+          SM_API_KEY: ${{ secrets.DIGICERT_API_KEY }}
+          SM_CLIENT_CERT_FILE: ${{ env.SM_CLIENT_CERT_FILE }}
+          SM_CLIENT_CERT_PASSWORD: ${{ secrets.SM_CLIENT_CERT_PASSWORD }}
 
       - name: vcpkg bootstrap
         shell: "bash"
@@ -109,6 +122,7 @@ jobs:
           fi
 
       - name: Build Clienttools
+        id: build_clienttools
         shell: "bash"
         run: |
           mkdir -p ./build
@@ -118,6 +132,12 @@ jobs:
           else
             cmake --build ./build ${{ inputs.cmake-build-options }} --target SIGN
           fi
+        env:
+          SM_HOST: https://clientauth.one.digicert.com
+          SM_API_KEY: ${{ secrets.DIGICERT_API_KEY }}
+          SM_CLIENT_CERT_FILE: ${{ env.SM_CLIENT_CERT_FILE }}
+          SM_CLIENT_CERT_PASSWORD: ${{ secrets.SM_CLIENT_CERT_PASSWORD }}
+
 
       - name: Release Community Clienttools
         if: ${{ inputs.ln-ref == '' }}
@@ -127,7 +147,7 @@ jobs:
           generateReleaseNotes: false
           prerelease: ${{ contains(github.ref, '-rc') }}
           makeLatest: ${{ inputs.make-latest }}
-          artifacts: "./build/hpccsystems-clienttools-community*.exe"
+          artifacts: "${{ steps.build_clienttools.outputs.packages }}"
 
       - name: Release Internal Clienttools to JFrog Repository
         if: ${{ inputs.ln-ref != '' && github.repository_owner == 'hpcc-systems' }}

.github/workflows/build-eclide.yml

@@ -154,19 +154,32 @@ jobs:
             -username "${{ github.repository_owner }}" \
             -password "${{ secrets.GITHUB_TOKEN }}"
 
-      - name: Load code signing
+      - name: Setup Certificate File
+      - name: Setup Certificate File
         shell: "pwsh"
         run: |
-          mkdir -p ./sign
-          cd sign
-          $pfxPath = "hpcc_code_signing.pfx"
-          $encodedBytes = [System.Convert]::FromBase64String("${{ secrets.SIGNING_CERTIFICATE }}")
-          $currentDirectory = Get-Location
-          $certificatePath = Join-Path -Path $currentDirectory -ChildPath $pfxPath
-          [IO.File]::WriteAllBytes("$certificatePath", $encodedBytes)
-          $passphrasePath = Join-Path -Path $currentDirectory -ChildPath "passphrase.txt"
-          Set-Content -Path $passphrasePath -Value "${{ secrets.SIGNING_CERTIFICATE_PASSPHRASE }}"
-          CertUtil -p ${{ secrets.SIGNING_CERTIFICATE_PASSPHRASE }} hpcc_code_signing.pfx
+          $base64 = "${{ secrets.SM_CLIENT_CERT_FILE_B64 }}"
+          if ([string]::IsNullOrEmpty($base64)) {
+            Write-Error "Secret SM_CLIENT_CERT_FILE_B64 is empty or not set."
+            exit 1
+          }
+          $bytes = [Convert]::FromBase64String($base64)
+          $outDir = "$env:RUNNER_TEMP"
+          $p12Path = Join-Path $outDir "sm_client_cert.p12"
+          [IO.File]::WriteAllBytes($p12Path, $bytes)
+          # Make sure the path is visible to subsequent steps
+          "SM_CLIENT_CERT_FILE=$p12Path" | Out-File -FilePath $env:GITHUB_ENV -Append
+          
+      - name: Setup Software Trust Manager
+        uses: digicert/code-signing-software-trust-action@v1.0.1
+        with:
+          simple-signing-mode: true
+          keypair-alias: "RIS_Data_Services_HPCC_Systems"
+        env:
+          SM_HOST: https://clientauth.one.digicert.com
+          SM_API_KEY: ${{ secrets.DIGICERT_API_KEY }}
+          SM_CLIENT_CERT_FILE: ${{ env.SM_CLIENT_CERT_FILE }}
+          SM_CLIENT_CERT_PASSWORD: ${{ secrets.SM_CLIENT_CERT_PASSWORD }}
 
       - name: Check workspace
         shell: "bash"
@@ -179,17 +192,27 @@ jobs:
           cd ECLIDE
           mkdir -p ./build
           cd ./build
-          cmake -A Win32 -S ../ -B . 
+          cmake -A Win32 -S ../ -B . -DDIGICERT_KEYPAIR_ALIAS=RIS_Data_Services_HPCC_Systems 
           cmake --build . --config RelWithDebInfo --target SIGN --parallel $NUMBER_OF_PROCESSORS
+        env:
+          SM_HOST: https://clientauth.one.digicert.com
+          SM_API_KEY: ${{ secrets.DIGICERT_API_KEY }}
+          SM_CLIENT_CERT_FILE: ${{ env.SM_CLIENT_CERT_FILE }}
+          SM_CLIENT_CERT_PASSWORD: ${{ secrets.SM_CLIENT_CERT_PASSWORD }}
 
       - name: Build Internal ECLIDE
         shell: "bash"
         run: |
           cd ECLIDE
           cd ./build
           rm -rf ./CMakeCache.txt ./CMakeFiles
-          cmake -A Win32 -S ../ -B . -DHPCC_AUDIENCE=internal
+          cmake -A Win32 -S ../ -B . -DHPCC_AUDIENCE=internal -DDIGICERT_KEYPAIR_ALIAS=RIS_Data_Services_HPCC_Systems
           cmake --build . --config RelWithDebInfo --target SIGN --parallel $NUMBER_OF_PROCESSORS
+        env:
+          SM_HOST: https://clientauth.one.digicert.com
+          SM_API_KEY: ${{ secrets.DIGICERT_API_KEY }}
+          SM_CLIENT_CERT_FILE: ${{ env.SM_CLIENT_CERT_FILE }}
+          SM_CLIENT_CERT_PASSWORD: ${{ secrets.SM_CLIENT_CERT_PASSWORD }}
 
       - name: Upload Assets
         uses: ncipollo/release-action@v1.16.0

cmake_modules/codeSigning.cmake

@@ -11,14 +11,15 @@ function(configure_windows_signing target_name package_file_path)
     endif()
 
     # Check for signing passphrase file
-    if(EXISTS "${PROJECT_SOURCE_DIR}/../sign/passphrase.txt")
-        file(STRINGS "${PROJECT_SOURCE_DIR}/../sign/passphrase.txt" PFX_PASSWORD LIMIT_COUNT 1)
-        message("-- Using passphrase from file: ${PROJECT_SOURCE_DIR}/../sign/passphrase.txt")
+    set(DO_CODE_SIGNING FALSE)
+    if(NOT "${DIGICERT_KEYPAIR_ALIAS}" STREQUAL "")
+        message("-- Using DIGICERT_KEYPAIR_ALIAS for code signing")
+        set(DO_CODE_SIGNING TRUE)
     endif()
 
-    if(PFX_PASSWORD)
+    if(DO_CODE_SIGNING)
         # Configure NSIS installer signing
-        set(CPACK_NSIS_FINALIZE_CMD "signtool sign /f \\\"${PROJECT_SOURCE_DIR}/../sign/hpcc_code_signing.pfx\\\" /fd SHA256 /p \\\"${PFX_PASSWORD}\\\" /tr http://timestamp.digicert.com /td SHA256")
+        set(CPACK_NSIS_FINALIZE_CMD "smctl sign --simple --keypair-alias \\\"${DIGICERT_KEYPAIR_ALIAS}\\\" --dynamic-auth --timestamp --verbose --exit-non-zero-on-fail --failfast --input")
 
         set(CPACK_NSIS_DEFINES "
             !define MUI_STARTMENUPAGE_DEFAULTFOLDER \\\"${CPACK_PACKAGE_VENDOR}\\\\${version}\\\\${CPACK_NSIS_DISPLAY_NAME}\\\"
@@ -30,7 +31,7 @@ function(configure_windows_signing target_name package_file_path)
         # Create custom target for package signing
         message("-- Signing package: ${package_file_path}")
         add_custom_target(${target_name}
-            COMMAND signtool sign /f "${PROJECT_SOURCE_DIR}/../sign/hpcc_code_signing.pfx" /fd "SHA256" /p "${PFX_PASSWORD}" /tr "http://timestamp.digicert.com" /td "SHA256" "${package_file_path}"
+            COMMAND smctl sign --simple --input "${package_file_path}" --keypair-alias "${DIGICERT_KEYPAIR_ALIAS}" --dynamic-auth --timestamp --verbose --exit-non-zero-on-fail --failfast
             COMMENT "Digital Signature"
         )
         add_dependencies(${target_name} PACKAGE)
@@ -44,6 +45,6 @@ function(configure_windows_signing target_name package_file_path)
             !define MUI_FINISHPAGE_NOAUTOCLOSE
         " PARENT_SCOPE)
 
-        message(STATUS "Code signing passphrase not found - basic NSIS configuration applied")
+        message(STATUS "Code signing keypair not found - basic NSIS configuration applied")
     endif()
 endfunction()

common/eventconsumption/eventiterator.cpp

@@ -92,24 +92,15 @@ bool CPropertyTreeEvents::nextEvent(CEvent& event)
     return true;
 }
 
-const char* CPropertyTreeEvents::queryFilename() const
+const EventFileProperties& CPropertyTreeEvents::queryFileProperties() const
 {
-    return events->queryProp("@filename");
-}
-
-uint32_t CPropertyTreeEvents::queryVersion() const
-{
-    return uint32_t(events->getPropInt64("@version"));
-}
-
-uint32_t CPropertyTreeEvents::queryBytesRead() const
-{
-    return uint32_t(events->getPropInt64("@bytesRead"));
+    return properties;
 }
 
 CPropertyTreeEvents::CPropertyTreeEvents(const IPropertyTree& _events)
     : CPropertyTreeEvents(_events, true)
 {
+    // delegating constructor
 }
 
 CPropertyTreeEvents::CPropertyTreeEvents(const IPropertyTree& _events, bool _strictParsing)
@@ -119,15 +110,19 @@ CPropertyTreeEvents::CPropertyTreeEvents(const IPropertyTree& _events, bool _str
 {
     // enable the "next" event to populate from the first matching node
     (void)eventsIt->first();
+    properties.path.set(events->queryProp("@filename"));
+    properties.version = uint32_t(events->getPropInt64("@version"));
+    properties.bytesRead = uint32_t(events->getPropInt("@bytesRead"));
 }
 
 void visitIterableEvents(IEventIterator& iter, IEventVisitor& visitor)
 {
     CEvent event;
-    visitor.visitFile(iter.queryFilename(), iter.queryVersion());
+    const EventFileProperties& props = iter.queryFileProperties();
+    visitor.visitFile(props.path, props.version);
     while (iter.nextEvent(event))
         visitor.visitEvent(event);
-    visitor.departFile(iter.queryBytesRead());
+    visitor.departFile(props.bytesRead);
 }
 
 #ifdef _USE_CPPUNIT

common/eventconsumption/eventiterator.h

@@ -21,22 +21,6 @@
 #include "jevent.hpp"
 #include "jptree.hpp"
 
-// An abstraction enabling an event pulling model. Consumers control the pace of event
-// production by calling nextEvent() as needed.
-//
-// The interface is more simplistic than other iterators. A binary event file reader is considered
-// to be a potential event source.
-//
-// The query* methods are used to identify a source of events. The queried values coincide with
-// `IEventVisitor::visitFile` and `IEventVisitor::departFile` parameters.
-interface IEventIterator : extends IInterface
-{
-    virtual bool nextEvent(CEvent& event) = 0;
-    virtual const char* queryFilename() const = 0;
-    virtual uint32_t queryVersion() const = 0;
-    virtual uint32_t queryBytesRead() const = 0;
-};
-
 // Implementation of IEventIterator that extracts event data from a property tree whose contents
 // conform to this format (shown here as YAML):
 //
@@ -62,15 +46,14 @@ class event_decl CPropertyTreeEvents : public CInterfaceOf<IEventIterator>
 {
 public:
     virtual bool nextEvent(CEvent& event) override;
-    virtual const char* queryFilename() const override;
-    virtual uint32_t queryVersion() const override;
-    virtual uint32_t queryBytesRead() const override;
+    virtual const EventFileProperties& queryFileProperties() const override;
 public:
     CPropertyTreeEvents(const IPropertyTree& events);
     CPropertyTreeEvents(const IPropertyTree& events, bool strictParsing);
 protected:
     Linked<const IPropertyTree> events;
     Owned<IPropertyTreeIterator> eventsIt;
+    EventFileProperties properties;
     bool strictParsing{true};
 };
 

dali/base/dafdesc.cpp

@@ -2812,36 +2812,41 @@ static void loadDefaultBases()
     ldbDone = true;
 
     SessionId mysessid = myProcessSession();
+    Owned<IPropertyTree> dirs;
+    // If connected to dali, then use the configuration from there, otherwise fall back to using the local config file)
     if (mysessid)
     {
         Owned<IRemoteConnection> conn = querySDS().connect("/Environment/Software/Directories", mysessid, RTM_LOCK_READ, SDS_CONNECT_TIMEOUT);
-        if (conn) {
-            IPropertyTree* dirs = conn->queryRoot();
-            for (unsigned groupType = 0; groupType < __grp_size; groupType++)
-            {
-                const char *component = componentNames[groupType];
-                for (unsigned replicationLevel = 0; replicationLevel < MAX_REPLICATION_LEVELS; replicationLevel++)
-                {
-                    StringBuffer dirout;
-                    const char *dirType = dirTypeNames[replicationLevel];
-                    if (replicationLevel==1 && groupType!=grp_roxie)
-                        dirType = "mirror";
-                    if (getConfigurationDirectory(dirs, dirType, component,
-                        "dummy",   // NB this is dummy value (but actually hopefully not used anyway)
-                        dirout))
-                       unixBaseDirectories[groupType][replicationLevel].set(dirout.str());
-                }
-            }
+        if (conn)
+            dirs.set(conn->queryRoot());
+    }
+
+    for (unsigned groupType = 0; groupType < __grp_size; groupType++)
+    {
+        const char *component = componentNames[groupType];
+        for (unsigned replicationLevel = 0; replicationLevel < MAX_REPLICATION_LEVELS; replicationLevel++)
+        {
+            StringBuffer dirout;
+            const char *dirType = dirTypeNames[replicationLevel];
+            if (replicationLevel==1 && groupType!=grp_roxie)
+                dirType = "mirror";
+            if (getConfigurationDirectory(dirs, dirType, component,
+                "dummy",   // NB this is dummy value (but actually hopefully not used anyway)
+                dirout))
+                unixBaseDirectories[groupType][replicationLevel].set(dirout.str());
         }
     }
+
     for (unsigned groupType = 0; groupType < __grp_size; groupType++)
+    {
         for (unsigned replicationLevel = 0; replicationLevel < MAX_REPLICATION_LEVELS; replicationLevel++)
         {
             if (unixBaseDirectories[groupType][replicationLevel].isEmpty())
                 unixBaseDirectories[groupType][replicationLevel].set(defaultUnixBaseDirectories[groupType][replicationLevel]);
             if (windowsBaseDirectories[groupType][replicationLevel].isEmpty())
                 windowsBaseDirectories[groupType][replicationLevel].set(defaultWindowsBaseDirectories[groupType][replicationLevel]);
         }
+    }
 }