Merge remote-tracking branch 'origin/beta-releases' into ga-releases

Author: github-actions[bot]

.evergreen/functions.yml

@@ -64,10 +64,6 @@ variables:
     E2E_TESTS_ATLAS_READANYDATABASE_STRING: ${e2e_tests_atlas_readanydatabase_string}
     E2E_TESTS_ATLAS_CUSTOMROLE_STRING: ${e2e_tests_atlas_customrole_string}
     E2E_TESTS_ATLAS_SPECIFICPERMISSION_STRING: ${e2e_tests_atlas_specificpermission_string}
-    NOTARY_URL: http://notary-service.build.10gen.cc:5000
-    NOTARY_AUTH_TOKEN: ${signing_auth_token}
-    NOTARY_SIGNING_KEY: ${signing_key_name}
-    NOTARY_SIGNING_COMMENT: Evergreen project mongodb/compass ${revision} - ${build_variant} - ${branch_name}
     MACOS_NOTARY_KEY: ${macos_notary_key}
     MACOS_NOTARY_SECRET: ${macos_notary_secret}
     MACOS_NOTARY_CLIENT_URL: 'https://macos-notary-1628249594.s3.amazonaws.com/releases/client/latest/darwin_amd64.zip'
@@ -80,6 +76,10 @@ variables:
     MONGODB_RUNNER_LOG_DIR: ${workdir}/src/.testserver/
     E2E_TESTS_ATLAS_CS_WITHOUT_SEARCH: ${e2e_tests_atlas_cs_without_search}
     E2E_TESTS_ATLAS_CS_WITH_SEARCH: ${e2e_tests_atlas_cs_with_search}
+    GARASIGN_USERNAME: ${garasign_username}
+    GARASIGN_PASSWORD: ${garasign_password}
+    ARTIFACTORY_USERNAME: ${artifactory_username}
+    ARTIFACTORY_PASSWORD: ${artifactory_password}
 
 # This is here with the variables because anchors aren't supported across includes
 post:
@@ -122,10 +122,9 @@ post:
       remote_file: ${project}/${revision}_${revision_order_id}/vulnerability-report.md
       content_type: text/markdown
       optional: true
-  - command: attach.results
+  - command: attach.xunit_results
     params:
-      file_location: src/packages/compass-e2e-tests/.log/report.json
-
+      file: src/.logs/*.xml
 functions:
   clone:
     - command: git.get_project
@@ -307,6 +306,85 @@ functions:
         # and be able to find the binary that is used for the tests
         file: src/packages/compass/expansions.yml
 
+  spawn-signing-server:
+    # spawn
+    - command: host.create
+      type: setup
+      params:
+        provider: ec2
+        distro: ubuntu2004-large
+        security_group_ids:
+          - sg-097bff6dd0d1d31d0 # Magic string that's needed for SSH'ing.
+    # write host info (this file will be read by signingtool when connection to ssh server)
+    - command: host.list
+      params:
+        num_hosts: 1
+        path: spawned_hosts.json
+        timeout_seconds: 1200
+        wait: true
+    # copy ssh key (this key will be used to connect to ssh server)
+    - command: shell.exec
+      params:
+        shell: bash
+        script: |
+          set -e
+          {
+          set +x
+          echo '${__project_aws_ssh_key_value}' > ~/.ssh/mcipacker.pem
+          chmod 0600 ~/.ssh/mcipacker.pem
+          set -x
+          }
+    # wait for host to be ready
+    - command: shell.exec
+      params:
+        exec_as_string: true
+        shell: bash
+        script: |
+          set -e
+          user=ubuntu
+          hostname=$(tr -d '"[]{}' < spawned_hosts.json | cut -d , -f 1 | awk -F : '{print $2}')
+          identity_file=$(echo ~/.ssh/mcipacker.pem)
+
+          attempts=0
+          connection_attempts=25
+
+          ## Check for remote connectivity
+          while ! ssh \
+            -i "$identity_file" \
+            -o ConnectTimeout=10 \
+            -o ForwardAgent=yes \
+            -o IdentitiesOnly=yes \
+            -o StrictHostKeyChecking=no \
+            "$(printf "%s@%s" "$user" "$hostname")" \
+            exit
+          do
+            if [ "$attempts" -ge "$connection_attempts" ]; then
+              echo "SSH connection failed after $connection_attempts attempts. Exiting..."
+              exit 1
+            fi
+            printf "SSH connection attempt %d/%d failed. Retrying...\n" "$((attempts++))" "$connection_attempts"
+            ## sleep for Permission denied (publickey) errors
+            sleep 20
+          done
+          echo "SSH connection established after $attempts attempts"
+    
+          # Write the host info so that it can be used by the signing tool
+          if [[ $OSTYPE == "cygwin" ]]; then
+            identity_file=$(cygpath -wa "$identity_file")
+          else
+            identity_file=$(eval echo "$identity_file")
+          fi
+          cat <<EOL > signing_host_info.yml
+          SIGNING_SERVER_HOSTNAME: $hostname
+          SIGNING_SERVER_PRIVATE_KEY: $identity_file
+          SIGNING_SERVER_USERNAME: $user
+          SIGNING_SERVER_PORT: 22
+          EOL
+    # Update the expansions
+    - command: expansions.update
+      params:
+        file: signing_host_info.yml
+
   package:
     - command: shell.exec
       params:
@@ -317,6 +395,10 @@ functions:
           DEBUG: ${debug}
           npm_config_loglevel: ${npm_loglevel}
           COMPASS_DISTRIBUTION: ${compass_distribution}
+          SIGNING_SERVER_HOSTNAME: ${SIGNING_SERVER_HOSTNAME}
+          SIGNING_SERVER_PRIVATE_KEY: ${SIGNING_SERVER_PRIVATE_KEY}
+          SIGNING_SERVER_USERNAME: ${SIGNING_SERVER_USERNAME}
+          SIGNING_SERVER_PORT: ${SIGNING_SERVER_PORT}
         script: |
           set -e
 
@@ -484,6 +566,29 @@ functions:
             tar -xvz)
           export COMPASS_CRYPT_LIBRARY_PATH=$(echo $PWD/mongodb-crypt/lib/mongo_*_v1.*)
           npm run test-csfle --workspace mongodb-data-service
+  
+  verify-artifacts:
+    - command: shell.exec
+      params:
+        working_dir: src
+        shell: bash
+        env:
+          # These are set in the apply-compass-target-expansion func
+          WINDOWS_EXE_NAME: ${windows_setup_filename}
+          WINDOWS_MSI_NAME: ${windows_msi_filename}
+          WINDOWS_ZIP_NAME: ${windows_zip_filename}
+          WINDOWS_NUPKG_NAME: ${windows_nupkg_full_filename}
+          OSX_DMG_NAME: ${osx_dmg_filename}
+          OSX_ZIP_NAME: ${osx_zip_filename}
+          RHEL_RPM_NAME: ${linux_rpm_filename}
+          RHEL_TAR_NAME: ${rhel_tar_filename}
+          LINUX_DEB_NAME: ${linux_deb_filename}
+          LINUX_TAR_NAME: ${linux_tar_filename}
+        script: |
+          set -e
+          # Load environment variables
+          eval $(.evergreen/print-compass-env.sh)
+          .evergreen/verify-artifacts.sh
 
   save-all-artifacts:
     - command: s3.put
@@ -505,12 +610,26 @@ functions:
         remote_file: ${project}/${revision}_${revision_order_id}/${windows_zip_filename}
         content_type: application/zip
         optional: true
+    - command: s3.put
+      params:
+        <<: *save-artifact-params-public
+        local_file: src/packages/compass/dist/${windows_zip_sign_filename}
+        remote_file: ${project}/${revision}_${revision_order_id}/${windows_zip_sign_filename}
+        content_type: application/pgp-signature
+        optional: true
     - command: s3.put
       params:
         <<: *save-artifact-params-public
         local_file: src/packages/compass/dist/${windows_nupkg_full_filename}
         remote_file: ${project}/${revision}_${revision_order_id}/${windows_nupkg_full_filename}
         optional: true
+    - command: s3.put
+      params:
+        <<: *save-artifact-params-public
+        local_file: src/packages/compass/dist/${windows_nupkg_full_sign_filename}
+        remote_file: ${project}/${revision}_${revision_order_id}/${windows_nupkg_full_sign_filename}
+        content_type: application/pgp-signature
+        optional: true
     - command: s3.put
       params:
         <<: *save-artifact-params-public
@@ -531,6 +650,13 @@ functions:
         remote_file: ${project}/${revision}_${revision_order_id}/${osx_zip_filename}
         content_type: application/zip
         optional: true
+    - command: s3.put
+      params:
+        <<: *save-artifact-params-public
+        local_file: src/packages/compass/dist/${osx_zip_sign_filename}
+        remote_file: ${project}/${revision}_${revision_order_id}/${osx_zip_sign_filename}
+        content_type: application/pgp-signature
+        optional: true
     - command: s3.put
       params:
         <<: *save-artifact-params-public
@@ -545,20 +671,41 @@ functions:
         remote_file: ${project}/${revision}_${revision_order_id}/${rhel_tar_filename}
         content_type: application/x-gzip
         optional: true
+    - command: s3.put
+      params:
+        <<: *save-artifact-params-public
+        local_file: src/packages/compass/dist/${rhel_tar_sign_filename}
+        remote_file: ${project}/${revision}_${revision_order_id}/${rhel_tar_sign_filename}
+        content_type: application/pgp-signature
+        optional: true
     - command: s3.put
       params:
         <<: *save-artifact-params-public
         local_file: src/packages/compass/dist/${linux_deb_filename}
         remote_file: ${project}/${revision}_${revision_order_id}/${linux_deb_filename}
         content_type: application/vnd.debian.binary-package
         optional: true
+    - command: s3.put
+      params:
+        <<: *save-artifact-params-public
+        local_file: src/packages/compass/dist/${linux_deb_sign_filename}
+        remote_file: ${project}/${revision}_${revision_order_id}/${linux_deb_sign_filename}
+        content_type: application/pgp-signature
+        optional: true
     - command: s3.put
       params:
         <<: *save-artifact-params-public
         local_file: src/packages/compass/dist/${linux_tar_filename}
         remote_file: ${project}/${revision}_${revision_order_id}/${linux_tar_filename}
         content_type: application/x-gzip
         optional: true
+    - command: s3.put
+      params:
+        <<: *save-artifact-params-public
+        local_file: src/packages/compass/dist/${linux_tar_sign_filename}
+        remote_file: ${project}/${revision}_${revision_order_id}/${linux_tar_sign_filename}
+        content_type: application/pgp-signature
+        optional: true
 
   get-all-artifacts:
     - command: shell.exec

.evergreen/tasks.in.yml

@@ -105,10 +105,12 @@ tasks:
       - func: apply-compass-target-expansion
         vars:
           compass_distribution: <% out(packageTask.vars.compass_distribution) %>
+      - func: spawn-signing-server
       - func: package
         vars:
           debug: 'hadron*,mongo*,compass*,electron*'
           compass_distribution: <% out(packageTask.vars.compass_distribution) %>
+      - func: verify-artifacts
       - func: save-all-artifacts
         vars:
           compass_distribution: <% out(packageTask.vars.compass_distribution) %>

.evergreen/tasks.yml

@@ -105,10 +105,12 @@ tasks:
       - func: apply-compass-target-expansion
         vars:
           compass_distribution: compass
+      - func: spawn-signing-server
       - func: package
         vars:
           debug: 'hadron*,mongo*,compass*,electron*'
           compass_distribution: compass
+      - func: verify-artifacts
       - func: save-all-artifacts
         vars:
           compass_distribution: compass
@@ -124,10 +126,12 @@ tasks:
       - func: apply-compass-target-expansion
         vars:
           compass_distribution: compass-readonly
+      - func: spawn-signing-server
       - func: package
         vars:
           debug: 'hadron*,mongo*,compass*,electron*'
           compass_distribution: compass-readonly
+      - func: verify-artifacts
       - func: save-all-artifacts
         vars:
           compass_distribution: compass-readonly
@@ -143,10 +147,12 @@ tasks:
       - func: apply-compass-target-expansion
         vars:
           compass_distribution: compass-isolated
+      - func: spawn-signing-server
       - func: package
         vars:
           debug: 'hadron*,mongo*,compass*,electron*'
           compass_distribution: compass-isolated
+      - func: verify-artifacts
       - func: save-all-artifacts
         vars:
           compass_distribution: compass-isolated

.evergreen/verify-artifacts.sh

@@ -0,0 +1,87 @@
+#! /usr/bin/env bash
+
+set -e
+
+ARTIFACTS_DIR="packages/compass/dist"
+echo "Verifying artifacts at $ARTIFACTS_DIR"
+ls -l $ARTIFACTS_DIR
+
+# Use tmp directory for all gpg operations
+GPG_HOME=$(mktemp -d)
+TMP_FILE=$(mktemp)
+COMPASS_KEY="https://pgp.mongodb.com/compass.asc"
+
+trap_handler() {
+  local code=$?
+  if [ $code -eq 0 ]; then
+    echo "Verification successful"
+  else
+    echo "Verification failed with exit code $code"
+    cat "$TMP_FILE"
+  fi
+  rm -f "$TMP_FILE"
+  rm -rf "$GPG_HOME"
+  exit $code
+}
+
+trap trap_handler ERR EXIT
+
+verify_using_gpg() {
+  echo "Verifying $1 using gpg"
+  gpg --homedir $GPG_HOME --verify $ARTIFACTS_DIR/$1.sig $ARTIFACTS_DIR/$1 > "$TMP_FILE" 2>&1
+}
+
+verify_using_powershell() {
+  echo "Verifying $1 using powershell"
+  powershell Get-AuthenticodeSignature -FilePath $ARTIFACTS_DIR/$1 > "$TMP_FILE" 2>&1
+}
+
+verify_using_codesign() {
+  echo "Verifying $1 using codesign"
+  codesign -dv --verbose=4 $ARTIFACTS_DIR/$1 > "$TMP_FILE" 2>&1
+}
+
+verify_using_rpm() {
+  # RPM packages are signed using gpg and the signature is embedded in the package.
+  # Here, we need to import the key in `rpm` and then verify the signature.
+  echo "Importing key into rpm"
+  rpm --import $COMPASS_KEY > "$TMP_FILE" 2>&1
+  # Even if the file is not signed, the command below will exit with 0 and output something like: sha1 md5 OK
+  # So we need to check the output of the command to see if the file is signed successfully.
+  echo "Verifying $1 using rpm"
+  output=$(rpm -K $ARTIFACTS_DIR/$1)
+  # Remove the imported key from rpm
+  rpm -e $(rpm -q gpg-pubkey --qf '%{name}-%{version}-%{release}:%{summary}\n' | grep compass | awk -F: '{print $1}')
+  
+  # Check if the output contains the string "pgp md5 OK"
+  if [[ $output != *"pgp md5 OK"* ]]; then
+    echo "File $1 is not signed"
+    exit 1
+  fi
+}
+
+setup_gpg() {
+  echo "Importing Compass public key"
+  curl $COMPASS_KEY | gpg --homedir $GPG_HOME --import > "$TMP_FILE" 2>&1
+}
+
+if [ "$IS_WINDOWS" = true ]; then
+  verify_using_powershell $WINDOWS_EXE_NAME
+  verify_using_powershell $WINDOWS_MSI_NAME
+  echo "Skipping verification for Windows artifacts using gpg: $WINDOWS_ZIP_NAME, $WINDOWS_NUPKG_NAME"
+elif [ "$IS_UBUNTU" = true ]; then
+  setup_gpg
+  verify_using_gpg $LINUX_DEB_NAME
+  verify_using_gpg $LINUX_TAR_NAME
+elif [ "$IS_RHEL" = true ]; then
+  setup_gpg
+  verify_using_rpm $RHEL_RPM_NAME
+  verify_using_gpg $RHEL_TAR_NAME
+elif [ "$IS_OSX" = true ]; then
+  setup_gpg
+  verify_using_gpg $OSX_ZIP_NAME
+  verify_using_codesign $OSX_DMG_NAME
+else
+  echo "Unknown OS, failed to verify file signing"
+  exit 1
+fi

.gitattributes

@@ -1,3 +1,10 @@
 * text=auto eol=lf
+*.gz binary
+*.png binary
+*.jpg binary
+*.gif binary
+*.exe binary
+*.ico binary
+*.icns binary
 /packages/bson-transpilers/lib/**/* linguist-generated=true
 packages/compass-crud/test/fixture-results/* linguist-generated=true