feat(build): add shared-openssl packages to releases MONGOSH-1231 (#1299)

- Move the package matrix definition from the evergreen template
  to a separate file in config/, and add data that would otherwise
  be computed in the build package (e.g. human-readable descriptions)
  there.
- Consistently use “package variant” instead of
  “distribution build variant”. The latter term is a leftover
  from when package artifacts were tightly coupled with evergreen
  build variants, but now these are completely independent
  and it’s probably better to be consistent in referring to
  these as “packages”.
- Make the `packageInformation` property in the build config
  a function that returns the package information for a given
  package variant – this is necessary so that the tasks that
  operate on all packaging variants at once (i.e. the publishing
  tasks) understand that the package information actually varies
  by packaging variant in the case of static OpenSSL/
  shared OpenSSL 1.1/shared OpenSSL 3.
- Merge all .rpm and all .deb package variants into a single
  entity each. Since we started using the crypt_shared library
  (formerly known as CSFLE shared library), our artifacts no longer
  differ between Linux distributions.
  This simplifies picking the correct package from a user perspective
  and reduces the publishing step complexity (i.e. counters the
  increase in the number of package variants coming from the
  shared-openssl additions).
- Adjust build package tests to account for all of the above.

Author: addaleax

.evergreen.yml

@@ -54,97 +54,7 @@ for (const packageInfo of MONGOSH_PACKAGES) {
 
 const ALL_UNIT_TEST_BUILD_VARIANTS = ['darwin_unit', 'linux_unit', 'win32_unit'];
 
-const EXECUTABLE_PKG_INFO = [
-  {
-    executableOsId: 'darwin-x64',
-    compileBuildVariant: 'darwin',
-    distributionBuildVariants: [
-      { name: 'darwin-x64', packageOn: 'darwin', smokeTestKind: 'macos' }
-    ]
-  },
-  {
-    executableOsId: 'darwin-arm64',
-    compileBuildVariant: 'darwin_arm64',
-    distributionBuildVariants: [
-      { name: 'darwin-arm64', packageOn: 'darwin', smokeTestKind: 'macos' }
-    ]
-  },
-  {
-    executableOsId: 'linux-x64',
-    compileBuildVariant: 'linux_x64_build',
-    distributionBuildVariants: [
-      { name: 'linux-x64', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['ubuntu20.04-tgz'] },
-      { name: 'debian-x64', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['ubuntu18.04-deb', 'ubuntu20.04-deb', 'ubuntu22.04-deb', 'debian9-deb', 'debian10-deb', 'debian11-deb'] },
-      { name: 'rhel7-x64', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['centos7-rpm', 'amazonlinux2-rpm'] },
-      { name: 'rhel8-x64', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['rocky8-rpm', 'fedora34-rpm'] },
-      { name: 'suse-x64', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['suse12-rpm', 'suse15-rpm'] },
-      { name: 'amzn1-x64', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['amazonlinux1-rpm'] }
-    ]
-  },
-  {
-    executableOsId: 'linux-x64-openssl11',
-    compileBuildVariant: 'linux_x64_build_openssl11',
-    distributionBuildVariants: [
-      { name: 'linux-x64-openssl11', packageOn: 'linux_package', smokeTestKind: 'none' },
-      { name: 'debian-x64-openssl11', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['ubuntu20.04-deb', 'debian10-deb', 'debian11-deb'] },
-      { name: 'rhel7-x64-openssl11', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['centos7-epel-rpm', 'amazonlinux2-rpm'] },
-      { name: 'rhel8-x64-openssl11', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['rocky8-rpm', 'fedora34-rpm'] }
-    ]
-  },
-  {
-    executableOsId: 'linux-x64-openssl3',
-    compileBuildVariant: 'linux_x64_build_openssl3',
-    distributionBuildVariants: [
-      { name: 'linux-x64-openssl3', packageOn: 'linux_package', smokeTestKind: 'none' },
-      { name: 'debian-x64-openssl3', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['ubuntu22.04-deb'] },
-      { name: 'rhel8-x64-openssl3', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['rocky8-epel-rpm'] }
-    ]
-  },
-  {
-    executableOsId: 'linux-arm64',
-    compileBuildVariant: 'linux_arm64_build',
-    distributionBuildVariants: [
-      { name: 'linux-arm64', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['ubuntu20.04-tgz'] },
-      { name: 'debian-arm64', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['ubuntu18.04-deb', 'ubuntu20.04-deb', 'ubuntu22.04-deb', 'debian10-deb', 'debian11-deb'] },
-      { name: 'rhel8-arm64', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['rocky8-rpm', 'fedora34-rpm'] },
-      { name: 'amzn2-arm64', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['amazonlinux2-rpm'] }
-    ]
-  },
-  {
-    executableOsId: 'linux-arm64-openssl11',
-    compileBuildVariant: 'linux_arm64_build_openssl11',
-    distributionBuildVariants: [
-      { name: 'linux-arm64-openssl11', packageOn: 'linux_package', smokeTestKind: 'none' },
-      { name: 'debian-arm64-openssl11', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['ubuntu20.04-deb', 'debian10-deb', 'debian11-deb'] },
-      { name: 'rhel8-arm64-openssl11', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['rocky8-rpm', 'fedora34-rpm'] },
-      { name: 'amzn2-arm64-openssl11', packageOn: 'linux_package', smokeTestKind: 'docker', smokeTestDockerfiles: ['amazonlinux2-rpm'] }
-    ]
-  },
-  {
-    executableOsId: 'linux-ppc64le',
-    compileBuildVariant: 'linux_ppc64le_build',
-    distributionBuildVariants: [
-      { name: 'linux-ppc64le', packageOn: 'linux_package', smokeTestKind: 'none' },
-      { name: 'rhel8-ppc64le', packageOn: 'linux_package', smokeTestKind: 'rpmextract' }
-    ]
-  },
-  {
-    executableOsId: 'linux-s390x',
-    compileBuildVariant: 'linux_s390x_build',
-    distributionBuildVariants: [
-      { name: 'linux-s390x', packageOn: 'linux_package', smokeTestKind: 'none' },
-      { name: 'rhel7-s390x', packageOn: 'linux_package', smokeTestKind: 'rpmextract' }
-    ]
-  },
-  {
-    executableOsId: 'win32',
-    compileBuildVariant: 'win32_build',
-    distributionBuildVariants: [
-      { name: 'win32-x64', packageOn: 'win32', smokeTestKind: 'ssh' },
-      { name: 'win32msi-x64', packageOn: 'win32', smokeTestKind: 'ssh' }
-    ]
-  }
-];
+const { RELEASE_PACKAGE_MATRIX } = require('../config/release-package-matrix');
 
 %>
 exec_timeout_secs: 3600
@@ -443,11 +353,11 @@ functions:
   #
   # package_and_upload_artifact expects the following arguments:
   # - distro_id
-  # - distribution_build_variant
+  # - package_variant
   # - executable_os_id
   #
   # get_artifact_url expects the following arguments:
-  # - distribution_build_variant
+  # - package_variant
   ###
   package_and_upload_artifact:
     - command: expansions.write
@@ -469,15 +379,15 @@ functions:
         env:
           NODE_JS_VERSION: ${node_js_version}
           DISTRO_ID_OVERRIDE: ${distro_id}
-          DISTRIBUTION_BUILD_VARIANT: ${distribution_build_variant}
+          PACKAGE_VARIANT: ${package_variant}
           MACOS_NOTARY_KEY: ${macos_notary_key}
           MACOS_NOTARY_SECRET: ${macos_notary_secret}
     - command: s3.put
       params:
         aws_key: ${aws_key}
         aws_secret: ${aws_secret}
         local_file: src/artifact-url.txt
-        remote_file: mongosh/binaries/${revision}/${revision_order_id}/artifact-url-${distribution_build_variant}.txt
+        remote_file: mongosh/binaries/${revision}/${revision_order_id}/artifact-url-${package_variant}.txt
         bucket: mciuploads
         permissions: public-read
         content_type: application/x-gzip
@@ -487,7 +397,7 @@ functions:
         aws_key: ${aws_key}
         aws_secret: ${aws_secret}
         local_file: artifact-url.txt
-        remote_file: mongosh/binaries/${revision}/${revision_order_id}/artifact-url-${source_distribution_build_variant}.txt
+        remote_file: mongosh/binaries/${revision}/${revision_order_id}/artifact-url-${source_package_variant}.txt
         bucket: mciuploads
 
   write_preload_script:
@@ -796,7 +706,7 @@ tasks:
   ###
   # E2E TESTS
   ###
-  <% for (const { executableOsId, compileBuildVariant } of EXECUTABLE_PKG_INFO) {
+  <% for (const { executableOsId, compileBuildVariant } of RELEASE_PACKAGE_MATRIX) {
     for (const mVersion of ['stable', 'unstable']) {
       for (const fipsVariant of ['fips', 'nofips']) {
     %>
@@ -824,9 +734,9 @@ tasks:
   ###
   # PACKAGING
   ###
-  <% for (const { executableOsId, compileBuildVariant, distributionBuildVariants } of EXECUTABLE_PKG_INFO) {
-       for (const { name: distributionBuildVariant } of distributionBuildVariants) { %>
-  - name: package_and_upload_artifact_<% out(distributionBuildVariant.replace(/-/g, '_')) %>
+  <% for (const { executableOsId, compileBuildVariant, packages } of RELEASE_PACKAGE_MATRIX) {
+       for (const { name: packageVariant } of packages) { %>
+  - name: package_and_upload_artifact_<% out(packageVariant.replace(/-/g, '_')) %>
     depends_on:
       - name: compile_artifact
         variant: <% out(compileBuildVariant) %>
@@ -839,15 +749,15 @@ tasks:
       - func: package_and_upload_artifact
         vars:
           node_js_version: "<% out(NODE_JS_VERSION_16) %>"
-          distribution_build_variant: <% out(distributionBuildVariant) %>
+          package_variant: <% out(packageVariant) %>
           executable_os_id: <% out(executableOsId) %>
   <% } } %>
 
   ###
   # SMOKE TESTS
   ###
-  <% for (const { distributionBuildVariants } of EXECUTABLE_PKG_INFO) {
-       for (const { name, packageOn, smokeTestKind, smokeTestDockerfiles } of distributionBuildVariants) if (smokeTestKind !== 'none') {
+  <% for (const { packages } of RELEASE_PACKAGE_MATRIX) {
+       for (const { name, packageOn, smokeTestKind, smokeTestDockerfiles } of packages) if (smokeTestKind !== 'none') {
         for (const dockerfile of smokeTestDockerfiles || ['']) { %>
   - name: pkg_test_<% out(`${smokeTestKind}_${name}${dockerfile ? `_${dockerfile}` : ''}`.replace(/[-.]/g, '_')) %>
     tags: ["smoke-test"]
@@ -858,7 +768,7 @@ tasks:
       - func: checkout
       - func: get_artifact_url
         vars:
-          source_distribution_build_variant: <% out(name) %>
+          source_package_variant: <% out(name) %>
       - func: write_preload_script
     <% switch (smokeTestKind) {
       case 'ssh': { %>
@@ -1007,10 +917,10 @@ buildvariants:
     display_name: "Ubuntu 18.04 x64 (Packaging)"
     run_on: ubuntu1804-small
     tasks:
-      <% for (const { executableOsId, distributionBuildVariants } of EXECUTABLE_PKG_INFO) {
-          for (const { name: distributionBuildVariant } of distributionBuildVariants) {
+      <% for (const { executableOsId, packages } of RELEASE_PACKAGE_MATRIX) {
+          for (const { name: packageVariant } of packages) {
             if (executableOsId.startsWith('linux')) { %>
-      - name: package_and_upload_artifact_<% out(distributionBuildVariant.replace(/-/g, '_')) %>
+      - name: package_and_upload_artifact_<% out(packageVariant.replace(/-/g, '_')) %>
       <% } } } %>
   - name: linux_x64_build
     display_name: "RHEL 7.0 x64 (build)"
@@ -1203,19 +1113,19 @@ buildvariants:
     display_name: "package smoke tests (x64 Docker)"
     run_on: ubuntu1804-small
     tasks:
-  <% for (const { distributionBuildVariants, executableOsId } of EXECUTABLE_PKG_INFO) {
+  <% for (const { packages, executableOsId } of RELEASE_PACKAGE_MATRIX) {
       if (executableOsId.includes('linux-x64')) {
-       for (const { name, smokeTestKind, smokeTestDockerfiles } of distributionBuildVariants) if (smokeTestKind !== 'none') {
+       for (const { name, smokeTestKind, smokeTestDockerfiles } of packages) if (smokeTestKind !== 'none') {
         for (const dockerfile of smokeTestDockerfiles || []) { %>
       - name: pkg_test_<% out(`${smokeTestKind}_${name}${dockerfile ? `_${dockerfile}` : ''}`.replace(/[-.]/g, '_')) %>
   <% } } } } %>
   - name: pkg_smoke_tests_docker_arm64
     display_name: "package smoke tests (arm64 Docker)"
     run_on: ubuntu1804-arm64-small
     tasks:
-  <% for (const { distributionBuildVariants, executableOsId } of EXECUTABLE_PKG_INFO) {
+  <% for (const { packages, executableOsId } of RELEASE_PACKAGE_MATRIX) {
       if (executableOsId.includes('linux-arm64')) {
-       for (const { name, smokeTestKind, smokeTestDockerfiles } of distributionBuildVariants) if (smokeTestKind !== 'none') {
+       for (const { name, smokeTestKind, smokeTestDockerfiles } of packages) if (smokeTestKind !== 'none') {
         for (const dockerfile of smokeTestDockerfiles || []) { %>
       - name: pkg_test_<% out(`${smokeTestKind}_${name}${dockerfile ? `_${dockerfile}` : ''}`.replace(/[-.]/g, '_')) %>
   <% } } } } %>
@@ -1244,17 +1154,17 @@ buildvariants:
     display_name: "package smoke tests (RHEL 7.2 s390x)"
     run_on: rhel72-zseries-small
     tasks:
-      - name: pkg_test_rpmextract_rhel7_s390x
+      - name: pkg_test_rpmextract_rpm_s390x
   - name: pkg_smoke_tests_rhel83_s390x
     display_name: "package smoke tests (RHEL 8.3 s390x)"
     run_on: rhel83-zseries-small
     tasks:
-      - name: pkg_test_rpmextract_rhel7_s390x
+      - name: pkg_test_rpmextract_rpm_s390x
   - name: pkg_smoke_tests_rhel81_ppc64le
     display_name: "package smoke tests (RHEL 8.1 ppc64le)"
     run_on: rhel81-power8-small
     tasks:
-      - name: pkg_test_rpmextract_rhel8_ppc64le
+      - name: pkg_test_rpmextract_rpm_ppc64le
 
   - name: draft_publish_release
     display_name: "Draft/Publish Release"

.evergreen/evergreen.yml.in

@@ -17,12 +17,12 @@ if [ "$(uname)" == Linux ]; then
   docker run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD \
     -e EVERGREEN_EXPANSIONS_PATH=/tmp/build/tmp/expansions.yaml \
     -e NODE_JS_VERSION \
-    -e DISTRIBUTION_BUILD_VARIANT \
+    -e PACKAGE_VARIANT \
     -e ARTIFACT_URL_FILE="/tmp/build/artifact-url.txt" \
     --rm -v $PWD:/tmp/build --network host rocky8-package \
     -c 'cd /tmp/build && npm run evergreen-release package && npm run evergreen-release upload'
 else
-  if [[ "$OS" == "Windows_NT" && "$DISTRIBUTION_BUILD_VARIANT" == "win32msi-x64" ]]; then
+  if [[ "$OS" == "Windows_NT" && "$PACKAGE_VARIANT" == "win32msi-x64" ]]; then
     # We have to setup a python venv for the notary client to work
     # in order to sign the MSI
     export PATH="/cygdrive/c/Python27:$PATH"