CSHARP-5626: Add evergreen script to generate CycloneDX SBOM

Added a bash script to generate a 'build' lifecycle CycloneDX SBOM using
the cyclonedx-dotnet tool. The script installed a fixed version of
cyclonedx-dotnet, runs a versioned dotnet restore, and generates an SBOM
for each of the 4 MongoDB.Driver Nuget packages.

To ensure accurate results, two queries are run against each of the
.csproj files to ensure all development packages are excluded and that
any local <ProjectReference> items are marked in the SBOM as Nuget
packages. Once all 4 SBOMs are created, they are merged into a single
heirarchical SBOM file.

The file is saved as sbom.cdx.json (as opposed to the current sbom.json)
which is the preferred file extention for CycloneDX files. There is not
yet any code to commit the new SBOM to the repo. This is to allow for
evaluation of the new SBOM first without intefering with the current
workflow to upload the current static sbom.json file to Kondukto.

There is also a line added to download-augmented-sbom.sh to copy the
augmented SBOM to ./vex.cdx.json (also not yet committed to repo) after
it has been uploaded to the release artifacts bucket, as this is how we
should be storing the augmented SBOM for public consumption.

Once the SBOM generation process has been approved, I will add commit
code for both of the cdx.json files, remove th sbom.json file and update
download-augmented-sbom.sh to use the new sbom.cdx.json file.

Author: jasonhills-mongodb

evergreen/download-augmented-sbom.sh

@@ -21,4 +21,6 @@ echo "KONDUKTO_TOKEN=$kondukto_token" > ${PWD}/kondukto_credentials.env
 docker run --platform="linux/amd64" --rm -v ${PWD}:/pwd \
   --env-file ${PWD}/kondukto_credentials.env \
   artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:2.0 \
-  augment --repo mongodb/mongo-csharp-driver --branch main --sbom-in /pwd/sbom.json --sbom-out /pwd/${SSDLC_PATH}/augmented-sbom.json
+  augment --repo mongodb/mongo-csharp-driver --branch main --sbom-in /pwd/sbom.json --sbom-out /pwd/${SSDLC_PATH}/augmented-sbom.json
+
+cp /pwd/${SSDLC_PATH}/augmented-sbom.json ./vex.cdx.json

evergreen/evergreen.yml

@@ -98,13 +98,9 @@ functions:
   install-dotnet:
     - command: shell.exec
       params:
-        include_expansions_in_env:
-          - "OS"
-          - "DOTNET_SDK_VERSION"
-          - "FRAMEWORK"
         script: |
           ${PREPARE_SHELL}
-          bash ${PROJECT_DIRECTORY}/evergreen/install-dotnet.sh
+          OS=${OS} DOTNET_SDK_VERSION=${DOTNET_SDK_VERSION} bash ${PROJECT_DIRECTORY}/evergreen/install-dotnet.sh
 
   prepare-resources:
     - command: shell.exec
@@ -280,6 +276,18 @@ functions:
         content_type: application/json
         display_name: augmented-sbom.json
 
+  generate-sbom:
+    - command: shell.exec
+      params:
+        working_dir: "mongo-csharp-driver"
+        env:
+          GITHUB_USER: ${github_user}
+          GITHUB_APIKEY: ${github_apikey}
+          PACKAGE_VERSION: ${PACKAGE_VERSION}
+        script: |
+          ${PREPARE_SHELL}
+          ./evergreen/generate-sbom.sh
+        
   generate-ssdlc-report:
     - command: shell.exec
       params:
@@ -318,10 +326,6 @@ functions:
   bootstrap-mongohoused:
     - command: shell.exec
       params:
-        include_expansions_in_env:
-          - "AWS_ACCESS_KEY_ID"
-          - "AWS_SECRET_ACCESS_KEY"
-          - "AWS_SESSION_TOKEN"
         script: |
           DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/pull-mongohouse-image.sh
     - command: shell.exec
@@ -360,18 +364,6 @@ functions:
           cd ${DRIVERS_TOOLS}/.evergreen
           DRIVERS_TOOLS=${DRIVERS_TOOLS} MONGODB_URI=${MONGODB_URI} bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh stop
 
-  run-unit-tests:
-    - command: shell.exec
-      type: test
-      params:
-        working_dir: mongo-csharp-driver
-        shell: "bash"
-        include_expansions_in_env:
-          - "FRAMEWORK"
-        script: |
-          ${PREPARE_SHELL}
-          bash evergreen/run-unit-tests.sh
-
   run-tests:
     - command: shell.exec
       type: test
@@ -459,10 +451,6 @@ functions:
       params:
         shell: "bash"
         working_dir: mongo-csharp-driver
-        include_expansions_in_env:
-          - "AWS_ACCESS_KEY_ID"
-          - "AWS_SECRET_ACCESS_KEY"
-          - "AWS_SESSION_TOKEN"
         script: |
           . ${DRIVERS_TOOLS}/.evergreen/secrets_handling/setup-secrets.sh drivers/atlas_connect
           . evergreen/run-atlas-connectivity-tests.sh
@@ -500,38 +488,9 @@ functions:
         script: |
           ${PREPARE_SHELL}
           MONGODB_URI="${MONGODB_URI}" ../../evergreen/run-perf-tests.sh
-    - command: shell.exec
+    - command: perf.send
       params:
-        script: |
-          # We use the requester expansion to determine whether the data is from a mainline evergreen run or not
-          if [ "${requester}" == "commit" ]; then
-            is_mainline=true
-          else
-            is_mainline=false
-          fi
-
-          # We parse the username out of the order_id as patches append that in and SPS does not need that information
-          parsed_order_id=$(echo "${revision_order_id}" | awk -F'_' '{print $NF}')
-
-          # Submit the performance data to the SPS endpoint
-          response=$(curl -s -w "\nHTTP_STATUS:%{http_code}" -X 'POST' \
-            "https://performance-monitoring-api.corp.mongodb.com/raw_perf_results/cedar_report?project=${project_id}&version=${version_id}&variant=${build_variant}&order=$parsed_order_id&task_name=${task_name}&task_id=${task_id}&execution=${execution}&mainline=$is_mainline" \
-            -H 'accept: application/json' \
-            -H 'Content-Type: application/json' \
-            -d @mongo-csharp-driver/benchmarks/MongoDB.Driver.Benchmarks/Benchmark.Artifacts/results/evergreen-results.json)
-
-          http_status=$(echo "$response" | grep "HTTP_STATUS" | awk -F':' '{print $2}')
-          response_body=$(echo "$response" | sed '/HTTP_STATUS/d')
-
-          # We want to throw an error if the data was not successfully submitted
-          if [ "$http_status" -ne 200 ]; then
-            echo "Error: Received HTTP status $http_status"
-            echo "Response Body: $response_body"
-            exit 1
-          fi
-
-          echo "Response Body: $response_body"
-          echo "HTTP Status: $http_status"
+        file: mongo-csharp-driver/benchmarks/MongoDB.Driver.Benchmarks/Benchmark.Artifacts/results/evergreen-results.json
 
   assume-ec2-role:
     - command: ec2.assume_role
@@ -1055,36 +1014,6 @@ post:
   - func: cleanup
 
 tasks:
-    - name: unit-tests-net472
-      commands:
-        - command: expansions.update
-          params:
-            updates:
-              - key: 'FRAMEWORK'
-                value: 'net472'
-        - func: install-dotnet
-        - func: run-unit-tests
-
-    - name: unit-tests-netstandard21
-      commands:
-        - command: expansions.update
-          params:
-            updates:
-              - key: 'FRAMEWORK'
-                value: 'netstandard2.1'
-        - func: install-dotnet
-        - func: run-unit-tests
-
-    - name: unit-tests-net60
-      commands:
-        - command: expansions.update
-          params:
-            updates:
-              - key: 'FRAMEWORK'
-                value: 'net6.0'
-        - func: install-dotnet
-        - func: run-unit-tests
-
     - name: test-net472
       commands:
         - func: setup-csfle-secrets
@@ -1266,7 +1195,6 @@ tasks:
 
     - name: atlas-data-lake-test
       commands:
-        - func: assume-ec2-role
         - func: bootstrap-mongohoused
         - func: run-atlas-data-lake-test
 
@@ -1679,7 +1607,7 @@ tasks:
     - name: generate-release-notes
       commands:
         - func: generate-release-notes
-
+       
     - name: validate-apidocs
       commands:
         - func: install-dotnet
@@ -2052,14 +1980,9 @@ task_groups:
     setup_group:
       - func: fetch-source
       - func: prepare-resources
-      - func: assume-ec2-role
       - command: subprocess.exec
         params:
           binary: bash
-          include_expansions_in_env:
-            - "AWS_ACCESS_KEY_ID"
-            - "AWS_SECRET_ACCESS_KEY"
-            - "AWS_SESSION_TOKEN"
           env:
             LAMBDA_STACK_NAME: dbx-csharp-lambda
           args:
@@ -2199,10 +2122,6 @@ task_groups:
           binary: bash
           env:
             VAULT_NAME: ${VAULT_NAME}
-          include_expansions_in_env:
-            - "AWS_ACCESS_KEY_ID"
-            - "AWS_SECRET_ACCESS_KEY"
-            - "AWS_SESSION_TOKEN"
           args:
             - ${DRIVERS_TOOLS}/.evergreen/serverless/create-instance.sh
       - command: expansions.update
@@ -2243,42 +2162,6 @@ task_groups:
       - validate-apicompat
 
 buildvariants:
-- name: unit-tests-windows
-  display_name: Unit Tests on Windows
-  run_on: windows-64-vs2017-test
-  expansions:
-    OS: "windows-64"
-  tasks:
-    - name: unit-tests-net472
-    - name: unit-tests-netstandard21
-    - name: unit-tests-net60
-
-- name: unit-tests-ubuntu
-  display_name: Unit Tests on Ubuntu
-  run_on: ubuntu2004-small
-  expansions:
-    OS: "ubuntu-2004"
-  tasks:
-    - name: unit-tests-netstandard21
-    - name: unit-tests-net60
-
-- name: unit-tests-macos
-  display_name: Unit Tests on MacOs
-  run_on: macos-14
-  expansions:
-    OS: "macos-14"
-  tasks:
-    - name: unit-tests-netstandard21
-    - name: unit-tests-net60
-
-- name: unit-tests-macos-arm
-  display_name: Unit Tests on MacOs Arm
-  run_on: macos-14-arm64
-  expansions:
-    OS: "macos-14-arm64"
-  tasks:
-    - name: unit-tests-net60
-
 - matrix_name: stable-api-tests
   matrix_spec: { version: ["5.0", "6.0", "7.0", "8.0", "rapid", "latest"], topology: "standalone", auth: "auth", ssl: "nossl", os: "windows-64" }
   display_name: "Stable API ${version} ${topology} ${auth} ${ssl} ${os}"

evergreen/generate-sbom.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+set -o errexit # Exit the script with error if any of the commands fail
+
+# Accommodate Git Bash or MSYS2 on Windows
+export MSYS_NO_PATHCONV=1
+
+echo -e "\n************************************************"
+
+# Get Packages Version if variable is empty
+if [[ -z "$PACKAGE_VERSION" ]]; then
+  PACKAGE_VERSION=$(bash ./evergreen/get-version.sh)
+fi
+
+echo "Package Version: ${PACKAGE_VERSION}"
+
+# Get array of Package Names
+source ./evergreen/packages.sh
+echo "Packages: ${PACKAGES[*]}"
+
+# Run a restore that will set the package version in the Directory.Build.props file, otherwise it is set to "0.0.0-local"
+# This will also cause the Choose...When conditions in the .csproj files to use PackageReference instead of ProjectReference
+echo "Restoring solution with version set to ${PACKAGE_VERSION}"
+dotnet restore /p:Version=${PACKAGE_VERSION}
+
+# Install cyclonedx-dotnet using latest version tested with this script
+echo "Installing cyclonedx-dotnet"
+dotnet tool install --global CycloneDX --version 5.3.1 --allow-downgrade
+
+echo -e "\nGenerating SBOMs"
+echo "************************************************"
+
+# Track SBOM file paths for merging
+SBOM_FILES=""
+
+for package in ${PACKAGES[*]}; do
+  echo -e "\n+++++++++++++++++++++++++++++++++++++"
+  echo "Processing: ${package}"
+
+  SBOM_FILE="sbom.${package}.cdx.json"
+  SBOM_FILES="${SBOM_FILES} /pwd/${SBOM_FILE}"
+
+  echo "SBOM file: ${SBOM_FILE}"
+
+  # There are nuances to how cyclonedx-dotnet handles <PackageReference> items in Directory.Build.props that lead to private packages being included in SBOM
+  # results even when PrivateAssets is set to "All". As a safeguard, this command lists the PackageReferences and adds the references with PrivateAssets="All"
+  # to an exclusion filter variable to be fed into cyclonedx-dotnet
+  EXCLUDE_FILTER=$(dotnet msbuild ./src/${package}/${package}.csproj -getItem:PackageReference | jq -r '[.Items.PackageReference[] | select(.PrivateAssets != null) | select(.PrivateAssets | test ("All"; "i")) | .Identity + "@" + .Version] | join(",")')
+  echo "Excluded Private Package References: ${EXCLUDE_FILTER}"
+
+  # The ProjectReference items do not resolve as the Nuget packages they represent. This causes duplicate components when the SBOMs are merged. To address this
+  # we add the Nuget PURL to the JSON. This command lists the ProjectReferences for processing.
+  PURL_PATCHES=$(dotnet msbuild ./src/${package}/${package}.csproj -getItem:ProjectReference | jq -r '[.Items.ProjectReference[] | .Filename] | join(",")')
+  echo "Project References requiring added PURL: ${PURL_PATCHES}"
+
+  echo "+++++++++++++++++++++++++++++++++++++"
+
+  ## Run cyclonedx-dotnet
+  # Attempt GitHub license resolution only if GITHUB_USER and GITHUB_APIKEY are both set
+  if [[ -v GITHUB_USER && -v GITHUB_APIKEY ]]; then
+    github_options=(--enable-github-licenses --github-username ${GITHUB_USER} --github-token ${GITHUB_APIKEY})
+  fi
+
+  echo "dotnet-CycloneDX src/${package}/${package}.csproj --disable-package-restore --set-type library --set-nuget-purl --exclude-dev --include-project-references --set-name ${package} --set-version ${PACKAGE_VERSION} --filename ${SBOM_FILE} --exclude-filter ${EXCLUDE_FILTER} ${github_options[@]}"
+  dotnet-CycloneDX src/${package}/${package}.csproj \
+    --disable-package-restore --set-type library --set-nuget-purl --exclude-dev --include-project-references \
+    --set-name ${package} --set-version ${PACKAGE_VERSION} --filename ${SBOM_FILE} \
+    --exclude-filter "${EXCLUDE_FILTER}" \
+    "${github_options[@]}"
+
+  # Patch JSON file with PURLs, as needed
+  for patch in $PURL_PATCHES; do
+    echo "Patching ${patch} with Nuget PURL"
+    contents=$(jq --arg package "$patch" --arg version "$PACKAGE_VERSION" '.components |= map(if [.name | startswith("MongoDB.")] and has("purl") == false then .purl = "pkg:nuget/\($package)@\($version)" else . end)' ${SBOM_FILE})
+    echo -E "${contents}" >${SBOM_FILE}
+  done
+
+done
+
+echo -e "\n================================="
+echo "Merging SBOMs using cyclonedx-cli"
+echo "================================="
+
+# Use cyclonedx-cli to merge the SBOMs into 1 hierarchical SBOM
+docker run --platform="linux/amd64" --rm -v ${PWD}:/pwd \
+  cyclonedx/cyclonedx-cli:0.28.2 \
+  merge --input-files ${SBOM_FILES} --output-file /pwd/sbom.cdx.json \
+  --hierarchical --group mongodb --name mongo-csharp-driver --version ${PACKAGE_VERSION}