[serverless] Upload serverless assets to Azure so we can release for a layer (#6561)

## Summary of changes

Compiles the serverless assets to an artifact, uploads them, and uses
those when building the layer for a release

## Reason for change

The serverless layer has been unable to build for a while for releases.
The SSI artifacts (which similarly build in gitlab) pull the assets from
the GitHub release page, but we're hesitant to add the serverless assets
there, because of the risk of confusion for customers.

As a workaround, we upload the assets to a public blob store that can be
pulled from later. Various other workflows already use this store, so
this is just adding an additional asset, and sidesteps the issue.

## Implementation details

- Remove the "aggregation" logic out of GitLab (which chooses _which_
artifacts are required)
- Add a new stage to AzDo that does the aggregation logic for the
required artifacts.
- Upload the artifact as a zip file to Azure
- In gitlab, pull either directly from azdo (for build branches) or from
Azure (for tag branches)

## Test coverage

- [Tested the aggregation and upload
here](https://dev.azure.com/datadoghq/dd-trace-dotnet/_build/results?buildId=171807&view=results)
- [Tested the pulling from gitlab
here](https://gitlab.ddbuild.io/DataDog/apm-reliability/dd-trace-dotnet/-/pipelines/53167864)
(using a temporary hack to force calling the tag-related code)

## Other details

Supersedes 
- #6543

Author: andrewlock

.azure-pipelines/ultimate-pipeline.yml

@@ -4257,9 +4257,49 @@ stages:
           displayName: Publish ssi artifacts
           artifact: ssi-artifacts
 
+- stage: store_serverless_artifacts
+  condition: and(succeeded(), eq(variables['isBenchmarksOnlyBuild'], 'False'), eq(variables.isMainRepository, true))
+  dependsOn: [build_linux_tracer_r2r, build_arm64_tracer_r2r, build_linux_universal, build_arm64_universal]
+  jobs:
+    - job: combine
+      timeoutInMinutes: 60 #default value
+      pool:
+        vmImage: ubuntu-latest
+      steps:
+        - checkout: none
+        # Download the files to the expected locations
+        - task: DownloadPipelineArtifact@2
+          displayName: Download serverless tracer home x64
+          inputs:
+            artifact: linux-tracer-home-linux-x64-r2r
+            path: $(Build.ArtifactStagingDirectory)/x64
+
+        - task: DownloadPipelineArtifact@2
+          displayName: Download serverless universal home x64
+          inputs:
+            artifact: linux-universal-home-linux-x64
+            path: $(Build.ArtifactStagingDirectory)/x64
+
+        - task: DownloadPipelineArtifact@2
+          displayName: Download serverless tracer home arm64
+          inputs:
+            artifact: linux-tracer-home-linux-arm64-r2r
+            path: $(Build.ArtifactStagingDirectory)/arm64
+
+        - task: DownloadPipelineArtifact@2
+          displayName: Download serverless universal home arm64
+          inputs:
+            artifact: linux-universal-home-linux-arm64
+            path: $(Build.ArtifactStagingDirectory)/arm64
+
+        # publish all tar files as single artifact
+        - publish: "$(Build.ArtifactStagingDirectory)"
+          displayName: Publish serverless artifacts
+          artifact: serverless-artifacts
+
 - stage: upload_to_azure
   condition: and(succeeded(), eq(variables['isBenchmarksOnlyBuild'], 'False'), eq(variables.isMainRepository, true))
-  dependsOn: [package_windows, package_linux, package_arm64, dotnet_tool, merge_commit_id]
+  dependsOn: [package_windows, package_linux, package_arm64, dotnet_tool, merge_commit_id, store_serverless_artifacts]
   variables:
     targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']]
     targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']]
@@ -4485,6 +4525,19 @@ stages:
             artifact: windows-msi-x64
             path: $(Build.ArtifactStagingDirectory)
 
+        # Download the artifacts required by the serverless layer and create the zip file
+        # This exapands the directory - but we want a zip file, so re-zip it!
+        - task: DownloadPipelineArtifact@2
+          displayName: Download serverless artifacts
+          inputs:
+            artifact: serverless-artifacts
+            path: $(Agent.TempDirectory)/serverless-artifacts
+
+        - bash: |
+            cd $(Agent.TempDirectory)/serverless-artifacts
+            zip -r $(Build.ArtifactStagingDirectory)/serverless-artifacts.zip .
+          displayName: Zip serverless artifacts and store in staging directory
+
         - bash: |
             # Write tracer version number to version.txt
             echo "$(tracer_version)" > $(Build.ArtifactStagingDirectory)/version.txt

.gitlab/download-serverless-artifacts.sh

@@ -10,7 +10,22 @@ set -eo pipefail
 target_dir=artifacts
 mkdir -p $target_dir
 
+if [ -n "$CI_COMMIT_TAG" ] && [ -n "$CI_COMMIT_SHA" ]; then
+  echo "Downloading artifacts from Azure"
+  curl --location --fail \
+    --output $target_dir/serverless-artifacts.zip \
+    "https://apmdotnetci.blob.core.windows.net/apm-dotnet-ci-artifacts-master/${CI_COMMIT_SHA}/serverless-artifacts.zip"
+
+  # Extract top level artifact
+  unzip $target_dir/serverless-artifacts.zip -d $target_dir/
+  rm -f $target_dir/serverless-artifacts.zip
+
+  ls -l $target_dir
+  exit 0
+fi
+
 branchName="refs/heads/$CI_COMMIT_BRANCH"
+artifactName="serverless-artifacts"
 
 echo "Looking for azure devops PR builds for branch '$branchName' for commit '$CI_COMMIT_SHA' to start"
 
@@ -32,41 +47,31 @@ fi
 
 echo "Found build with id '$buildId' for commit '$CI_COMMIT_SHA' on branch '$branchName'"
 
-architectures=("x64" "arm64")
-for architecture in "${architectures[@]}"; do
-  echo "Looking for artifacts for architecture '$architecture'"
-
-  artifacts=("linux-tracer-home-linux-$architecture-r2r" "linux-universal-home-linux-$architecture")
-
-  # Now try to download the artifacts from the build
-  for artifactName in "${artifacts[@]}"; do
-    artifactsUrl="https://dev.azure.com/datadoghq/dd-trace-dotnet/_apis/build/builds/$buildId/artifacts?api-version=7.1&artifactName=$artifactName"
-
-    # Keep trying to get the artifact for 30 minutes
-    downloadUrl=""
-    TIMEOUT=1800
-    STARTED=0
-    until (( STARTED == TIMEOUT )) || [ ! -z "${downloadUrl}" ] ; do
-        echo "Checking for '$artifactName' at '$artifactsUrl'..."
-        # If the artifact doesn't exist, .resource.downloadUrl will be null, so we filter that out
-        downloadUrl=$(curl -s $artifactsUrl | jq -r '.resource.downloadUrl | select( . != null )')
-        sleep 100
-        (( STARTED += 100 ))
-    done
-    (( STARTED < TIMEOUT ))
-
-    if [ -z "${downloadUrl}" ]; then
-      echo "No downloadUrl found after 30 minutes for commit '$CI_COMMIT_SHA' on branch '$branchName'"
-      exit 1
-    fi
-
-    echo "Downloading '$artifactName' from '$downloadUrl'..."
-    curl -o $target_dir/artifacts.zip "$downloadUrl"
-    unzip $target_dir/artifacts.zip -d $target_dir/$architecture
-    mv $target_dir/$architecture/$artifactName/* $target_dir/$architecture
-    rm -rf $target_dir/artifacts.zip
-    rmdir $target_dir/$architecture/$artifactName
-  done
+# Now try to download the artifacts from the build
+artifactsUrl="https://dev.azure.com/datadoghq/dd-trace-dotnet/_apis/build/builds/$buildId/artifacts?api-version=7.1&artifactName=$artifactName"
+
+# Keep trying to get the artifact for 40 minutes
+TIMEOUT=2400
+STARTED=0
+until (( STARTED == TIMEOUT )) || [ ! -z "${downloadUrl}" ] ; do
+    echo "Checking for artifacts at '$artifactsUrl'..."
+    # If the artifact doesn't exist, .resource.downloadUrl will be null, so we filter that out
+    downloadUrl=$(curl -s $artifactsUrl | jq -r '.resource.downloadUrl | select( . != null )')
+    sleep 100
+    (( STARTED += 100 ))
 done
+(( STARTED < TIMEOUT ))
+
+if [ -z "${downloadUrl}" ]; then
+  echo "No downloadUrl found after 30 minutes for commit '$CI_COMMIT_SHA' on branch '$branchName'"
+  exit 1
+fi
+
+echo "Downloading artifacts from '$downloadUrl'..."
+curl -o $target_dir/artifacts.zip "$downloadUrl"
+unzip $target_dir/artifacts.zip -d $target_dir
+mv $target_dir/$artifactName/* $target_dir
+rm -rf $target_dir/artifacts.zip
+rmdir $target_dir/$artifactName
 
 ls -l $target_dir