[main] Update common Docker engineering infrastructure with latest

eng/common/Dockerfile.syft

@@ -0,0 +1,16 @@
+ARG SYFT_IMAGE_NAME
+ARG TARGET_IMAGE_NAME
+
+FROM ${SYFT_IMAGE_NAME} AS syft
+FROM ${TARGET_IMAGE_NAME} AS scan-image
+
+FROM syft AS run-scan
+ARG TARGET_IMAGE_NAME
+ENV SYFT_CHECK_FOR_APP_UPDATE=0 \
+    SYFT_SOURCE_NAME=${TARGET_IMAGE_NAME}
+USER root
+RUN --mount=from=scan-image,source=/,target=/rootfs \
+    ["/syft", "scan", "/rootfs/", "--select-catalogers", "image", "--output", "spdx-json=/manifest.spdx.json"]
+
+FROM scratch AS output
+COPY --from=run-scan /manifest.spdx.json /manifest.spdx.json

eng/common/Pull-Image.ps1

@@ -0,0 +1,18 @@
+#!/usr/bin/env pwsh
+
+[cmdletbinding()]
+param(
+    [Parameter(Mandatory = $true, Position = 0)]
+    [string]$Image,
+
+    [Parameter(Mandatory = $false)]
+    [int]$Retries = 2,
+
+    [Parameter(Mandatory = $false)]
+    [int]$WaitFactor = 6
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+& "$PSScriptRoot/Invoke-WithRetry.ps1" "docker pull $Image" -Retries $Retries -WaitFactor $WaitFactor

eng/common/templates/1es.yml

@@ -0,0 +1,70 @@
+# When extending this template, pipelines using a repository resource containing versions files for image caching must
+# do the following:
+#
+# - Do not rely on any source code from the versions repo so as to not circumvent SDL and CG guidelines
+# - The versions repo resource must be named `VersionsRepo` to avoid SDL scans
+# - The versions repo must be checked out to `$(Build.SourcesDirectory)/versions` to avoid CG scans
+#
+# If the pipeline is not using a separate repository resource, ensure that there is no source code checked out in
+# `$(Build.SourcesDirectory)/versions`, as it will not be scanned.
+
+parameters:
+- name: stages
+  type: stageList
+  default: []
+# List of repositories that will be excluded from SDL scanning. This should
+# only be used when including other repos without building their source code.
+# E.g. for the dotnet/versions repo.
+- name: reposToExcludeFromScanning
+  type: object
+  default: []
+# The pool that will be used for initializing service connections.
+- name: pool
+  type: object
+  default:
+    name: $(default1ESInternalPoolName)
+    image: $(default1ESInternalPoolImage)
+    os: linux
+# The pool that will be used for SDL jobs.
+- name: sourceAnalysisPool
+  type: object
+  default:
+    name: $(defaultSourceAnalysisPoolName)
+    image: $(defaultSourceAnalysisPoolImage)
+    os: windows
+
+resources:
+  repositories:
+  - repository: 1ESPipelineTemplates
+    type: git
+    name: 1ESPipelineTemplates/1ESPipelineTemplates
+    ref: refs/tags/release
+
+extends:
+  template: /eng/common/templates/task-prefix-decorator.yml@self
+  parameters:
+    baseTemplate: v1/1ES.${{ iif(contains(variables['Build.DefinitionName'], '-official'), 'Official', 'Unofficial') }}.PipelineTemplate.yml@1ESPipelineTemplates
+    templateParameters:
+      pool: ${{ parameters.pool }}
+      sdl:
+        # Required for unofficial pipelines because we rely on the ManifestGeneratorTask that is
+        # automatically installed by 1ES pipeline templates
+        sbom:
+          enabled: true
+        binskim:
+          enabled: true
+        componentgovernance:
+          ignoreDirectories: $(Build.SourcesDirectory)/versions
+          showAlertLink: true
+        policheck:
+          enabled: true
+        ${{ if ne(length(parameters.reposToExcludeFromScanning), 0) }}:
+          sourceRepositoriesToScan:
+            exclude:
+            - ${{ each repo in parameters.reposToExcludeFromScanning }}:
+              - repository: ${{ repo }}
+        sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }}
+        tsa:
+          enabled: true
+    stages:
+    - ${{ parameters.stages }}

eng/common/templates/jobs/build-images.yml

@@ -6,10 +6,11 @@ parameters:
   buildJobTimeout: 60
   commonInitStepsForMatrixAndBuild: []
   customInitSteps: []
+  publishConfig: null
   noCache: false
   internalProjectName: null
   publicProjectName: null
-  isInternalServicingValidation: false
+  storageAccountServiceConnection: null
 
 jobs:
 - job: ${{ parameters.name }}
@@ -48,13 +49,13 @@ jobs:
       # to escape the single quotes that are in the string which would need to be done outside the context of PowerShell. Since
       # all we need is for that value to be in a PowerShell variable, we can get that by the fact that AzDO automatically creates
       # the environment variable for us.
-      $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $(imageBuilder.queueArgs) --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json $(commonMatrixAndBuildOptions)"
+      $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $env:IMAGEBUILDER_QUEUEARGS --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json $(commonMatrixAndBuildOptions)"
       if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest") {
-        $imageBuilderBuildArgs = "$imageBuilderBuildArgs --repo-prefix $(stagingRepoPrefix) --push"
+        $imageBuilderBuildArgs = "$imageBuilderBuildArgs --repo-prefix ${{ parameters.publishConfig.buildAcr.repoPrefix }} --push"
       }
 
       # If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache
-      if ("$(pipelineDisabledCache)" -ne "true" -and $env:NOCACHE -ne "true") {
+      if ("$(pipelineDisabledCache)" -ne "true" -and "${{ parameters.noCache }}" -ne "true") {
         $imageBuilderBuildArgs = "$imageBuilderBuildArgs --image-info-source-path $(versionsBasePath)$(imageInfoVersionsPath)"
       }
 
@@ -66,15 +67,17 @@ jobs:
       name: BuildImages
       displayName: Build Images
       serviceConnections:
+      # "name" here refers to the argument name, not the service connection name.
+      # It should probably be changed to "argName".
       - name: acr
-        id: $(build.serviceConnection.id)
-        tenantId: $(build.serviceConnection.tenantId)
-        clientId: $(build.serviceConnection.clientId)
-      - ${{ if eq(parameters.isInternalServicingValidation, true) }}:
+        id: ${{ parameters.publishConfig.buildAcr.serviceConnection.id }}
+        tenantId: ${{ parameters.publishConfig.buildAcr.serviceConnection.tenantId }}
+        clientId: ${{ parameters.publishConfig.buildAcr.serviceConnection.clientId }}
+      - ${{ if parameters.storageAccountServiceConnection }}:
         - name: storage
-          id: $(dotnetstaging.serviceConnection.id)
-          tenantId: $(dotnetstaging.serviceConnection.tenantId)
-          clientId: $(dotnetstaging.serviceConnection.clientId)
+          id: ${{ parameters.storageAccountServiceConnection.id }}
+          tenantId: ${{ parameters.storageAccountServiceConnection.tenantId }}
+          clientId: ${{ parameters.storageAccountServiceConnection.clientId }}
       internalProjectName: ${{ parameters.internalProjectName }}
       dockerClientOS: ${{ parameters.dockerClientOS }}
       args: >-
@@ -86,8 +89,8 @@ jobs:
         --architecture $(architecture)
         --retry
         --digests-out-var 'builtImages'
-        --acr-subscription '$(acr-staging.subscription)'
-        --acr-resource-group '$(acr-staging.resourceGroup)'
+        --acr-subscription '${{ parameters.publishConfig.buildAcr.subscription }}'
+        --acr-resource-group '${{ parameters.publishConfig.buildAcr.resourceGroup }}'
         $(manifestVariables)
         $(imageBuilderBuildArgs)
   - template: /eng/common/templates/steps/publish-artifact.yml@self
@@ -97,63 +100,27 @@ jobs:
       displayName: Publish Image Info File Artifact
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
-  - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:
-    # The following task depends on the SBOM Manifest Generator task installed on the agent.
-    # This task is auto-injected by 1ES Pipeline Templates so we don't need to install it ourselves.
+  - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.dockerClientOS, 'linux')) }}:
     - powershell: |
         $images = "$(BuildImages.builtImages)"
         if (-not $images) { return 0 }
-
-        # There can be leftover versions of the task left on the agent if it's not fresh. So find the latest version.
-        $taskDir = $(Get-ChildItem -Recurse -Directory -Filter "ManifestGeneratorTask*" -Path '$(Agent.WorkFolder)')[-1].FullName
-
-        # There may be multiple version directories within the task directory. Use the latest.
-        $taskVersionDir = $(Get-ChildItem -Directory $taskDir | Sort-Object)[-1].FullName
-
-        $manifestToolDllPath = $(Get-ChildItem -Recurse -File -Filter "Microsoft.ManifestTool.dll" -Path $taskVersionDir).FullName
-
-        # Check whether the manifest task installed its own version of .NET.
-        # To be more robust, we'll handle varying implementations that it's had.
-        # First check for a dotnet folder in the task location
-        $dotnetDir = $(Get-ChildItem -Recurse -Directory -Filter "dotnet-*" -Path $taskVersionDir).FullName
-        if (-not $dotnetDir) {
-          # If it's not there, check in the agent tools location
-          $dotnetDir = $(Get-ChildItem -Recurse -Directory -Filter "*dotnet-*" -Path "$(Agent.ToolsDirectory)").FullName
-        }
-
-        # If the manifest task installed its own version of .NET use that; otherwise it's reusing an existing install of .NET
-        # which is executable by default.
-        if ($dotnetDir) {
-          $dotnetPath = "$dotnetDir/dotnet"
-        }
-        else {
-          $dotnetPath = "dotnet"
-        }
-
-        # Call the manifest tool for each image to produce seperate SBOMs
-        # Manifest tool docs: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/secure-supply-chain/custom-sbom-generation-workflows
+        $syftImageName = "${{ parameters.publishConfig.publicMirrorAcr.server }}/$(imageNames.syft)"
+        & $(engCommonPath)/Pull-Image.ps1 $syftImageName
         $images -Split ',' | ForEach-Object {
           echo "Generating SBOM for $_";
-          $formattedImageName = $_.Replace('$(acr-staging.server)/$(stagingRepoPrefix)', "").Replace('/', '_').Replace(':', '_');
+          $targetImageName = "$_";
+          $formattedImageName = $targetImageName.Replace('${{ parameters.publishConfig.buildAcr.server }}/${{ parameters.publishConfig.buildAcr.repoPrefix }}', "").Replace('/', '_').Replace(':', '_');
           $sbomChildDir = "$(sbomDirectory)/$formattedImageName";
           New-Item -Type Directory -Path $sbomChildDir > $null;
-          & $dotnetPath "$manifestToolDllPath" `
-            Generate `
-            -BuildDropPath '$(Build.ArtifactStagingDirectory)' `
-            -BuildComponentPath '$(Agent.BuildDirectory)' `
-            -PackageName '.NET' `
-            -PackageVersion '$(Build.BuildNumber)' `
-            -ManifestDirPath $sbomChildDir `
-            -DockerImagesToScan $_ `
-            -Verbosity Information
+          docker build --output=$sbomChildDir -f $(engCommonPath)/Dockerfile.syft --build-arg SYFT_IMAGE_NAME=$syftImageName --build-arg TARGET_IMAGE_NAME=$targetImageName -t syft-sbom $(engCommonPath);
         }
       displayName: Generate SBOMs
       condition: and(succeeded(), ne(variables['BuildImages.builtImages'], ''))
   - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
     - template: /eng/common/templates/jobs/${{ format('../steps/test-images-{0}-client.yml', parameters.dockerClientOS) }}@self
       parameters:
         condition: ne(variables.testScriptPath, '')
-  - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:
+  - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.dockerClientOS, 'linux')) }}:
     - template: /eng/common/templates/steps/publish-artifact.yml@self
       parameters:
         path: $(sbomDirectory)

eng/common/templates/jobs/copy-base-images-staging.yml

@@ -5,6 +5,9 @@ parameters:
 - name: pool
   type: object
   default: {}
+- name: publishConfig
+  type: object
+  default: null
 - name: customInitSteps
   type: stepList
   default: []
@@ -22,12 +25,5 @@ jobs:
     pool: ${{ parameters.pool }}
     customInitSteps: ${{ parameters.customInitSteps }}
     additionalOptions: ${{ parameters.additionalOptions }}
-    acr:
-      server: $(acr-staging.server)
-      serviceConnection:
-        tenantId: $(internal-mirror.serviceConnection.tenantId)
-        clientId: $(internal-mirror.serviceConnection.clientId)
-        id: $(internal-mirror.serviceConnection.id)
-      subscription: $(acr-staging.subscription)
-      resourceGroup: $(acr-staging.resourceGroup)
-    repoPrefix: $(mirrorRepoPrefix)
+    acr: ${{ parameters.publishConfig.internalMirrorAcr }}
+    repoPrefix: ${{ parameters.publishConfig.internalMirrorAcr.repoPrefix }}

eng/common/templates/jobs/generate-matrix.yml

@@ -6,6 +6,7 @@ parameters:
   isTestStage: false
   internalProjectName: null
   noCache: false
+  publishConfig: null
   customInitSteps: []
   commonInitStepsForMatrixAndBuild: []
   sourceBuildPipelineRunId: ""
@@ -20,6 +21,7 @@ jobs:
   - ${{ parameters.customInitSteps }}
   - template: /eng/common/templates/steps/validate-branch.yml@self
     parameters:
+      publishConfig: ${{ parameters.publishConfig }}
       internalProjectName: ${{ parameters.internalProjectName }}
   - template: /eng/common/templates/steps/set-image-info-path-var.yml
     parameters:
@@ -36,7 +38,7 @@ jobs:
       if ("${{ parameters.isTestStage}}" -eq "true") {
         $additionalGenerateBuildMatrixOptions = "$additionalGenerateBuildMatrixOptions --image-info $(artifactsPath)/image-info.json"
       }
-      elseif ("$(pipelineDisabledCache)" -ne "true" -and $env:NOCACHE -ne "true" -and "$(trimCachedImagesForMatrix)" -eq "true") {
+      elseif ("$(pipelineDisabledCache)" -ne "true" -and "${{ parameters.noCache }}" -ne "true" -and "$(trimCachedImagesForMatrix)" -eq "true") {
         # If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache
         $additionalGenerateBuildMatrixOptions = "$additionalGenerateBuildMatrixOptions --image-info $(versionsBasePath)$(imageInfoVersionsPath) --trim-cached-images"
       }