From 778bcb33417eae5c87d36db6edd167b7ee42fe6b Mon Sep 17 00:00:00 2001 From: dotnet-docker-bot <60522487+dotnet-docker-bot@users.noreply.github.com> Date: Fri, 5 Sep 2025 07:47:36 -0700 Subject: [PATCH 1/6] Update common Docker engineering infrastructure with latest --- eng/common/Dockerfile.syft | 16 ++++ eng/common/Pull-Image.ps1 | 18 ++++ eng/common/templates/1es.yml | 70 ++++++++++++++ eng/common/templates/jobs/build-images.yml | 79 +++++----------- .../jobs/copy-base-images-staging.yml | 14 +-- eng/common/templates/jobs/generate-matrix.yml | 4 +- eng/common/templates/jobs/publish.yml | 48 +++++----- .../jobs/test-images-linux-client.yml | 2 + .../jobs/test-images-windows-client.yml | 2 + .../templates/stages/build-and-test.yml | 63 +++++++++---- .../stages/dotnet/build-and-test.yml | 12 ++- .../stages/dotnet/build-test-publish-repo.yml | 9 +- .../stages/dotnet/publish-config-nonprod.yml | 91 +++++++++++++++++++ .../stages/dotnet/publish-config-prod.yml | 91 +++++++++++++++++++ .../templates/stages/dotnet/publish.yml | 8 +- eng/common/templates/stages/publish.yml | 85 ++++++++--------- .../stages/setup-service-connections.yml | 4 + .../templates/steps/annotate-eol-digests.yml | 17 ++-- .../templates/steps/init-docker-linux.yml | 4 +- .../steps/init-matrix-build-publish.yml | 5 +- .../templates/steps/publish-readmes.yml | 2 +- eng/common/templates/steps/set-dry-run.yml | 27 +++++- .../steps/test-images-linux-client.yml | 9 +- .../steps/test-images-windows-client.yml | 11 ++- .../templates/steps/validate-branch.yml | 17 +++- .../steps/wait-for-mcr-image-ingestion.yml | 21 ++++- .../templates/task-prefix-decorator.yml | 4 +- eng/common/templates/variables/common.yml | 17 ---- .../templates/variables/docker-images.yml | 3 +- .../variables/dotnet/build-test-publish.yml | 11 +-- .../templates/variables/dotnet/common.yml | 10 -- .../variables/dotnet/secrets-unofficial.yml | 5 + 32 files changed, 547 insertions(+), 232 deletions(-) create mode 100644 eng/common/Dockerfile.syft create mode 100644 eng/common/Pull-Image.ps1 create mode 100644 eng/common/templates/1es.yml create mode 100644 eng/common/templates/stages/dotnet/publish-config-nonprod.yml create mode 100644 eng/common/templates/stages/dotnet/publish-config-prod.yml create mode 100644 eng/common/templates/variables/dotnet/secrets-unofficial.yml diff --git a/eng/common/Dockerfile.syft b/eng/common/Dockerfile.syft new file mode 100644 index 00000000..2e564e2a --- /dev/null +++ b/eng/common/Dockerfile.syft @@ -0,0 +1,16 @@ +ARG SYFT_IMAGE_NAME +ARG TARGET_IMAGE_NAME + +FROM ${SYFT_IMAGE_NAME} AS syft +FROM ${TARGET_IMAGE_NAME} AS scan-image + +FROM syft AS run-scan +ARG TARGET_IMAGE_NAME +ENV SYFT_CHECK_FOR_APP_UPDATE=0 \ + SYFT_SOURCE_NAME=${TARGET_IMAGE_NAME} +USER root +RUN --mount=from=scan-image,source=/,target=/rootfs \ + ["/syft", "scan", "/rootfs/", "--select-catalogers", "image", "--output", "spdx-json=/manifest.spdx.json"] + +FROM scratch AS output +COPY --from=run-scan /manifest.spdx.json /manifest.spdx.json diff --git a/eng/common/Pull-Image.ps1 b/eng/common/Pull-Image.ps1 new file mode 100644 index 00000000..2d0a82cf --- /dev/null +++ b/eng/common/Pull-Image.ps1 @@ -0,0 +1,18 @@ +#!/usr/bin/env pwsh + +[cmdletbinding()] +param( + [Parameter(Mandatory = $true, Position = 0)] + [string]$Image, + + [Parameter(Mandatory = $false)] + [int]$Retries = 2, + + [Parameter(Mandatory = $false)] + [int]$WaitFactor = 6 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +& "$PSScriptRoot/Invoke-WithRetry.ps1" "docker pull $Image" -Retries $Retries -WaitFactor $WaitFactor diff --git a/eng/common/templates/1es.yml b/eng/common/templates/1es.yml new file mode 100644 index 00000000..19f8c1f1 --- /dev/null +++ b/eng/common/templates/1es.yml @@ -0,0 +1,70 @@ +# When extending this template, pipelines using a repository resource containing versions files for image caching must +# do the following: +# +# - Do not rely on any source code from the versions repo so as to not circumvent SDL and CG guidelines +# - The versions repo resource must be named `VersionsRepo` to avoid SDL scans +# - The versions repo must be checked out to `$(Build.SourcesDirectory)/versions` to avoid CG scans +# +# If the pipeline is not using a separate repository resource, ensure that there is no source code checked out in +# `$(Build.SourcesDirectory)/versions`, as it will not be scanned. + +parameters: +- name: stages + type: stageList + default: [] +# List of repositories that will be excluded from SDL scanning. This should +# only be used when including other repos without building their source code. +# E.g. for the dotnet/versions repo. +- name: reposToExcludeFromScanning + type: object + default: [] +# The pool that will be used for initializing service connections. +- name: pool + type: object + default: + name: $(default1ESInternalPoolName) + image: $(default1ESInternalPoolImage) + os: linux +# The pool that will be used for SDL jobs. +- name: sourceAnalysisPool + type: object + default: + name: $(defaultSourceAnalysisPoolName) + image: $(defaultSourceAnalysisPoolImage) + os: windows + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: /eng/common/templates/task-prefix-decorator.yml@self + parameters: + baseTemplate: v1/1ES.${{ iif(contains(variables['Build.DefinitionName'], '-official'), 'Official', 'Unofficial') }}.PipelineTemplate.yml@1ESPipelineTemplates + templateParameters: + pool: ${{ parameters.pool }} + sdl: + # Required for unofficial pipelines because we rely on the ManifestGeneratorTask that is + # automatically installed by 1ES pipeline templates + sbom: + enabled: true + binskim: + enabled: true + componentgovernance: + ignoreDirectories: $(Build.SourcesDirectory)/versions + showAlertLink: true + policheck: + enabled: true + ${{ if ne(length(parameters.reposToExcludeFromScanning), 0) }}: + sourceRepositoriesToScan: + exclude: + - ${{ each repo in parameters.reposToExcludeFromScanning }}: + - repository: ${{ repo }} + sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }} + tsa: + enabled: true + stages: + - ${{ parameters.stages }} diff --git a/eng/common/templates/jobs/build-images.yml b/eng/common/templates/jobs/build-images.yml index 420120d2..24fa3e79 100644 --- a/eng/common/templates/jobs/build-images.yml +++ b/eng/common/templates/jobs/build-images.yml @@ -6,10 +6,11 @@ parameters: buildJobTimeout: 60 commonInitStepsForMatrixAndBuild: [] customInitSteps: [] + publishConfig: null noCache: false internalProjectName: null publicProjectName: null - isInternalServicingValidation: false + storageAccountServiceConnection: null jobs: - job: ${{ parameters.name }} @@ -48,13 +49,13 @@ jobs: # to escape the single quotes that are in the string which would need to be done outside the context of PowerShell. Since # all we need is for that value to be in a PowerShell variable, we can get that by the fact that AzDO automatically creates # the environment variable for us. - $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $(imageBuilder.queueArgs) --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json $(commonMatrixAndBuildOptions)" + $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $env:IMAGEBUILDER_QUEUEARGS --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json $(commonMatrixAndBuildOptions)" if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest") { - $imageBuilderBuildArgs = "$imageBuilderBuildArgs --repo-prefix $(stagingRepoPrefix) --push" + $imageBuilderBuildArgs = "$imageBuilderBuildArgs --repo-prefix ${{ parameters.publishConfig.buildAcr.repoPrefix }} --push" } # If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache - if ("$(pipelineDisabledCache)" -ne "true" -and $env:NOCACHE -ne "true") { + if ("$(pipelineDisabledCache)" -ne "true" -and "${{ parameters.noCache }}" -ne "true") { $imageBuilderBuildArgs = "$imageBuilderBuildArgs --image-info-source-path $(versionsBasePath)$(imageInfoVersionsPath)" } @@ -66,15 +67,17 @@ jobs: name: BuildImages displayName: Build Images serviceConnections: + # "name" here refers to the argument name, not the service connection name. + # It should probably be changed to "argName". - name: acr - id: $(build.serviceConnection.id) - tenantId: $(build.serviceConnection.tenantId) - clientId: $(build.serviceConnection.clientId) - - ${{ if eq(parameters.isInternalServicingValidation, true) }}: + id: ${{ parameters.publishConfig.buildAcr.serviceConnection.id }} + tenantId: ${{ parameters.publishConfig.buildAcr.serviceConnection.tenantId }} + clientId: ${{ parameters.publishConfig.buildAcr.serviceConnection.clientId }} + - ${{ if parameters.storageAccountServiceConnection }}: - name: storage - id: $(dotnetstaging.serviceConnection.id) - tenantId: $(dotnetstaging.serviceConnection.tenantId) - clientId: $(dotnetstaging.serviceConnection.clientId) + id: ${{ parameters.storageAccountServiceConnection.id }} + tenantId: ${{ parameters.storageAccountServiceConnection.tenantId }} + clientId: ${{ parameters.storageAccountServiceConnection.clientId }} internalProjectName: ${{ parameters.internalProjectName }} dockerClientOS: ${{ parameters.dockerClientOS }} args: >- @@ -86,8 +89,8 @@ jobs: --architecture $(architecture) --retry --digests-out-var 'builtImages' - --acr-subscription '$(acr-staging.subscription)' - --acr-resource-group '$(acr-staging.resourceGroup)' + --acr-subscription '${{ parameters.publishConfig.buildAcr.subscription }}' + --acr-resource-group '${{ parameters.publishConfig.buildAcr.resourceGroup }}' $(manifestVariables) $(imageBuilderBuildArgs) - template: /eng/common/templates/steps/publish-artifact.yml@self @@ -97,55 +100,19 @@ jobs: displayName: Publish Image Info File Artifact internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} - - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - # The following task depends on the SBOM Manifest Generator task installed on the agent. - # This task is auto-injected by 1ES Pipeline Templates so we don't need to install it ourselves. + - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.dockerClientOS, 'linux')) }}: - powershell: | $images = "$(BuildImages.builtImages)" if (-not $images) { return 0 } - - # There can be leftover versions of the task left on the agent if it's not fresh. So find the latest version. - $taskDir = $(Get-ChildItem -Recurse -Directory -Filter "ManifestGeneratorTask*" -Path '$(Agent.WorkFolder)')[-1].FullName - - # There may be multiple version directories within the task directory. Use the latest. - $taskVersionDir = $(Get-ChildItem -Directory $taskDir | Sort-Object)[-1].FullName - - $manifestToolDllPath = $(Get-ChildItem -Recurse -File -Filter "Microsoft.ManifestTool.dll" -Path $taskVersionDir).FullName - - # Check whether the manifest task installed its own version of .NET. - # To be more robust, we'll handle varying implementations that it's had. - # First check for a dotnet folder in the task location - $dotnetDir = $(Get-ChildItem -Recurse -Directory -Filter "dotnet-*" -Path $taskVersionDir).FullName - if (-not $dotnetDir) { - # If it's not there, check in the agent tools location - $dotnetDir = $(Get-ChildItem -Recurse -Directory -Filter "*dotnet-*" -Path "$(Agent.ToolsDirectory)").FullName - } - - # If the manifest task installed its own version of .NET use that; otherwise it's reusing an existing install of .NET - # which is executable by default. - if ($dotnetDir) { - $dotnetPath = "$dotnetDir/dotnet" - } - else { - $dotnetPath = "dotnet" - } - - # Call the manifest tool for each image to produce seperate SBOMs - # Manifest tool docs: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/secure-supply-chain/custom-sbom-generation-workflows + $syftImageName = "${{ parameters.publishConfig.publicMirrorAcr.server }}/$(imageNames.syft)" + & $(engCommonPath)/Pull-Image.ps1 $syftImageName $images -Split ',' | ForEach-Object { echo "Generating SBOM for $_"; - $formattedImageName = $_.Replace('$(acr-staging.server)/$(stagingRepoPrefix)', "").Replace('/', '_').Replace(':', '_'); + $targetImageName = "$_"; + $formattedImageName = $targetImageName.Replace('${{ parameters.publishConfig.buildAcr.server }}/${{ parameters.publishConfig.buildAcr.repoPrefix }}', "").Replace('/', '_').Replace(':', '_'); $sbomChildDir = "$(sbomDirectory)/$formattedImageName"; New-Item -Type Directory -Path $sbomChildDir > $null; - & $dotnetPath "$manifestToolDllPath" ` - Generate ` - -BuildDropPath '$(Build.ArtifactStagingDirectory)' ` - -BuildComponentPath '$(Agent.BuildDirectory)' ` - -PackageName '.NET' ` - -PackageVersion '$(Build.BuildNumber)' ` - -ManifestDirPath $sbomChildDir ` - -DockerImagesToScan $_ ` - -Verbosity Information + docker build --output=$sbomChildDir -f $(engCommonPath)/Dockerfile.syft --build-arg SYFT_IMAGE_NAME=$syftImageName --build-arg TARGET_IMAGE_NAME=$targetImageName -t syft-sbom $(engCommonPath); } displayName: Generate SBOMs condition: and(succeeded(), ne(variables['BuildImages.builtImages'], '')) @@ -153,7 +120,7 @@ jobs: - template: /eng/common/templates/jobs/${{ format('../steps/test-images-{0}-client.yml', parameters.dockerClientOS) }}@self parameters: condition: ne(variables.testScriptPath, '') - - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.dockerClientOS, 'linux')) }}: - template: /eng/common/templates/steps/publish-artifact.yml@self parameters: path: $(sbomDirectory) diff --git a/eng/common/templates/jobs/copy-base-images-staging.yml b/eng/common/templates/jobs/copy-base-images-staging.yml index 71ddb4a5..17f4d359 100644 --- a/eng/common/templates/jobs/copy-base-images-staging.yml +++ b/eng/common/templates/jobs/copy-base-images-staging.yml @@ -5,6 +5,9 @@ parameters: - name: pool type: object default: {} +- name: publishConfig + type: object + default: null - name: customInitSteps type: stepList default: [] @@ -22,12 +25,5 @@ jobs: pool: ${{ parameters.pool }} customInitSteps: ${{ parameters.customInitSteps }} additionalOptions: ${{ parameters.additionalOptions }} - acr: - server: $(acr-staging.server) - serviceConnection: - tenantId: $(internal-mirror.serviceConnection.tenantId) - clientId: $(internal-mirror.serviceConnection.clientId) - id: $(internal-mirror.serviceConnection.id) - subscription: $(acr-staging.subscription) - resourceGroup: $(acr-staging.resourceGroup) - repoPrefix: $(mirrorRepoPrefix) + acr: ${{ parameters.publishConfig.internalMirrorAcr }} + repoPrefix: ${{ parameters.publishConfig.internalMirrorAcr.repoPrefix }} diff --git a/eng/common/templates/jobs/generate-matrix.yml b/eng/common/templates/jobs/generate-matrix.yml index 8037aca5..663aeb01 100644 --- a/eng/common/templates/jobs/generate-matrix.yml +++ b/eng/common/templates/jobs/generate-matrix.yml @@ -6,6 +6,7 @@ parameters: isTestStage: false internalProjectName: null noCache: false + publishConfig: null customInitSteps: [] commonInitStepsForMatrixAndBuild: [] sourceBuildPipelineRunId: "" @@ -20,6 +21,7 @@ jobs: - ${{ parameters.customInitSteps }} - template: /eng/common/templates/steps/validate-branch.yml@self parameters: + publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} - template: /eng/common/templates/steps/set-image-info-path-var.yml parameters: @@ -36,7 +38,7 @@ jobs: if ("${{ parameters.isTestStage}}" -eq "true") { $additionalGenerateBuildMatrixOptions = "$additionalGenerateBuildMatrixOptions --image-info $(artifactsPath)/image-info.json" } - elseif ("$(pipelineDisabledCache)" -ne "true" -and $env:NOCACHE -ne "true" -and "$(trimCachedImagesForMatrix)" -eq "true") { + elseif ("$(pipelineDisabledCache)" -ne "true" -and "${{ parameters.noCache }}" -ne "true" -and "$(trimCachedImagesForMatrix)" -eq "true") { # If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache $additionalGenerateBuildMatrixOptions = "$additionalGenerateBuildMatrixOptions --image-info $(versionsBasePath)$(imageInfoVersionsPath) --trim-cached-images" } diff --git a/eng/common/templates/jobs/publish.yml b/eng/common/templates/jobs/publish.yml index 5295ad96..68cfb4fe 100644 --- a/eng/common/templates/jobs/publish.yml +++ b/eng/common/templates/jobs/publish.yml @@ -1,6 +1,7 @@ parameters: pool: {} internalProjectName: null + publishConfig: null customInitSteps: [] customPublishVariables: [] sourceBuildPipelineDefinitionId: "" @@ -20,7 +21,7 @@ jobs: - name: imageBuilder.commonCmdArgs value: >- --manifest '$(manifest)' - --registry-override '$(acr.server)' + --registry-override '${{ parameters.publishConfig.publishAcr.server }}' $(manifestVariables) $(imageBuilder.queueArgs) - name: publishNotificationRepoName @@ -46,7 +47,7 @@ jobs: steps: - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: - cloneVersionsRepo: ${{ variables.publishImageInfo }} + publishConfig: ${{ parameters.publishConfig }} versionsRepoRef: ${{ parameters.versionsRepoRef }} - template: /eng/common/templates/steps/retain-build.yml@self @@ -80,6 +81,8 @@ jobs: publicSourceBranch: $(publicSourceBranch) - template: /eng/common/templates/steps/set-dry-run.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} - script: echo "##vso[task.setvariable variable=imageQueueTime]$(date --rfc-2822)" displayName: Set Publish Variables @@ -94,19 +97,19 @@ jobs: displayName: Copy Images serviceConnections: - name: acr - id: $(publish.serviceConnection.id) - tenantId: $(publish.serviceConnection.tenantId) - clientId: $(publish.serviceConnection.clientId) + id: ${{ parameters.publishConfig.publishAcr.serviceConnection.id }} + tenantId: ${{ parameters.publishConfig.publishAcr.serviceConnection.tenantId }} + clientId: ${{ parameters.publishConfig.publishAcr.serviceConnection.clientId }} internalProjectName: ${{ parameters.internalProjectName }} args: >- copyAcrImages - '$(acr.subscription)' - '$(acr.resourceGroup)' - '$(stagingRepoPrefix)' - '$(acr-staging.server)' + '${{ parameters.publishConfig.buildAcr.subscription }}' + '${{ parameters.publishConfig.buildAcr.resourceGroup }}' + '${{ parameters.publishConfig.buildAcr.repoPrefix }}' + '${{ parameters.publishConfig.buildAcr.server }}' --os-type '*' --architecture '*' - --repo-prefix '$(publishRepoPrefix)' + --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}' --image-info '$(imageInfoContainerDir)/image-info.json' $(dryRunArg) $(imageBuilder.pathArgs) @@ -117,15 +120,15 @@ jobs: displayName: Publish Manifest serviceConnections: - name: acr - id: $(publish.serviceConnection.id) - tenantId: $(publish.serviceConnection.tenantId) - clientId: $(publish.serviceConnection.clientId) + id: ${{ parameters.publishConfig.publishAcr.serviceConnection.id }} + tenantId: ${{ parameters.publishConfig.publishAcr.serviceConnection.tenantId }} + clientId: ${{ parameters.publishConfig.publishAcr.serviceConnection.clientId }} internalProjectName: ${{ parameters.internalProjectName }} dockerClientOS: ${{ parameters.dockerClientOS }} args: >- publishManifest '$(imageInfoContainerDir)/image-info.json' - --repo-prefix '$(publishRepoPrefix)' + --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}' --os-type '*' --architecture '*' $(dryRunArg) @@ -142,6 +145,7 @@ jobs: - template: /eng/common/templates/steps/wait-for-mcr-image-ingestion.yml@self parameters: + publishConfig: ${{ parameters.publishConfig }} imageInfoPath: '$(imageinfoContainerDir)/image-info.json' minQueueTime: $(imageQueueTime) dryRunArg: $(dryRunArg) @@ -206,18 +210,18 @@ jobs: displayName: Generate EOL Annotation Data serviceConnections: - name: acr - id: $(publish.serviceConnection.id) - tenantId: $(publish.serviceConnection.tenantId) - clientId: $(publish.serviceConnection.clientId) + id: ${{ parameters.publishConfig.publishAcr.serviceConnection.id }} + tenantId: ${{ parameters.publishConfig.publishAcr.serviceConnection.tenantId }} + clientId: ${{ parameters.publishConfig.publishAcr.serviceConnection.clientId }} internalProjectName: internal condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) args: >- - generateEolAnnotationData + generateEolAnnotationDataForPublish + '${{ parameters.publishConfig.publishAcr.server }}' + '${{ parameters.publishConfig.publishAcr.repoPrefix }}' '$(artifactsPath)/eol-annotation-data/eol-annotation-data.json' '$(imageInfoContainerDir)/full-image-info-orig.json' '$(imageInfoContainerDir)/full-image-info-new.json' - '$(acr.server)' - '$(publishRepoPrefix)' $(generateEolAnnotationDataExtraOptions) $(dryRunArg) @@ -232,7 +236,7 @@ jobs: - template: /eng/common/templates/steps/annotate-eol-digests.yml@self parameters: - internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} dataFile: $(artifactsPath)/eol-annotation-data/eol-annotation-data.json - script: > @@ -271,7 +275,7 @@ jobs: $(gitHubNotificationsRepoInfo.authArgs) '$(gitHubNotificationsRepoInfo.org)' '$(gitHubNotificationsRepoInfo.repo)' - --repo-prefix '$(publishRepoPrefix)' + --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}' --task "🟪 Copy Images" --task "🟪 Publish Manifest" --task "🟪 Wait for Image Ingestion" diff --git a/eng/common/templates/jobs/test-images-linux-client.yml b/eng/common/templates/jobs/test-images-linux-client.yml index 2e9319f2..9aa30e65 100644 --- a/eng/common/templates/jobs/test-images-linux-client.yml +++ b/eng/common/templates/jobs/test-images-linux-client.yml @@ -5,6 +5,7 @@ parameters: testJobTimeout: 60 preBuildValidation: false internalProjectName: null + publishConfig: null customInitSteps: [] sourceBuildPipelineRunId: "" @@ -24,5 +25,6 @@ jobs: parameters: preBuildValidation: ${{ parameters.preBuildValidation }} internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} customInitSteps: ${{ parameters.customInitSteps }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} diff --git a/eng/common/templates/jobs/test-images-windows-client.yml b/eng/common/templates/jobs/test-images-windows-client.yml index 498fee6c..94965f4c 100644 --- a/eng/common/templates/jobs/test-images-windows-client.yml +++ b/eng/common/templates/jobs/test-images-windows-client.yml @@ -4,6 +4,7 @@ parameters: matrix: {} testJobTimeout: 60 internalProjectName: null + publishConfig: null customInitSteps: [] sourceBuildPipelineRunId: "" @@ -19,5 +20,6 @@ jobs: - template: /eng/common/templates/steps/test-images-windows-client.yml@self parameters: internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} customInitSteps: ${{ parameters.customInitSteps }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} diff --git a/eng/common/templates/stages/build-and-test.yml b/eng/common/templates/stages/build-and-test.yml index d1930a2b..2d5ed702 100644 --- a/eng/common/templates/stages/build-and-test.yml +++ b/eng/common/templates/stages/build-and-test.yml @@ -18,13 +18,14 @@ parameters: windowsAmdTestJobTimeout: 60 noCache: false + publishConfig: null internalProjectName: null publicProjectName: null versionsRepoRef: "" - isInternalServicingValidation: false + storageAccountServiceConnection: null linuxAmd64Pool: vmImage: $(defaultLinuxAmd64PoolImage) @@ -58,21 +59,23 @@ stages: testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} preBuildValidation: true internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} customInitSteps: - - ${{ parameters.customTestInitSteps }} - # These variables are normally set by the matrix. Since this test job is not generated - # by a matrix, we need to set them manually. They can be set to empty values since their - # values aren't actually used for the pre-build tests. - - powershell: | - echo "##vso[task.setvariable variable=productVersion]" - echo "##vso[task.setvariable variable=imageBuilderPaths]" - echo "##vso[task.setvariable variable=osVersions]" - echo "##vso[task.setvariable variable=architecture]" - displayName: Initialize Test Variables + - ${{ parameters.customTestInitSteps }} + # These variables are normally set by the matrix. Since this test job is not generated + # by a matrix, we need to set them manually. They can be set to empty values since their + # values aren't actually used for the pre-build tests. + - powershell: | + echo "##vso[task.setvariable variable=productVersion]" + echo "##vso[task.setvariable variable=imageBuilderPaths]" + echo "##vso[task.setvariable variable=osVersions]" + echo "##vso[task.setvariable variable=architecture]" + displayName: Initialize Test Variables - template: /eng/common/templates/jobs/copy-base-images-staging.yml@self parameters: name: CopyBaseImages + publishConfig: ${{ parameters.publishConfig }} pool: ${{ parameters.linuxAmd64Pool }} additionalOptions: "--manifest '$(manifest)' $(imageBuilder.pathArgs) $(manifestVariables)" customInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }} @@ -85,10 +88,12 @@ stages: customBuildLegGroupArgs: ${{ parameters.buildMatrixCustomBuildLegGroupArgs }} internalProjectName: ${{ parameters.internalProjectName }} noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} commonInitStepsForMatrixAndBuild: - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: + publishConfig: ${{ parameters.publishConfig }} versionsRepoRef: ${{ parameters.versionsRepoRef }} - template: /eng/common/templates/jobs/build-images.yml@self @@ -101,12 +106,14 @@ stages: commonInitStepsForMatrixAndBuild: - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: + publishConfig: ${{ parameters.publishConfig }} versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} - template: /eng/common/templates/jobs/build-images.yml@self parameters: name: Linux_arm64 @@ -117,12 +124,14 @@ stages: commonInitStepsForMatrixAndBuild: - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: + publishConfig: ${{ parameters.publishConfig }} versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} - template: /eng/common/templates/jobs/build-images.yml@self parameters: name: Linux_arm32 @@ -133,12 +142,14 @@ stages: commonInitStepsForMatrixAndBuild: - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: + publishConfig: ${{ parameters.publishConfig }} versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} - template: /eng/common/templates/jobs/build-images.yml@self parameters: name: Windows1809_amd64 @@ -149,12 +160,14 @@ stages: commonInitStepsForMatrixAndBuild: - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: + publishConfig: ${{ parameters.publishConfig }} versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} - template: /eng/common/templates/jobs/build-images.yml@self parameters: name: Windows2022_amd64 @@ -165,12 +178,14 @@ stages: commonInitStepsForMatrixAndBuild: - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: + publishConfig: ${{ parameters.publishConfig }} versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} - template: /eng/common/templates/jobs/build-images.yml@self parameters: name: Windows2025_amd64 @@ -181,13 +196,15 @@ stages: commonInitStepsForMatrixAndBuild: - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: + publishConfig: ${{ parameters.publishConfig }} versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} versionsRepoRef: ${{ parameters.versionsRepoRef }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} - template: /eng/common/templates/jobs/build-images.yml@self parameters: name: WindowsLtsc2016_amd64 @@ -198,12 +215,14 @@ stages: commonInitStepsForMatrixAndBuild: - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: + publishConfig: ${{ parameters.publishConfig }} versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} ################################################################################ # Post-Build @@ -251,6 +270,7 @@ stages: - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: versionsRepoRef: ${{ parameters.versionsRepoRef }} + publishConfig: ${{ parameters.publishConfig }} - template: /eng/common/templates/jobs/test-images-linux-client.yml@self parameters: name: Linux_amd64 @@ -258,6 +278,7 @@ stages: matrix: dependencies.GenerateTestMatrix.outputs['matrix.LinuxAmd64'] testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} customInitSteps: ${{ parameters.customTestInitSteps }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - template: /eng/common/templates/jobs/test-images-linux-client.yml@self @@ -267,6 +288,7 @@ stages: matrix: dependencies.GenerateTestMatrix.outputs['matrix.LinuxArm64'] testJobTimeout: ${{ parameters.linuxArmTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} customInitSteps: ${{ parameters.customTestInitSteps }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - template: /eng/common/templates/jobs/test-images-linux-client.yml@self @@ -276,6 +298,7 @@ stages: matrix: dependencies.GenerateTestMatrix.outputs['matrix.LinuxArm32'] testJobTimeout: ${{ parameters.linuxArmTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} customInitSteps: ${{ parameters.customTestInitSteps }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - template: /eng/common/templates/jobs/test-images-windows-client.yml@self @@ -285,6 +308,7 @@ stages: matrix: dependencies.GenerateTestMatrix.outputs['matrix.Windows1809Amd64'] testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} customInitSteps: ${{ parameters.customTestInitSteps }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - template: /eng/common/templates/jobs/test-images-windows-client.yml@self @@ -294,6 +318,7 @@ stages: matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2022Amd64'] testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} customInitSteps: ${{ parameters.customTestInitSteps }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - template: /eng/common/templates/jobs/test-images-windows-client.yml@self @@ -303,6 +328,7 @@ stages: matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2025Amd64'] testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} customInitSteps: ${{ parameters.customTestInitSteps }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - template: /eng/common/templates/jobs/test-images-windows-client.yml@self @@ -312,5 +338,6 @@ stages: matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2016Amd64'] testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} customInitSteps: ${{ parameters.customTestInitSteps }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} diff --git a/eng/common/templates/stages/dotnet/build-and-test.yml b/eng/common/templates/stages/dotnet/build-and-test.yml index 3e6c80d8..dd8afd9e 100644 --- a/eng/common/templates/stages/dotnet/build-and-test.yml +++ b/eng/common/templates/stages/dotnet/build-and-test.yml @@ -3,7 +3,13 @@ parameters: linuxAmd64Pool: "" - isInternalServicingValidation: false + + # (Optional) This service connection should be an Azure Resource Manager + # service connection to a storage account that's needed during image builds. + # It can be used to build images with access to private/internal bits. + # If specified, this service connection will be used to pass a storage + # account access token as `--build-arg ACCESSTOKEN=***` to all image builds. + storageAccountServiceConnection: null # Parameters for pre-build jobs customGenerateMatrixInitSteps: [] @@ -11,6 +17,7 @@ parameters: # Build parameters noCache: false + publishConfig: null buildMatrixType: platformDependencyGraph buildMatrixCustomBuildLegGroupArgs: "" linuxAmdBuildJobTimeout: 60 @@ -36,9 +43,10 @@ stages: - template: /eng/common/templates/stages/build-and-test.yml@self parameters: noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} + storageAccountServiceConnection: ${{ parameters.storageAccountServiceConnection }} customGenerateMatrixInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} buildMatrixCustomBuildLegGroupArgs: ${{ parameters.buildMatrixCustomBuildLegGroupArgs }} testMatrixCustomBuildLegGroupArgs: ${{ parameters.testMatrixCustomBuildLegGroupArgs }} diff --git a/eng/common/templates/stages/dotnet/build-test-publish-repo.yml b/eng/common/templates/stages/dotnet/build-test-publish-repo.yml index f7f54fce..ee8f43af 100644 --- a/eng/common/templates/stages/dotnet/build-test-publish-repo.yml +++ b/eng/common/templates/stages/dotnet/build-test-publish-repo.yml @@ -2,7 +2,6 @@ parameters: linuxAmd64Pool: "" - isInternalServicingValidation: false # Parameters for pre-build jobs customGenerateMatrixInitSteps: [] @@ -10,6 +9,7 @@ parameters: # Build parameters noCache: false + publishConfig: null buildMatrixType: platformDependencyGraph buildMatrixCustomBuildLegGroupArgs: "" linuxAmdBuildJobTimeout: 60 @@ -32,19 +32,18 @@ parameters: # Other common parameters internalProjectName: null publicProjectName: null - versionsRepoRef: null - + versionsRepoRef: "" stages: - template: /eng/common/templates/stages/dotnet/build-and-test.yml@self parameters: linuxAmd64Pool: ${{ parameters.linuxAmd64Pool }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} # Pre-build customGenerateMatrixInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} customCopyBaseImagesInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }} # Build noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} buildMatrixType: ${{ parameters.buildMatrixType }} buildMatrixCustomBuildLegGroupArgs: ${{ parameters.buildMatrixCustomBuildLegGroupArgs }} linuxAmdBuildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }} @@ -67,9 +66,9 @@ stages: - template: /eng/common/templates/stages/dotnet/publish.yml@self parameters: pool: ${{ parameters.linuxAmd64Pool }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} customPublishInitSteps: ${{ parameters.customPublishInitSteps }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} + publishConfig: ${{ parameters.publishConfig }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} versionsRepoRef: ${{ parameters.versionsRepoRef }} diff --git a/eng/common/templates/stages/dotnet/publish-config-nonprod.yml b/eng/common/templates/stages/dotnet/publish-config-nonprod.yml new file mode 100644 index 00000000..bc236962 --- /dev/null +++ b/eng/common/templates/stages/dotnet/publish-config-nonprod.yml @@ -0,0 +1,91 @@ +# This pipeline template injects the publish config for the dotnet-docker +# non-production (unofficial) environment. +# The overall structure of this file should stay mostly in-sync with the +# publish-config-prod.yml template. + +parameters: +# By default, images are staged in repos that are prefixed with this pipeline +# build ID. This is makes it easy to look up which pipeline images were built +# from and vice versa. +- name: sourceBuildPipelineRunId + type: string + default: $(Build.BuildId) + +# This prefix is added to the staging repo when pushing images. If the trailing +# slash is omitted, it will not be added automatically. +- name: stagingRepoPrefix + type: string + default: "build-staging/" + +# Images will have this prefix added to their repo name when pushed to the +# publishing ACR. If the trailing slash is omitted, it will not be added +# automatically. +- name: publishRepoPrefix + type: string + default: "public/" + +# This template will have the publishConfig, internalProjectName, and +# publicProjectName parameters passed to it automatically. +- name: stagesTemplate + type: string + +# These parameters will be passed to the template referred to by the +# stagesTemplate parameter. +# Note: publishConfig, internalProjectName, and publicProjectName are passed +# automatically by this template. Don't define them in this parameter - they +# will get overwritten. +- name: stagesTemplateParameters + type: object + default: {} + + +stages: +- template: ${{ parameters.stagesTemplate }} + parameters: + ${{ insert }}: ${{ parameters.stagesTemplateParameters }} + + internalProjectName: "internal" + publicProjectName: "public" + + publishConfig: + internalMirrorAcr: + server: $(acr-staging-test.server) + repoPrefix: $(mirrorRepoPrefix) + resourceGroup: $(testResourceGroup) + subscription: $(testSubscription) + serviceConnection: + name: $(internal-mirror-test.serviceConnectionName) + id: $(internal-mirror-test.serviceConnection.id) + clientId: $(internal-mirror-test.serviceConnection.clientId) + tenantId: $(testTenant) + + publicMirrorAcr: + server: $(public-mirror.server) + + buildAcr: + server: $(acr-staging-test.server) + resourceGroup: $(testResourceGroup) + subscription: $(testSubscription) + repoPrefix: "${{ parameters.stagingRepoPrefix }}${{ parameters.sourceBuildPipelineRunId }}/" + serviceConnection: + name: $(build-test.serviceConnectionName) + id: $(build-test.serviceConnection.id) + clientId: $(build-test.serviceConnection.clientId) + tenantId: $(testTenant) + + testServiceConnection: + name: $(test-nonprod.serviceConnectionName) + id: $(test-nonprod.serviceConnection.id) + clientId: $(test-nonprod.serviceConnection.clientId) + tenantId: $(testTenant) + + publishAcr: + server: $(acr-test.server) + resourceGroup: $(testResourceGroup) + subscription: $(testSubscription) + repoPrefix: "${{ parameters.publishRepoPrefix }}" + serviceConnection: + name: $(publish-test.serviceConnectionName) + id: $(publish-test.serviceConnection.id) + clientId: $(publish-test.serviceConnection.clientId) + tenantId: $(testTenant) diff --git a/eng/common/templates/stages/dotnet/publish-config-prod.yml b/eng/common/templates/stages/dotnet/publish-config-prod.yml new file mode 100644 index 00000000..7ac47d69 --- /dev/null +++ b/eng/common/templates/stages/dotnet/publish-config-prod.yml @@ -0,0 +1,91 @@ +# This pipeline template injects the publish config for the dotnet-docker +# production (official) environment. +# The overall structure of this file should stay mostly in-sync with the +# publish-config-nonprod.yml template. + +parameters: +# By default, images are staged in repos that are prefixed with this pipeline +# build ID. This is makes it easy to look up which pipeline images were built +# from and vice versa. +- name: sourceBuildPipelineRunId + type: string + default: $(Build.BuildId) + +# This prefix is added to the staging repo when pushing images. If the trailing +# slash is omitted, it will not be added automatically. +- name: stagingRepoPrefix + type: string + default: "build-staging/" + +# Images will have this prefix added to their repo name when pushed to the +# publishing ACR. If the trailing slash is omitted, it will not be added +# automatically. +- name: publishRepoPrefix + type: string + default: "public/" + +# This template will have the publishConfig, internalProjectName, and +# publicProjectName parameters passed to it automatically. +- name: stagesTemplate + type: string + +# These parameters will be passed to the template referred to by the +# stagesTemplate parameter. +# Note: publishConfig, internalProjectName, and publicProjectName are passed +# automatically by this template. Don't define them in this parameter - they +# will get overwritten. +- name: stagesTemplateParameters + type: object + default: {} + + +stages: +- template: ${{ parameters.stagesTemplate }} + parameters: + ${{ insert }}: ${{ parameters.stagesTemplateParameters }} + + internalProjectName: "internal" + publicProjectName: "public" + + publishConfig: + internalMirrorAcr: + server: $(acr-staging.server) + repoPrefix: $(mirrorRepoPrefix) + resourceGroup: $(acr-staging.resourceGroup) + subscription: $(acr-staging.subscription) + serviceConnection: + name: $(internal-mirror.serviceConnectionName) + id: $(internal-mirror.serviceConnection.id) + clientId: $(internal-mirror.serviceConnection.clientId) + tenantId: $(internal-mirror.serviceConnection.tenantId) + + publicMirrorAcr: + server: $(public-mirror.server) + + buildAcr: + server: $(acr-staging.server) + resourceGroup: $(acr-staging.resourceGroup) + subscription: $(acr-staging.subscription) + repoPrefix: "${{ parameters.stagingRepoPrefix }}${{ parameters.sourceBuildPipelineRunId }}/" + serviceConnection: + name: $(build.serviceConnectionName) + id: $(build.serviceConnection.id) + clientId: $(build.serviceConnection.clientId) + tenantId: $(build.serviceConnection.tenantId) + + testServiceConnection: + name: $(test.serviceConnectionName) + id: $(test.serviceConnection.id) + clientId: $(test.serviceConnection.clientId) + tenantId: $(test.serviceConnection.tenantId) + + publishAcr: + server: $(acr.server) + resourceGroup: $(acr.resourceGroup) + subscription: $(acr.subscription) + repoPrefix: "${{ parameters.publishRepoPrefix }}" + serviceConnection: + name: $(publish.serviceConnectionName) + id: $(publish.serviceConnection.id) + clientId: $(publish.serviceConnection.clientId) + tenantId: $(publish.serviceConnection.tenantId) diff --git a/eng/common/templates/stages/dotnet/publish.yml b/eng/common/templates/stages/dotnet/publish.yml index d396174c..d041b3d5 100644 --- a/eng/common/templates/stages/dotnet/publish.yml +++ b/eng/common/templates/stages/dotnet/publish.yml @@ -4,8 +4,8 @@ parameters: internalProjectName: null publicProjectName: null + publishConfig: null pool: "" - isInternalServicingValidation: false isStandalonePublish: false customPublishInitSteps: [] sourceBuildPipelineDefinitionId: '' @@ -18,7 +18,7 @@ stages: parameters: internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} + publishConfig: ${{ parameters.publishConfig }} isStandalonePublish: ${{ parameters.isStandalonePublish }} sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} @@ -45,10 +45,6 @@ stages: displayName: "Set Custom Repo Name Var" - ${{ parameters.customPublishInitSteps }} - ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: - customPublishVariables: - - group: DotNet-AllOrgs-Darc-Pats - pool: ${{ if ne(parameters.pool, '') }}: ${{ parameters.pool }} diff --git a/eng/common/templates/stages/publish.yml b/eng/common/templates/stages/publish.yml index 18e7c492..f1569fba 100644 --- a/eng/common/templates/stages/publish.yml +++ b/eng/common/templates/stages/publish.yml @@ -5,7 +5,8 @@ parameters: internalProjectName: null publicProjectName: null - isInternalServicingValidation: false + publishConfig: null + isStandalonePublish: false pool: @@ -27,53 +28,53 @@ parameters: # Publish Images ################################################################################ stages: -- ${{ if eq(parameters.isInternalServicingValidation, 'false') }}: - - stage: Publish - ${{ if eq(parameters.isStandalonePublish, true) }}: - dependsOn: [] +- stage: Publish + ${{ if eq(parameters.isStandalonePublish, true) }}: + dependsOn: [] + ${{ else }}: + ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + dependsOn: Test ${{ else }}: - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - dependsOn: Test - ${{ else }}: - dependsOn: Post_Build - condition: " + dependsOn: Post_Build + condition: " + and( + not(canceled()), and( - not(canceled()), - and( - contains(variables['stages'], 'publish'), + contains(variables['stages'], 'publish'), + or( or( + and( + and( + contains(variables['stages'], 'build'), + succeeded('Post_Build')), + and( + contains(variables['stages'], 'test'), + in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))), or( and( - and( - contains(variables['stages'], 'build'), - succeeded('Post_Build')), + not(contains(variables['stages'], 'build')), and( contains(variables['stages'], 'test'), in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))), - or( - and( - not(contains(variables['stages'], 'build')), - and( - contains(variables['stages'], 'test'), - in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))), + and( + not(contains(variables['stages'], 'test')), and( - not(contains(variables['stages'], 'test')), - and( - contains(variables['stages'], 'build'), - succeeded('Post_Build'))))), - not( - or( - contains(variables['stages'], 'build'), - contains(variables['stages'], 'test'))))))" - jobs: - - template: /eng/common/templates/jobs/publish.yml@self - parameters: - pool: ${{ parameters.pool }} - internalProjectName: ${{ parameters.internalProjectName }} - customPublishVariables: ${{ parameters.customPublishVariables }} - customInitSteps: ${{ parameters.customPublishInitSteps }} - sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }} - sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - versionsRepoRef: ${{ parameters.versionsRepoRef }} - versionsRepoPath: ${{ parameters.versionsRepoPath }} - overrideImageInfoCommit: ${{ parameters.overrideImageInfoCommit }} + contains(variables['stages'], 'build'), + succeeded('Post_Build'))))), + not( + or( + contains(variables['stages'], 'build'), + contains(variables['stages'], 'test'))))))" + jobs: + - template: /eng/common/templates/jobs/publish.yml@self + parameters: + pool: ${{ parameters.pool }} + internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} + customPublishVariables: ${{ parameters.customPublishVariables }} + customInitSteps: ${{ parameters.customPublishInitSteps }} + sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + versionsRepoRef: ${{ parameters.versionsRepoRef }} + versionsRepoPath: ${{ parameters.versionsRepoPath }} + overrideImageInfoCommit: ${{ parameters.overrideImageInfoCommit }} diff --git a/eng/common/templates/stages/setup-service-connections.yml b/eng/common/templates/stages/setup-service-connections.yml index f4c74ead..2ef74e90 100644 --- a/eng/common/templates/stages/setup-service-connections.yml +++ b/eng/common/templates/stages/setup-service-connections.yml @@ -6,6 +6,10 @@ parameters: - name: pool type: object + default: + name: $(default1ESInternalPoolName) + image: $(default1ESInternalPoolImage) + os: linux # serviceConnections object shape: # - name: string - name: serviceConnections diff --git a/eng/common/templates/steps/annotate-eol-digests.yml b/eng/common/templates/steps/annotate-eol-digests.yml index e6306bc5..0e7d5c32 100644 --- a/eng/common/templates/steps/annotate-eol-digests.yml +++ b/eng/common/templates/steps/annotate-eol-digests.yml @@ -1,5 +1,10 @@ parameters: - dataFile: null +- name: publishConfig + type: object +# Path to EOL annotation data JSON file generated by 'generateEolAnnotationData*' command +- name: dataFile + type: string + steps: - script: mkdir -p $(Build.ArtifactStagingDirectory)/annotation-digests displayName: Create Annotation Digests Directory @@ -9,16 +14,16 @@ steps: displayName: Annotate EOL Images serviceConnections: - name: acr - id: $(publish.serviceConnection.id) - tenantId: $(publish.serviceConnection.tenantId) - clientId: $(publish.serviceConnection.clientId) + id: ${{ parameters.publishConfig.publishAcr.serviceConnection.id }} + tenantId: ${{ parameters.publishConfig.publishAcr.serviceConnection.tenantId }} + clientId: ${{ parameters.publishConfig.publishAcr.serviceConnection.clientId }} internalProjectName: internal condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) args: >- annotateEolDigests ${{ parameters.dataFile }} - $(acr.server) - $(publishRepoPrefix) + ${{ parameters.publishConfig.publishAcr.server }} + ${{ parameters.publishConfig.publishAcr.repoPrefix }} $(artifactsPath)/annotation-digests/annotation-digests.txt $(dryRunArg) - template: /eng/common/templates/steps/publish-artifact.yml@self diff --git a/eng/common/templates/steps/init-docker-linux.yml b/eng/common/templates/steps/init-docker-linux.yml index 729546a3..8c554ad9 100644 --- a/eng/common/templates/steps/init-docker-linux.yml +++ b/eng/common/templates/steps/init-docker-linux.yml @@ -25,7 +25,7 @@ steps: ################################################################################ - ${{ if eq(parameters.setupImageBuilder, 'true') }}: - - script: $(engCommonPath)/pull-image.sh $(imageNames.imageBuilder) + - powershell: $(engCommonPath)/Pull-Image.ps1 $(imageNames.imageBuilder) displayName: Pull Image Builder condition: and(succeeded(), ${{ parameters.condition }}) @@ -78,7 +78,7 @@ steps: # Setup Test Runner (Optional) ################################################################################ - ${{ if eq(parameters.setupTestRunner, 'true') }}: - - script: $(engCommonPath)/pull-image.sh $(imageNames.testrunner) + - powershell: $(engCommonPath)/Pull-Image.ps1 $(imageNames.testrunner) displayName: Pull Test Runner condition: and(succeeded(), ${{ parameters.condition }}) - script: > diff --git a/eng/common/templates/steps/init-matrix-build-publish.yml b/eng/common/templates/steps/init-matrix-build-publish.yml index 95408c66..5ba018fe 100644 --- a/eng/common/templates/steps/init-matrix-build-publish.yml +++ b/eng/common/templates/steps/init-matrix-build-publish.yml @@ -5,6 +5,7 @@ # - Publishing images parameters: + publishConfig: null versionsRepoRef: "" versionsRepoPath: "versions" @@ -15,11 +16,11 @@ steps: path: s/${{ parameters.versionsRepoPath }} persistCredentials: true fetchDepth: 1 - condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) + condition: succeeded() - powershell: | $commonMatrixAndBuildOptions = "--source-repo $(publicGitRepoUri)" if ("$(System.TeamProject)" -eq "internal" -and "$(Build.Reason)" -ne "PullRequest") { - $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --source-repo-prefix $(mirrorRepoPrefix) --registry-override $(acr-staging.server)" + $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --source-repo-prefix ${{ parameters.publishConfig.internalMirrorAcr.repoPrefix }} --registry-override ${{ parameters.publishConfig.buildAcr.server }}" } if ("$(System.TeamProject)" -eq "public" -and "$(public-mirror.server)" -ne "") { diff --git a/eng/common/templates/steps/publish-readmes.yml b/eng/common/templates/steps/publish-readmes.yml index 4f7f77dd..4cf80ecf 100644 --- a/eng/common/templates/steps/publish-readmes.yml +++ b/eng/common/templates/steps/publish-readmes.yml @@ -6,7 +6,7 @@ steps: - script: > $(runImageBuilderCmd) publishMcrDocs --manifest '$(manifest)' - --registry-override '$(acr.server)' + --registry-override '${{ parameters.publishConfig.publishAcr.server }}' '$(mcrDocsRepoInfo.userName)' '$(mcrDocsRepoInfo.email)' $(mcrDocsRepoInfo.authArgs) diff --git a/eng/common/templates/steps/set-dry-run.yml b/eng/common/templates/steps/set-dry-run.yml index d8e3aeaa..44e3062b 100644 --- a/eng/common/templates/steps/set-dry-run.yml +++ b/eng/common/templates/steps/set-dry-run.yml @@ -1,11 +1,30 @@ +parameters: + name: publishConfig + type: object + steps: - powershell: | - # Use dry-run option for certain publish operations if this is not a production build - $dryRunArg="" - if (-not "$(officialRepoPrefixes)".Split(',').Contains("$(publishRepoPrefix)") ` - -or "$(System.TeamProject)" -eq "$(publicProjectName)") + if ("$env:ONEESPT_BUILDTYPE" -eq "Unofficial") { + # Don't use dry-run mode for unofficial builds, since they publish to a + # non-production environment + $dryRunArg="" + } + elseif ("$(System.TeamProject)" -eq "$(publicProjectName)") + { + # Public builds need to use dry-run mode since they don't publish anywhere. $dryRunArg="--dry-run" } + elseif (-not "$(officialRepoPrefixes)".Split(',').Contains("${{ parameters.publishConfig.publishAcr.repoPrefix }}")) + { + # If we're running an internal build on an official pipeline but not + # publishing to an official repo prefix, then use dry run mode. + $dryRunArg="--dry-run" + } + else + { + $dryRunArg="" + } + echo "##vso[task.setvariable variable=dryRunArg]$dryRunArg" displayName: Set dry-run arg for non-prod diff --git a/eng/common/templates/steps/test-images-linux-client.yml b/eng/common/templates/steps/test-images-linux-client.yml index 1af54d0c..e97f015c 100644 --- a/eng/common/templates/steps/test-images-linux-client.yml +++ b/eng/common/templates/steps/test-images-linux-client.yml @@ -1,6 +1,7 @@ parameters: preBuildValidation: false internalProjectName: null + publishConfig: null condition: true customInitSteps: [] sourceBuildPipelineRunId: "" @@ -23,7 +24,7 @@ steps: additionalTestArgs="$additionalTestArgs -TestCategories pre-build" else if [ "${{ variables['System.TeamProject'] }}" == "${{ parameters.internalProjectName }}" ] && [ "${{ variables['Build.Reason'] }}" != "PullRequest" ]; then - additionalTestArgs="$additionalTestArgs -PullImages -Registry $(acr-staging.server) -RepoPrefix $(stagingRepoPrefix) -ImageInfoPath $(artifactsPath)/image-info.json" + additionalTestArgs="$additionalTestArgs -PullImages -Registry ${{ parameters.publishConfig.buildAcr.server }} -RepoPrefix ${{ parameters.publishConfig.buildAcr.repoPrefix }} -ImageInfoPath $(artifactsPath)/image-info.json" if [ "$TESTCATEGORIESOVERRIDE" != "" ]; then additionalTestArgs="$additionalTestArgs -TestCategories $TESTCATEGORIESOVERRIDE" fi @@ -46,13 +47,13 @@ steps: - template: /eng/common/templates/steps/run-pwsh-with-auth.yml@self parameters: displayName: Docker login - serviceConnection: $(test.serviceConnectionName) + serviceConnection: ${{ parameters.publishConfig.testServiceConnection.name }} condition: and(succeeded(), ${{ parameters.condition }}) command: >- $azLoginArgs = '--service-principal --tenant $env:AZURE_TENANT_ID -u $env:AZURE_CLIENT_ID --federated-token $env:AZURE_FEDERATED_TOKEN'; docker exec -e AZURE_TENANT_ID=$env:tenantId -e AZURE_CLIENT_ID=$env:servicePrincipalId -e AZURE_FEDERATED_TOKEN=$env:idToken $(testRunner.container) pwsh -File $(engCommonRelativePath)/Invoke-WithRetry.ps1 - "az login $azLoginArgs; az acr login -n $(acr-staging.server)" + "az login $azLoginArgs; az acr login -n ${{ parameters.publishConfig.buildAcr.server }}" - ${{ if eq(parameters.preBuildValidation, 'false') }}: - template: /eng/common/templates/steps/download-build-artifact.yml@self parameters: @@ -75,7 +76,7 @@ steps: displayName: Test Images condition: and(succeeded(), ${{ parameters.condition }}) - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - - script: docker exec $(testRunner.container) docker logout $(acr-staging.server) + - script: docker exec $(testRunner.container) docker logout ${{ parameters.publishConfig.buildAcr.server }} displayName: Docker logout condition: and(always(), ${{ parameters.condition }}) continueOnError: true diff --git a/eng/common/templates/steps/test-images-windows-client.yml b/eng/common/templates/steps/test-images-windows-client.yml index d53a42b6..00fa062f 100644 --- a/eng/common/templates/steps/test-images-windows-client.yml +++ b/eng/common/templates/steps/test-images-windows-client.yml @@ -1,5 +1,6 @@ parameters: internalProjectName: null + publishConfig: null condition: true customInitSteps: [] sourceBuildPipelineRunId: "" @@ -14,17 +15,17 @@ steps: - template: /eng/common/templates/steps/run-pwsh-with-auth.yml@self parameters: displayName: Docker login - serviceConnection: $(test.serviceConnectionName) + serviceConnection: ${{ parameters.publishConfig.testServiceConnection.name }} dockerClientOS: windows condition: and(succeeded(), ${{ parameters.condition }}) command: >- az login --service-principal --tenant $env:tenantId -u $env:servicePrincipalId --federated-token $env:idToken; - $accessToken = $(az acr login -n $(acr-staging.server) --expose-token --query accessToken --output tsv); - docker login $(acr-staging.server) -u 00000000-0000-0000-0000-000000000000 -p $accessToken + $accessToken = $(az acr login -n ${{ parameters.publishConfig.buildAcr.server }} --expose-token --query accessToken --output tsv); + docker login ${{ parameters.publishConfig.buildAcr.server }} -u 00000000-0000-0000-0000-000000000000 -p $accessToken - ${{ parameters.customInitSteps }} - powershell: | if ("${{ variables['System.TeamProject'] }}" -eq "${{ parameters.internalProjectName }}" -and "${{ variables['Build.Reason'] }}" -ne "PullRequest") { - $additionalTestArgs="$env:ADDITIONALTESTARGS -PullImages -Registry ${env:ACR-STAGING_SERVER} -RepoPrefix $env:STAGINGREPOPREFIX -ImageInfoPath $(artifactsPath)/image-info.json" + $additionalTestArgs="$env:ADDITIONALTESTARGS -PullImages -Registry ${{ parameters.publishConfig.buildAcr.server }} -RepoPrefix ${{ parameters.publishConfig.buildAcr.repoPrefix }} -ImageInfoPath $(artifactsPath)/image-info.json" } echo "##vso[task.setvariable variable=additionalTestArgs]$additionalTestArgs" displayName: Set Test Variables @@ -49,7 +50,7 @@ steps: displayName: Test Images condition: and(succeeded(), ${{ parameters.condition }}) - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - - script: docker logout $(acr-staging.server) + - script: docker logout ${{ parameters.publishConfig.buildAcr.server }} displayName: Docker logout condition: and(always(), ${{ parameters.condition }}) continueOnError: true diff --git a/eng/common/templates/steps/validate-branch.yml b/eng/common/templates/steps/validate-branch.yml index 03ca58da..0bfcf9c9 100644 --- a/eng/common/templates/steps/validate-branch.yml +++ b/eng/common/templates/steps/validate-branch.yml @@ -1,28 +1,39 @@ parameters: + publishConfig: null internalProjectName: null steps: - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - powershell: | + if ("$env:ONEESPT_BUILDTYPE" -eq "Unofficial") + { + echo "Build is from an unofficial pipeline, continuing..." + exit 0 + } + if ("$(officialBranches)".Split(',').Contains("$(sourceBranch)") ` - -and "$(officialRepoPrefixes)".Split(',').Contains("$(publishRepoPrefix)")) + -and "$(officialRepoPrefixes)".Split(',').Contains("${{ parameters.publishConfig.publishAcr.repoPrefix }}")) { echo "Conditions met for official build, continuing..." exit 0 } - if (-not "$(officialRepoPrefixes)".Split(',').Contains("$(publishRepoPrefix)")) + if (-not "$(officialRepoPrefixes)".Split(',').Contains("${{ parameters.publishConfig.publishAcr.repoPrefix }}")) { echo "This build is a test build, continuing..." exit 0 } - if ("$(overrideOfficialBranchValidation)" -eq "true") + if ("${{ variables['overrideOfficialBranchValidation'] }}" -eq "true") { echo "Variable overrideOfficialBranchValidation is set to true, continuing..." exit 0 } echo "##vso[task.logissue type=error]Official builds must be done from an official branch ($(officialBranches)) and repo prefix ($(officialRepoPrefixes))." + echo "Build definition: $(Build.DefinitionName)" + echo "1ESPT build type: $(OneESPT.BuildType)" + echo "Current branch: $(sourceBranch)" + echo "Publish repo prefix: ${{ parameters.publishConfig.publishAcr.repoPrefix }}" exit 1 displayName: Validate Branch diff --git a/eng/common/templates/steps/wait-for-mcr-image-ingestion.yml b/eng/common/templates/steps/wait-for-mcr-image-ingestion.yml index a3c84ddf..f8ad85d5 100644 --- a/eng/common/templates/steps/wait-for-mcr-image-ingestion.yml +++ b/eng/common/templates/steps/wait-for-mcr-image-ingestion.yml @@ -1,8 +1,19 @@ parameters: - imageInfoPath: null - minQueueTime: null - condition: true - dryRunArg: "" +- name: publishConfig + type: object + +- name: imageInfoPath + type: string + +- name: minQueueTime + type: string + +- name: dryRunArg + type: string + +- name: condition + type: string + default: "true" steps: - template: /eng/common/templates/steps/run-imagebuilder.yml@self @@ -19,7 +30,7 @@ steps: waitForMcrImageIngestion '${{ parameters.imageInfoPath }}' --manifest '$(manifest)' - --repo-prefix '$(publishRepoPrefix)' + --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}' --min-queue-time '${{ parameters.minQueueTime }}' --timeout '$(mcrImageIngestionTimeout)' $(manifestVariables) diff --git a/eng/common/templates/task-prefix-decorator.yml b/eng/common/templates/task-prefix-decorator.yml index 4f1a86ce..598bfbe2 100644 --- a/eng/common/templates/task-prefix-decorator.yml +++ b/eng/common/templates/task-prefix-decorator.yml @@ -1,4 +1,4 @@ -# This Azure Pipelines template is adds a prefix to the display name of each +# This Azure Pipelines template adds a prefix to the display name of each # task passed through the `stages` parameter. When used in conjunction with # an "extends" template which injects a lot of tasks into the pipeline, the # added prefix helps to identify which tasks were passed through this template @@ -20,7 +20,7 @@ parameters: default: null # These stages will be modified and passed to the `baseTemplate` as the -# `stages` parameter. The +# `stages` parameter. - name: stages type: stageList default: [] diff --git a/eng/common/templates/variables/common.yml b/eng/common/templates/variables/common.yml index 65d4ce9d..525709f4 100644 --- a/eng/common/templates/variables/common.yml +++ b/eng/common/templates/variables/common.yml @@ -1,24 +1,7 @@ -parameters: -# sourceBuildPipelineRunId should be overridden when skipping a build to run -# tests or publish images that were produced in a different pipeline run. -# Defaults to $(Build.BuildId) which refers to the current pipeline run. -- name: sourceBuildPipelineRunId - type: string - default: "" - variables: - template: /eng/common/templates/variables/docker-images.yml@self - template: /eng/common/templates/variables/common-paths.yml@self -# stagingRepoPrefix defines where images will be pushed to in the staging repo, -# which is defined in the $(acr-staging.server) variable. -- ${{ if ne(parameters.sourceBuildPipelineRunId, '') }}: - - name: stagingRepoPrefix - value: build-staging/${{ parameters.sourceBuildPipelineRunId }}/ -- ${{ else }}: - - name: stagingRepoPrefix - value: build-staging/$(Build.BuildId)/ - - name: publishReadme value: true - name: publishImageInfo diff --git a/eng/common/templates/variables/docker-images.yml b/eng/common/templates/variables/docker-images.yml index 74f59646..abf257cc 100644 --- a/eng/common/templates/variables/docker-images.yml +++ b/eng/common/templates/variables/docker-images.yml @@ -1,6 +1,7 @@ variables: - imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2766581 + imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2786011 imageNames.imageBuilder: $(imageNames.imageBuilderName) imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId) imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux3.0-docker-testrunner imageNames.testRunner.withrepo: testrunner-withrepo:$(Build.BuildId)-$(System.JobId) + imageNames.syft: anchore/syft:v1.31.0-debug diff --git a/eng/common/templates/variables/dotnet/build-test-publish.yml b/eng/common/templates/variables/dotnet/build-test-publish.yml index 005018a0..94732039 100644 --- a/eng/common/templates/variables/dotnet/build-test-publish.yml +++ b/eng/common/templates/variables/dotnet/build-test-publish.yml @@ -1,16 +1,7 @@ # Common variables for building/testing/publishing in the .NET team's pipelines -parameters: -# sourceBuildPipelineRunId should be overridden when skipping a build to run -# tests or publish images that were produced in a different pipeline run. -# Defaults to $(Build.BuildId) which refers to the current pipeline run. -- name: sourceBuildPipelineRunId - type: string - default: "" variables: - template: /eng/common/templates/variables/dotnet/common.yml@self - parameters: - sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - name: commonVersionsImageInfoPath value: build-info/docker @@ -20,8 +11,10 @@ variables: value: ./tests/run-tests.ps1 - name: testResultsDirectory value: tests/Microsoft.DotNet.Docker.Tests/TestResults/ + - name: officialRepoPrefixes value: public/,internal/private/,unlisted/ + readonly: true - name: mcrDocsRepoInfo.userName value: $(gitHubApp.marDocsUpdater.userName) diff --git a/eng/common/templates/variables/dotnet/common.yml b/eng/common/templates/variables/dotnet/common.yml index 69665b24..bb39d82c 100644 --- a/eng/common/templates/variables/dotnet/common.yml +++ b/eng/common/templates/variables/dotnet/common.yml @@ -1,15 +1,5 @@ -parameters: -# sourceBuildPipelineRunId should be overridden when skipping a build to run -# tests or publish images that were produced in a different pipeline run. -# Defaults to $(Build.BuildId) which refers to the current pipeline run. -- name: sourceBuildPipelineRunId - type: string - default: "" - variables: - template: /eng/common/templates/variables/common.yml@self - parameters: - sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - name: publicProjectName value: public diff --git a/eng/common/templates/variables/dotnet/secrets-unofficial.yml b/eng/common/templates/variables/dotnet/secrets-unofficial.yml new file mode 100644 index 00000000..1744ad28 --- /dev/null +++ b/eng/common/templates/variables/dotnet/secrets-unofficial.yml @@ -0,0 +1,5 @@ +variables: +- group: DotNet-Docker-Secrets-Low + +- name: dockerHubRegistryCreds + value: --registry-creds 'docker.io=$(dotnet-dockerhub-bot-username);$(dotnet-dockerhub-bot-pat-low)' From f4d832bd34ff05cb665877bee7acb713c7682169 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Thu, 4 Sep 2025 10:00:54 -0700 Subject: [PATCH 2/6] Update framework pipeline --- eng/pipelines/dotnet-framework.yml | 52 +++++++------- .../stages/dotnet-framework-base.yml | 46 +++++++++++++ eng/pipelines/variables/common.yml | 69 +++++++++++++------ 3 files changed, 123 insertions(+), 44 deletions(-) create mode 100644 eng/pipelines/stages/dotnet-framework-base.yml diff --git a/eng/pipelines/dotnet-framework.yml b/eng/pipelines/dotnet-framework.yml index 7d436b1c..25118a17 100644 --- a/eng/pipelines/dotnet-framework.yml +++ b/eng/pipelines/dotnet-framework.yml @@ -1,13 +1,6 @@ trigger: none pr: none -resources: - repositories: - - repository: VersionsRepo - type: github - endpoint: dotnet - name: dotnet/versions - parameters: - name: sourceBuildPipelineRunId displayName: > @@ -17,32 +10,45 @@ parameters: building new images, leave this value alone. type: string default: $(Build.BuildId) +- name: noCache + displayName: > + Run build with no cache. When this is unchecked, the build will use the + info in the dotnet/versions repo to build only the images which have had + Dockerfile changes or base images updates. When this is checked, all images + will be built regardless of caching status. + type: boolean + default: false variables: - template: /eng/pipelines/variables/common.yml@self - parameters: - sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} -- template: /eng/common/templates/variables/dotnet/secrets.yml@self +- template: /eng/common/templates/variables/dotnet/secrets${{ iif(contains(variables['Build.DefinitionName'], '-official'), '', '-unofficial') }}.yml@self - name: manifest value: manifest.json - name: mcrImageIngestionTimeout value: "00:30:00" +resources: + repositories: + - repository: VersionsRepo + type: github + endpoint: dotnet + name: dotnet/versions + extends: - template: /eng/common/templates/1es-official.yml@self + template: /eng/common/templates/1es.yml@self parameters: - serviceConnections: - - name: $(internal-mirror.serviceConnectionName) - - name: $(build.serviceConnectionName) - - name: $(publish.serviceConnectionName) - - name: $(kusto.serviceConnectionName) - - name: $(marStatus.serviceConnectionName) + reposToExcludeFromScanning: + - VersionsRepo stages: - - template: /eng/common/templates/stages/dotnet/build-test-publish-repo.yml@self + - template: /eng/common/templates/stages/dotnet/publish-config-${{ iif(contains(variables['Build.DefinitionName'], '-official'), 'prod', 'nonprod') }}.yml@self parameters: - internalProjectName: ${{ variables.internalProjectName }} - publicProjectName: ${{ variables.publicProjectName }} - versionsRepoRef: VersionsRepo sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - windowsAmdBuildJobTimeout: 240 - windowsAmdTestJobTimeout: 90 + stagesTemplate: /eng/pipelines/stages/dotnet-framework-base.yml@self + stagesTemplateParameters: + versionsRepoRef: VersionsRepo + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + noCache: ${{ parameters.noCache }} + ${{ if contains(variables['Build.DefinitionName'], '-official') }}: + additionalServiceConnections: + - name: $(kusto.serviceConnectionName) + - name: $(marStatus.serviceConnectionName) diff --git a/eng/pipelines/stages/dotnet-framework-base.yml b/eng/pipelines/stages/dotnet-framework-base.yml new file mode 100644 index 00000000..add2f747 --- /dev/null +++ b/eng/pipelines/stages/dotnet-framework-base.yml @@ -0,0 +1,46 @@ +parameters: +- name: publishConfig + type: object +- name: internalProjectName + type: string +- name: publicProjectName + type: string + +- name: sourceBuildPipelineRunId + type: string + default: $(Build.BuildId) +- name: versionsRepoRef + type: string + default: "" +- name: noCache + type: boolean + default: false +- name: windowsAmdBuildJobTimeout + default: 240 +- name: windowsAmdTestJobTimeout + default: 90 +# Additional service connections needed by this pipeline that are not included +# in the publishConfig. +- name: additionalServiceConnections + type: object + default: [] + + +stages: +- ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - template: /eng/common/templates/stages/setup-service-connections.yml@self + parameters: + serviceConnections: + - name: ${{ parameters.publishConfig.internalMirrorAcr.serviceConnection.name }} + - name: ${{ parameters.publishConfig.buildAcr.serviceConnection.name }} + - name: ${{ parameters.publishConfig.publishAcr.serviceConnection.name }} + - ${{ each serviceConnection in parameters.additionalServiceConnections }}: + - name: ${{ serviceConnection.name }} + +- template: /eng/common/templates/stages/dotnet/build-test-publish-repo.yml@self + parameters: + versionsRepoRef: ${{ parameters.versionsRepoRef }} + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + noCache: ${{ parameters.noCache }} + windowsAmdBuildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} + windowsAmdTestJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} diff --git a/eng/pipelines/variables/common.yml b/eng/pipelines/variables/common.yml index 743d64e2..94d4af5a 100644 --- a/eng/pipelines/variables/common.yml +++ b/eng/pipelines/variables/common.yml @@ -1,25 +1,52 @@ -parameters: -# sourceBuildPipelineRunId should be overridden when skipping a build to run -# tests or publish images that were produced in a different pipeline run. -- name: sourceBuildPipelineRunId - type: string - default: "" - variables: - template: /eng/common/templates/variables/dotnet/build-test-publish.yml@self - parameters: - sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} -- name: publicGitRepoUri - value: https://github.com/microsoft/dotnet-framework-docker -- name: productVersionComponents + +- name: "publicGitRepoUri" + value: "https://github.com/microsoft/dotnet-framework-docker" + +- name: "productVersionComponents" value: 3 -- name: testScriptPath - value: ./tests/run-tests.ps1 -- name: testResultsDirectory - value: tests/Microsoft.DotNet.Framework.Docker.Tests/TestResults/ -- name: publicSourceBranch - value: main -- name: additionalPublishMcrDocsArgs + +- name: "testScriptPath" + value: "./tests/run-tests.ps1" +- name: "testResultsDirectory" + value: "tests/Microsoft.DotNet.Framework.Docker.Tests/TestResults/" + +# .NET Framework images only publish from one branch +- name: "publicSourceBranch" + value: "main" + readonly: true + +- name: "additionalPublishMcrDocsArgs" value: "--root /repo/.portal-docs" -- name: publishEolAnnotations - value: true + +- ${{ if contains(variables['Build.DefinitionName'], '-official') }}: + - name: "publishEolAnnotations" + value: true + +- ${{ else }}: + # Disable EOL annotations for unofficial build + - name: "publishEolAnnotations" + value: false + readonly: true + + # Unofficial secrets group has no GitHub token for publishing image info, notifications, or readmes + - name: "publishImageInfo" + value: false + readonly: true + - name: "publishNotificationsEnabled" + value: false + readonly: true + - name: "publishReadme" + value: false + readonly: true + + # There aren't test/unofficial Kusto tables for build telemetry yet + - name: "ingestKustoImageInfo" + value: false + readonly: true + + # Always run unofficial pipelines in debug mode + - name: "System.Debug" + value: true + readonly: true From f169be2e5d5e557ac045316082ad2227a2143ee9 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Thu, 4 Sep 2025 13:22:33 -0700 Subject: [PATCH 3/6] Pass more parameters through --- eng/pipelines/stages/dotnet-framework-base.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eng/pipelines/stages/dotnet-framework-base.yml b/eng/pipelines/stages/dotnet-framework-base.yml index add2f747..a0157d9d 100644 --- a/eng/pipelines/stages/dotnet-framework-base.yml +++ b/eng/pipelines/stages/dotnet-framework-base.yml @@ -39,6 +39,9 @@ stages: - template: /eng/common/templates/stages/dotnet/build-test-publish-repo.yml@self parameters: + publishConfig: ${{ parameters.publishConfig }} + internalProjectName: ${{ parameters.internalProjectName }} + publicProjectName: ${{ parameters.publicProjectName }} versionsRepoRef: ${{ parameters.versionsRepoRef }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} noCache: ${{ parameters.noCache }} From 59eb52cba8a45aa0160adc90758b8b2cb7e74f22 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Thu, 4 Sep 2025 14:56:05 -0700 Subject: [PATCH 4/6] Reduce unnecessary diff --- eng/pipelines/dotnet-framework.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/eng/pipelines/dotnet-framework.yml b/eng/pipelines/dotnet-framework.yml index 25118a17..5708171b 100644 --- a/eng/pipelines/dotnet-framework.yml +++ b/eng/pipelines/dotnet-framework.yml @@ -1,6 +1,13 @@ trigger: none pr: none +resources: + repositories: + - repository: VersionsRepo + type: github + endpoint: dotnet + name: dotnet/versions + parameters: - name: sourceBuildPipelineRunId displayName: > @@ -27,13 +34,6 @@ variables: - name: mcrImageIngestionTimeout value: "00:30:00" -resources: - repositories: - - repository: VersionsRepo - type: github - endpoint: dotnet - name: dotnet/versions - extends: template: /eng/common/templates/1es.yml@self parameters: From f4d9fc393e5285bb0576b11d16906f02a023117b Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 5 Sep 2025 08:20:27 -0700 Subject: [PATCH 5/6] Migrate PR pipelines --- eng/pipelines/dotnet-framework-pr-no-cache.yml | 17 ++++++----------- eng/pipelines/dotnet-framework-pr.yml | 14 +++++--------- eng/pipelines/stages/dotnet-framework-base.yml | 18 ++++++++++++------ 3 files changed, 23 insertions(+), 26 deletions(-) diff --git a/eng/pipelines/dotnet-framework-pr-no-cache.yml b/eng/pipelines/dotnet-framework-pr-no-cache.yml index 5a08b540..5a1f44ab 100644 --- a/eng/pipelines/dotnet-framework-pr-no-cache.yml +++ b/eng/pipelines/dotnet-framework-pr-no-cache.yml @@ -16,18 +16,13 @@ resources: name: dotnet/versions variables: -- template: variables/common.yml +- template: /eng/pipelines/variables/common.yml@self - name: manifest value: manifest.json -stages: -- template: ../common/templates/stages/dotnet/build-test-publish-repo.yml +- template: /eng/common/templates/stages/dotnet/publish-config-nonprod.yml@self parameters: - internalProjectName: ${{ variables.internalProjectName }} - publicProjectName: ${{ variables.publicProjectName }} - versionsRepoRef: VersionsRepo - buildMatrixCustomBuildLegGroupArgs: --custom-build-leg-group pr-build - windowsAmdBuildJobTimeout: 330 - noCache: true - ${{ if eq(variables['System.TeamProject'], variables.publicProjectName) }}: - buildMatrixType: platformVersionedOs + stagesTemplate: /eng/pipelines/stages/dotnet-framework-base.yml@self + stagesTemplateParameters: + versionsRepoRef: VersionsRepo + noCache: true diff --git a/eng/pipelines/dotnet-framework-pr.yml b/eng/pipelines/dotnet-framework-pr.yml index 1c53354d..59bccfd5 100644 --- a/eng/pipelines/dotnet-framework-pr.yml +++ b/eng/pipelines/dotnet-framework-pr.yml @@ -18,17 +18,13 @@ resources: name: dotnet/versions variables: -- template: variables/common.yml +- template: /eng/pipelines/variables/common.yml@self - name: manifest value: manifest.json stages: -- template: ../common/templates/stages/dotnet/build-test-publish-repo.yml +- template: /eng/common/templates/stages/dotnet/publish-config-nonprod.yml@self parameters: - internalProjectName: ${{ variables.internalProjectName }} - publicProjectName: ${{ variables.publicProjectName }} - versionsRepoRef: VersionsRepo - buildMatrixCustomBuildLegGroupArgs: --custom-build-leg-group pr-build - windowsAmdBuildJobTimeout: 330 - ${{ if eq(variables['System.TeamProject'], variables.publicProjectName) }}: - buildMatrixType: platformVersionedOs + stagesTemplate: /eng/pipelines/stages/dotnet-framework-base.yml@self + stagesTemplateParameters: + versionsRepoRef: VersionsRepo diff --git a/eng/pipelines/stages/dotnet-framework-base.yml b/eng/pipelines/stages/dotnet-framework-base.yml index a0157d9d..641f02ae 100644 --- a/eng/pipelines/stages/dotnet-framework-base.yml +++ b/eng/pipelines/stages/dotnet-framework-base.yml @@ -15,10 +15,6 @@ parameters: - name: noCache type: boolean default: false -- name: windowsAmdBuildJobTimeout - default: 240 -- name: windowsAmdTestJobTimeout - default: 90 # Additional service connections needed by this pipeline that are not included # in the publishConfig. - name: additionalServiceConnections @@ -45,5 +41,15 @@ stages: versionsRepoRef: ${{ parameters.versionsRepoRef }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} noCache: ${{ parameters.noCache }} - windowsAmdBuildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }} - windowsAmdTestJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} + + ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + windowsAmdBuildJobTimeout: 330 + ${{ else }}: + windowsAmdBuildJobTimeout: 240 + windowsAmdTestJobTimeout: 90 + + ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + buildMatrixCustomBuildLegGroupArgs: --custom-build-leg-group pr-build + + ${{ if eq(variables['System.TeamProject'], variables.publicProjectName) }}: + buildMatrixType: platformVersionedOs From b0585843571de9a09558cda532b6e4938f87f7f1 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 5 Sep 2025 08:23:41 -0700 Subject: [PATCH 6/6] Add missing stages declaration --- eng/pipelines/dotnet-framework-pr-no-cache.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/dotnet-framework-pr-no-cache.yml b/eng/pipelines/dotnet-framework-pr-no-cache.yml index 5a1f44ab..b86f2208 100644 --- a/eng/pipelines/dotnet-framework-pr-no-cache.yml +++ b/eng/pipelines/dotnet-framework-pr-no-cache.yml @@ -20,6 +20,7 @@ variables: - name: manifest value: manifest.json +stages: - template: /eng/common/templates/stages/dotnet/publish-config-nonprod.yml@self parameters: stagesTemplate: /eng/pipelines/stages/dotnet-framework-base.yml@self