Skip to content

Commit 778a83c

Browse files
dotnet-docker-botlbussell
dotnet-docker-bot
and
lbussell
authored
[main] Update common Docker engineering infrastructure with latest (#1256)
Co-authored-by: Logan Bussell <[email protected]>
1 parent f982027 commit 778a83c

17 files changed

+209
-86
lines changed

‎eng/common/templates/1es-official.yml‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# do the following:
33
#
44
# - Do not rely on any source code from the versions repo so as to not circumvent SDL and CG guidelines
5-
# - The versions repo resource must be named `InternalVersionsRepo` or `PublicVersionsRepo` to avoid SDL scans
5+
# - The versions repo resource must be named `VersionsRepo` to avoid SDL scans
66
# - The versions repo must be checked out to `$(Build.SourcesDirectory)/versions` to avoid CG scans
77
#
88
# If the pipeline is not using a separate repository resource, ensure that there is no source code checked out in
@@ -57,14 +57,14 @@ extends:
5757
enabled: true
5858
sourceRepositoriesToScan:
5959
exclude:
60-
- repository: InternalVersionsRepo
61-
- repository: PublicVersionsRepo
60+
- repository: VersionsRepo
6261
sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }}
6362
tsa:
6463
enabled: true
6564
stages:
66-
- template: /eng/common/templates/stages/setup-service-connections.yml@self
67-
parameters:
68-
pool: ${{ parameters.pool }}
69-
serviceConnections: ${{ parameters.serviceConnections }}
65+
- ${{ if gt(length(parameters.serviceConnections), 0) }}:
66+
- template: /eng/common/templates/stages/setup-service-connections.yml@self
67+
parameters:
68+
pool: ${{ parameters.pool }}
69+
serviceConnections: ${{ parameters.serviceConnections }}
7070
- ${{ parameters.stages }}

‎eng/common/templates/1es-unofficial.yml‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -71,8 +71,9 @@ extends:
7171
tsa:
7272
enabled: true
7373
stages:
74-
- template: /eng/common/templates/stages/setup-service-connections.yml@self
75-
parameters:
76-
pool: ${{ parameters.pool }}
77-
serviceConnections: ${{ parameters.serviceConnections }}
74+
- ${{ if gt(length(parameters.serviceConnections), 0) }}:
75+
- template: /eng/common/templates/stages/setup-service-connections.yml@self
76+
parameters:
77+
pool: ${{ parameters.pool }}
78+
serviceConnections: ${{ parameters.serviceConnections }}
7879
- ${{ parameters.stages }}

‎eng/common/templates/jobs/build-images.yml‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ jobs:
4949
# all we need is for that value to be in a PowerShell variable, we can get that by the fact that AzDO automatically creates
5050
# the environment variable for us.
5151
$imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $(imageBuilder.queueArgs) --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json $(commonMatrixAndBuildOptions)"
52-
if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest" -and "${{ parameters.isInternalServicingValidation }}" -ne "true") {
52+
if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest") {
5353
$imageBuilderBuildArgs = "$imageBuilderBuildArgs --repo-prefix $(stagingRepoPrefix) --push"
5454
}
5555
@@ -70,6 +70,11 @@ jobs:
7070
id: $(build.serviceConnection.id)
7171
tenantId: $(build.serviceConnection.tenantId)
7272
clientId: $(build.serviceConnection.clientId)
73+
- ${{ if eq(parameters.isInternalServicingValidation, true) }}:
74+
- name: storage
75+
id: $(dotnetstaging.serviceConnection.id)
76+
tenantId: $(dotnetstaging.serviceConnection.tenantId)
77+
clientId: $(dotnetstaging.serviceConnection.clientId)
7378
internalProjectName: ${{ parameters.internalProjectName }}
7479
dockerClientOS: ${{ parameters.dockerClientOS }}
7580
args: >-
@@ -92,7 +97,7 @@ jobs:
9297
displayName: Publish Image Info File Artifact
9398
internalProjectName: ${{ parameters.internalProjectName }}
9499
publicProjectName: ${{ parameters.publicProjectName }}
95-
- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}:
100+
- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:
96101
# The following task depends on the SBOM Manifest Generator task installed on the agent.
97102
# This task is auto-injected by 1ES Pipeline Templates so we don't need to install it ourselves.
98103
- powershell: |
@@ -144,11 +149,11 @@ jobs:
144149
}
145150
displayName: Generate SBOMs
146151
condition: and(succeeded(), ne(variables['BuildImages.builtImages'], ''))
147-
- ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'true')) }}:
152+
- ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
148153
- template: /eng/common/templates/jobs/${{ format('../steps/test-images-{0}-client.yml', parameters.dockerClientOS) }}@self
149154
parameters:
150155
condition: ne(variables.testScriptPath, '')
151-
- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}:
156+
- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:
152157
- template: /eng/common/templates/steps/publish-artifact.yml@self
153158
parameters:
154159
path: $(sbomDirectory)

‎eng/common/templates/jobs/publish.yml‎

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,11 @@ parameters:
55
customPublishVariables: []
66
sourceBuildPipelineDefinitionId: ""
77
sourceBuildPipelineRunId: ""
8+
versionsRepoRef: null
9+
versionsRepoPath: ""
10+
# When true, overrides the commit SHA in merged image info files to use the current repository commit.
11+
# This ensures that updated images reference the correct commit in their commitUrl properties.
12+
overrideImageInfoCommit: false
813

914
jobs:
1015
- job: Publish
@@ -31,16 +36,28 @@ jobs:
3136
value: $(artifactsPath)/imageInfo
3237
- name: sourceBuildIdOutputDir
3338
value: $(Build.ArtifactStagingDirectory)/sourceBuildId
39+
- name: commitOverrideArg
40+
${{ if eq(parameters.overrideImageInfoCommit, true) }}:
41+
value: --commit-override $(Build.SourceVersion)
42+
${{ else }}:
43+
value: ''
3444
- ${{ parameters.customPublishVariables }}
3545

3646
steps:
47+
- template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
48+
parameters:
49+
cloneVersionsRepo: ${{ variables.publishImageInfo }}
50+
versionsRepoRef: ${{ parameters.versionsRepoRef }}
51+
3752
- template: /eng/common/templates/steps/retain-build.yml@self
3853

3954
- template: /eng/common/templates/steps/init-docker-linux.yml@self
4055

4156
- pwsh: |
4257
$azdoOrgName = Split-Path -Leaf $Env:SYSTEM_COLLECTIONURI
4358
echo "##vso[task.setvariable variable=azdoOrgName]$azdoOrgName"
59+
$versionsRepoRoot = "$(Pipeline.Workspace)/s/${{ parameters.versionsRepoPath }}"
60+
echo "##vso[task.setvariable variable=versionsRepoRoot]$versionsRepoRoot"
4461
displayName: Set Publish Variables
4562
4663
- ${{ parameters.customInitSteps }}
@@ -138,13 +155,16 @@ jobs:
138155
- script: mkdir -p $(Build.ArtifactStagingDirectory)/eol-annotation-data
139156
displayName: Create EOL Annotation Data Directory
140157

141-
- powershell: >-
142-
$(engCommonPath)/Invoke-WithRetry.ps1
143-
"curl -fSL
144-
--output $(imageInfoHostDir)/full-image-info-orig.json
145-
https://raw.githubusercontent.com/$(gitHubVersionsRepoInfo.org)/$(gitHubVersionsRepoInfo.repo)/refs/heads/$(gitHubVersionsRepoInfo.branch)/$(gitHubImageInfoVersionsPath)"
158+
- script: |-
159+
cd $(versionsRepoRoot)
160+
git pull origin $(gitHubVersionsRepoInfo.branch)
161+
condition: and(succeeded(), eq(variables['publishImageInfo'], 'true'))
162+
displayName: Pull Latest Changes from Versions Repo
163+
164+
- script: >-
165+
cp $(versionsRepoRoot)/$(gitHubImageInfoVersionsPath) $(imageInfoHostDir)/full-image-info-orig.json
146166
condition: and(succeeded(), eq(variables['publishImageInfo'], 'true'))
147-
displayName: Download Latest Image Info
167+
displayName: Copy Latest Image Info from Versions Repo
148168
149169
- script: >
150170
$(runImageBuilderCmd) mergeImageInfo
@@ -155,6 +175,7 @@ jobs:
155175
--manifest $(manifest)
156176
--publish
157177
--initial-image-info-path $(imageInfoContainerDir)/full-image-info-orig.json
178+
$(commitOverrideArg)
158179
condition: and(succeeded(), eq(variables['publishImageInfo'], 'true'))
159180
displayName: Merge Image Info
160181

‎eng/common/templates/stages/build-and-test.yml‎

Lines changed: 25 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,7 @@ parameters:
2222
internalProjectName: null
2323
publicProjectName: null
2424

25-
internalVersionsRepoRef: null
26-
publicVersionsRepoRef: null
25+
versionsRepoRef: ""
2726

2827
isInternalServicingValidation: false
2928

@@ -51,6 +50,7 @@ stages:
5150
condition: and(succeeded(), contains(variables['stages'], 'build'))
5251
dependsOn: []
5352
jobs:
53+
5454
- template: /eng/common/templates/jobs/test-images-linux-client.yml@self
5555
parameters:
5656
name: PreBuildValidation
@@ -69,12 +69,14 @@ stages:
6969
echo "##vso[task.setvariable variable=osVersions]"
7070
echo "##vso[task.setvariable variable=architecture]"
7171
displayName: Initialize Test Variables
72+
7273
- template: /eng/common/templates/jobs/copy-base-images-staging.yml@self
7374
parameters:
7475
name: CopyBaseImages
7576
pool: ${{ parameters.linuxAmd64Pool }}
7677
additionalOptions: "--manifest '$(manifest)' $(imageBuilder.pathArgs) $(manifestVariables)"
7778
customInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }}
79+
7880
- template: /eng/common/templates/jobs/generate-matrix.yml@self
7981
parameters:
8082
matrixType: ${{ parameters.buildMatrixType }}
@@ -85,12 +87,10 @@ stages:
8587
noCache: ${{ parameters.noCache }}
8688
customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }}
8789
commonInitStepsForMatrixAndBuild:
88-
- template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
90+
- template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
8991
parameters:
90-
noCache: ${{ parameters.noCache }}
91-
internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
92-
publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
93-
isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
92+
versionsRepoRef: ${{ parameters.versionsRepoRef }}
93+
9494
- template: /eng/common/templates/jobs/build-images.yml@self
9595
parameters:
9696
name: Linux_amd64
@@ -99,12 +99,9 @@ stages:
9999
dockerClientOS: linux
100100
buildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }}
101101
commonInitStepsForMatrixAndBuild:
102-
- template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
102+
- template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
103103
parameters:
104-
noCache: ${{ parameters.noCache }}
105-
internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
106-
publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
107-
isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
104+
versionsRepoRef: ${{ parameters.versionsRepoRef }}
108105
customInitSteps: ${{ parameters.customBuildInitSteps }}
109106
noCache: ${{ parameters.noCache }}
110107
internalProjectName: ${{ parameters.internalProjectName }}
@@ -118,12 +115,9 @@ stages:
118115
dockerClientOS: linux
119116
buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }}
120117
commonInitStepsForMatrixAndBuild:
121-
- template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
118+
- template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
122119
parameters:
123-
noCache: ${{ parameters.noCache }}
124-
internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
125-
publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
126-
isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
120+
versionsRepoRef: ${{ parameters.versionsRepoRef }}
127121
customInitSteps: ${{ parameters.customBuildInitSteps }}
128122
noCache: ${{ parameters.noCache }}
129123
internalProjectName: ${{ parameters.internalProjectName }}
@@ -137,12 +131,9 @@ stages:
137131
dockerClientOS: linux
138132
buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }}
139133
commonInitStepsForMatrixAndBuild:
140-
- template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
134+
- template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
141135
parameters:
142-
noCache: ${{ parameters.noCache }}
143-
internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
144-
publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
145-
isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
136+
versionsRepoRef: ${{ parameters.versionsRepoRef }}
146137
customInitSteps: ${{ parameters.customBuildInitSteps }}
147138
noCache: ${{ parameters.noCache }}
148139
internalProjectName: ${{ parameters.internalProjectName }}
@@ -156,12 +147,9 @@ stages:
156147
dockerClientOS: windows
157148
buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
158149
commonInitStepsForMatrixAndBuild:
159-
- template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
150+
- template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
160151
parameters:
161-
noCache: ${{ parameters.noCache }}
162-
internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
163-
publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
164-
isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
152+
versionsRepoRef: ${{ parameters.versionsRepoRef }}
165153
customInitSteps: ${{ parameters.customBuildInitSteps }}
166154
noCache: ${{ parameters.noCache }}
167155
internalProjectName: ${{ parameters.internalProjectName }}
@@ -175,12 +163,9 @@ stages:
175163
dockerClientOS: windows
176164
buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
177165
commonInitStepsForMatrixAndBuild:
178-
- template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
166+
- template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
179167
parameters:
180-
noCache: ${{ parameters.noCache }}
181-
internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
182-
publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
183-
isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
168+
versionsRepoRef: ${{ parameters.versionsRepoRef }}
184169
customInitSteps: ${{ parameters.customBuildInitSteps }}
185170
noCache: ${{ parameters.noCache }}
186171
internalProjectName: ${{ parameters.internalProjectName }}
@@ -194,18 +179,14 @@ stages:
194179
dockerClientOS: windows
195180
buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
196181
commonInitStepsForMatrixAndBuild:
197-
- template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
182+
- template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
198183
parameters:
199-
noCache: ${{ parameters.noCache }}
200-
internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
201-
publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
202-
isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
184+
versionsRepoRef: ${{ parameters.versionsRepoRef }}
203185
customInitSteps: ${{ parameters.customBuildInitSteps }}
204186
noCache: ${{ parameters.noCache }}
205187
internalProjectName: ${{ parameters.internalProjectName }}
206188
publicProjectName: ${{ parameters.publicProjectName }}
207-
internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
208-
publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
189+
versionsRepoRef: ${{ parameters.versionsRepoRef }}
209190
isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
210191
- template: /eng/common/templates/jobs/build-images.yml@self
211192
parameters:
@@ -215,12 +196,9 @@ stages:
215196
dockerClientOS: windows
216197
buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
217198
commonInitStepsForMatrixAndBuild:
218-
- template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
199+
- template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
219200
parameters:
220-
noCache: ${{ parameters.noCache }}
221-
internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
222-
publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
223-
isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
201+
versionsRepoRef: ${{ parameters.versionsRepoRef }}
224202
customInitSteps: ${{ parameters.customBuildInitSteps }}
225203
noCache: ${{ parameters.noCache }}
226204
internalProjectName: ${{ parameters.internalProjectName }}
@@ -243,7 +221,7 @@ stages:
243221
################################################################################
244222
# Test Images
245223
################################################################################
246-
- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}:
224+
- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:
247225
- stage: Test
248226
dependsOn: Post_Build
249227
condition: "
@@ -270,11 +248,9 @@ stages:
270248
customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }}
271249
sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
272250
commonInitStepsForMatrixAndBuild:
273-
- template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
251+
- template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
274252
parameters:
275-
noCache: ${{ parameters.noCache }}
276-
internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
277-
publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
253+
versionsRepoRef: ${{ parameters.versionsRepoRef }}
278254
- template: /eng/common/templates/jobs/test-images-linux-client.yml@self
279255
parameters:
280256
name: Linux_amd64

‎eng/common/templates/stages/dotnet/build-and-test.yml‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ parameters:
3030
internalProjectName: null
3131
publicProjectName: null
3232

33+
versionsRepoRef: null
34+
3335
stages:
3436
- template: /eng/common/templates/stages/build-and-test.yml@self
3537
parameters:
@@ -51,8 +53,9 @@ stages:
5153
testMatrixType: ${{ parameters.testMatrixType }}
5254
sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
5355

54-
internalVersionsRepoRef: InternalVersionsRepo
55-
publicVersionsRepoRef: PublicVersionsRepo
56+
# Only clone versions repo if we need to reference it during the build in order to cache images.
57+
${{ if eq(parameters.noCache, false) }}:
58+
versionsRepoRef: ${{ parameters.versionsRepoRef }}
5659

5760
# Linux AMD64
5861
linuxAmd64Pool:

‎eng/common/templates/stages/dotnet/build-test-publish-repo.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ parameters:
3232
# Other common parameters
3333
internalProjectName: null
3434
publicProjectName: null
35+
versionsRepoRef: null
3536

3637

3738
stages:
@@ -61,6 +62,7 @@ stages:
6162
# Other
6263
internalProjectName: ${{ parameters.internalProjectName }}
6364
publicProjectName: ${{ parameters.publicProjectName }}
65+
versionsRepoRef: ${{ parameters.versionsRepoRef }}
6466

6567
- template: /eng/common/templates/stages/dotnet/publish.yml@self
6668
parameters:
@@ -70,3 +72,4 @@ stages:
7072
internalProjectName: ${{ parameters.internalProjectName }}
7173
publicProjectName: ${{ parameters.publicProjectName }}
7274
sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
75+
versionsRepoRef: ${{ parameters.versionsRepoRef }}

0 commit comments

Comments
 (0)