Skip to content

Commit 7d4d569

Browse files
dotnet-docker-botdotnet-docker-bot
authored
[main] Update common Docker engineering infrastructure with latest (#5410)
1 parent d2797c0 commit 7d4d569

19 files changed

+340
-156
lines changed

‎eng/common/templates/1es-unofficial.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ extends:
4444
ignoreDirectories: $(Build.SourcesDirectory)/versions
4545
whatIf: true
4646
showAlertLink: true
47+
sbom:
48+
enabled: true
4749
sourceRepositoriesToScan:
4850
exclude:
4951
- repository: InternalVersionsRepo

‎eng/common/templates/jobs/build-images.yml‎

Lines changed: 22 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ jobs:
9797
New-Item -Path $(imageInfoHostDir) -ItemType Directory -Force
9898
$imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $(imageBuilder.queueArgs) --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json"
9999
if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest") {
100-
$imageBuilderBuildArgs = "$imageBuilderBuildArgs --registry-override $(acr.server) --repo-prefix $(stagingRepoPrefix) --source-repo-prefix $(mirrorRepoPrefix) --push --registry-creds ""$(acr.server)=$(acr.userName);$(acr.password)"""
100+
$imageBuilderBuildArgs = "$imageBuilderBuildArgs --registry-override $(acr.server) --repo-prefix $(stagingRepoPrefix) --source-repo-prefix $(mirrorRepoPrefix) --push"
101101
}
102102
103103
# If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache
@@ -107,25 +107,27 @@ jobs:
107107
108108
echo "##vso[task.setvariable variable=imageBuilderBuildArgs]$imageBuilderBuildArgs"
109109
displayName: Set Image Builder Build Args
110-
- powershell: >
111-
$(runImageBuilderCmd) build
112-
--manifest $(manifest)
113-
$(imageBuilderPaths)
114-
$(osVersions)
115-
--os-type $(osType)
116-
--architecture $(architecture)
117-
--retry
118-
--source-repo $(publicGitRepoUri)
119-
--digests-out-var 'builtImages'
120-
--acr-subscription '$(acr.subscription)'
121-
--acr-resource-group '$(acr.resourceGroup)'
122-
--acr-client-id '$(acr.servicePrincipalName)'
123-
--acr-password '$(acr.servicePrincipalPassword)'
124-
--acr-tenant '$(acr.servicePrincipalTenant)'
125-
$(manifestVariables)
126-
$(imageBuilderBuildArgs)
127-
name: BuildImages
128-
displayName: Build Images
110+
- template: /eng/common/templates/steps/run-imagebuilder.yml@self
111+
parameters:
112+
name: BuildImages
113+
displayName: Build Images
114+
serviceConnection: $(acr.serviceConnectionName)
115+
internalProjectName: ${{ parameters.internalProjectName }}
116+
dockerClientOS: ${{ parameters.dockerClientOS }}
117+
args: >
118+
build
119+
--manifest $(manifest)
120+
$(imageBuilderPaths)
121+
$(osVersions)
122+
--os-type $(osType)
123+
--architecture $(architecture)
124+
--retry
125+
--source-repo $(publicGitRepoUri)
126+
--digests-out-var 'builtImages'
127+
--acr-subscription '$(acr.subscription)'
128+
--acr-resource-group '$(acr.resourceGroup)'
129+
$(manifestVariables)
130+
$(imageBuilderBuildArgs)
129131
- template: /eng/common/templates/steps/publish-artifact.yml@self
130132
parameters:
131133
path: $(imageInfoHostDir)

‎eng/common/templates/jobs/copy-base-images.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ parameters:
33
pool: {}
44
additionalOptions: null
55
publicProjectName: null
6+
internalProjectName: null
67
customInitSteps: []
78

89
jobs:
@@ -15,4 +16,5 @@ jobs:
1516
parameters:
1617
additionalOptions: ${{ parameters.additionalOptions }}
1718
publicProjectName: ${{ parameters.publicProjectName }}
19+
internalProjectName: ${{ parameters.internalProjectName }}
1820
continueOnError: true

‎eng/common/templates/jobs/publish.yml‎

Lines changed: 54 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -64,33 +64,38 @@ jobs:
6464
$(runImageBuilderCmd) trimUnchangedPlatforms
6565
'$(imageInfoContainerDir)/image-info.json'
6666
displayName: Trim Unchanged Images
67-
- script: >
68-
$(runImageBuilderCmd) copyAcrImages
69-
'$(acr.servicePrincipalName)'
70-
'$(acr.servicePrincipalPassword)'
71-
'$(acr.servicePrincipalTenant)'
72-
'$(acr.subscription)'
73-
'$(acr.resourceGroup)'
74-
'$(stagingRepoPrefix)'
75-
--os-type '*'
76-
--architecture '*'
77-
--repo-prefix '$(publishRepoPrefix)'
78-
--image-info '$(imageInfoContainerDir)/image-info.json'
79-
$(dryRunArg)
80-
$(imageBuilder.pathArgs)
81-
$(imageBuilder.commonCmdArgs)
82-
displayName: Copy Images
83-
- script: >
84-
$(runImageBuilderCmd) publishManifest
85-
'$(imageInfoContainerDir)/image-info.json'
86-
--repo-prefix '$(publishRepoPrefix)'
87-
--registry-creds '$(acr.server)=$(acr.userName);$(acr.password)'
88-
--os-type '*'
89-
--architecture '*'
90-
$(dryRunArg)
91-
$(imageBuilder.pathArgs)
92-
$(imageBuilder.commonCmdArgs)
93-
displayName: Publish Manifest
67+
- template: /eng/common/templates/steps/run-imagebuilder.yml@self
68+
parameters:
69+
displayName: Copy Images
70+
serviceConnection: $(acr.serviceConnectionName)
71+
internalProjectName: ${{ parameters.internalProjectName }}
72+
args: >
73+
copyAcrImages
74+
'$(acr.subscription)'
75+
'$(acr.resourceGroup)'
76+
'$(stagingRepoPrefix)'
77+
--os-type '*'
78+
--architecture '*'
79+
--repo-prefix '$(publishRepoPrefix)'
80+
--image-info '$(imageInfoContainerDir)/image-info.json'
81+
$(dryRunArg)
82+
$(imageBuilder.pathArgs)
83+
$(imageBuilder.commonCmdArgs)
84+
- template: /eng/common/templates/steps/run-imagebuilder.yml@self
85+
parameters:
86+
displayName: Publish Manifest
87+
serviceConnection: $(acr.serviceConnectionName)
88+
internalProjectName: ${{ parameters.internalProjectName }}
89+
dockerClientOS: ${{ parameters.dockerClientOS }}
90+
args: >
91+
publishManifest
92+
'$(imageInfoContainerDir)/image-info.json'
93+
--repo-prefix '$(publishRepoPrefix)'
94+
--os-type '*'
95+
--architecture '*'
96+
$(dryRunArg)
97+
$(imageBuilder.pathArgs)
98+
$(imageBuilder.commonCmdArgs)
9499
- template: /eng/common/templates/steps/publish-artifact.yml@self
95100
parameters:
96101
path: $(imageInfoHostDir)
@@ -122,22 +127,23 @@ jobs:
122127
$(imageBuilder.commonCmdArgs)
123128
condition: and(succeeded(), eq(variables['publishImageInfo'], 'true'))
124129
displayName: Publish Image Info
125-
- script: >
126-
$(runImageBuilderCmd) ingestKustoImageInfo
127-
'$(imageInfoContainerDir)/image-info.json'
128-
'$(kusto.cluster)'
129-
'$(kusto.database)'
130-
'$(kusto.imageTable)'
131-
'$(kusto.layerTable)'
132-
'$(kusto.servicePrincipalName)'
133-
'$(kusto.servicePrincipalPassword)'
134-
'$(kusto.servicePrincipalTenant)'
135-
--os-type '*'
136-
--architecture '*'
137-
$(dryRunArg)
138-
$(imageBuilder.commonCmdArgs)
139-
displayName: Ingest Kusto Image Info
140-
condition: and(succeeded(), eq(variables['ingestKustoImageInfo'], 'true'))
130+
- template: /eng/common/templates/steps/run-imagebuilder.yml@self
131+
parameters:
132+
displayName: Ingest Kusto Image Info
133+
serviceConnection: $(kusto.serviceConnectionName)
134+
internalProjectName: ${{ parameters.internalProjectName }}
135+
condition: and(succeeded(), eq(variables['ingestKustoImageInfo'], 'true'))
136+
args: >
137+
ingestKustoImageInfo
138+
'$(imageInfoContainerDir)/image-info.json'
139+
'$(kusto.cluster)'
140+
'$(kusto.database)'
141+
'$(kusto.imageTable)'
142+
'$(kusto.layerTable)'
143+
--os-type '*'
144+
--architecture '*'
145+
$(dryRunArg)
146+
$(imageBuilder.commonCmdArgs)
141147
- script: >
142148
$(runImageBuilderCmd) postPublishNotification
143149
'$(publishNotificationRepoName)'
@@ -151,13 +157,13 @@ jobs:
151157
'$(gitHubNotificationsRepoInfo.org)'
152158
'$(gitHubNotificationsRepoInfo.repo)'
153159
--repo-prefix '$(publishRepoPrefix)'
154-
--task "Copy Images"
155-
--task "Publish Manifest"
156-
--task "Wait for Image Ingestion"
160+
--task "Copy Images (Authenticated)"
161+
--task "Publish Manifest (Authenticated)"
162+
--task "Wait for Image Ingestion (Authenticated)"
157163
--task "Publish Readmes"
158-
--task "Wait for MCR Doc Ingestion"
164+
--task "Wait for MCR Doc Ingestion (Authenticated)"
159165
--task "Publish Image Info"
160-
--task "Ingest Kusto Image Info"
166+
--task "Ingest Kusto Image Info (Authenticated)"
161167
$(dryRunArg)
162168
$(imageBuilder.commonCmdArgs)
163169
displayName: Post Publish Notification

‎eng/common/templates/stages/build-test-publish-repo.yml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ parameters:
88
customTestInitSteps: []
99
customPublishInitSteps: []
1010
customPublishVariables: []
11-
11+
1212
linuxAmdBuildJobTimeout: 60
1313
linuxArmBuildJobTimeout: 60
1414
windowsAmdBuildJobTimeout: 60
@@ -70,7 +70,8 @@ stages:
7070
pool: ${{ parameters.linuxAmd64Pool }}
7171
additionalOptions: "--manifest '$(manifest)' $(imageBuilder.pathArgs) $(manifestVariables)"
7272
publicProjectName: ${{ parameters.publicProjectName }}
73-
customInitSteps: ${{ parameters.customCopyBaseImagesInitSteps}}
73+
internalProjectName: ${{ parameters.internalProjectName }}
74+
customInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }}
7475
- template: /eng/common/templates/jobs/generate-matrix.yml@self
7576
parameters:
7677
matrixType: ${{ parameters.buildMatrixType }}

‎eng/common/templates/steps/clean-acr-images.yml‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,19 @@ parameters:
33
action: null
44
age: null
55
customArgs: ""
6+
internalProjectName: null
67
steps:
7-
- script: >
8-
$(runImageBuilderCmd) cleanAcrImages
9-
${{ parameters.repo }}
10-
$(acr.servicePrincipalName)
11-
$(app-dotnetdockerbuild-client-secret)
12-
$(acr.servicePrincipalTenant)
13-
$(acr.subscription)
14-
$(acr.resourceGroup)
15-
$(acr.server)
16-
--action ${{ parameters.action }}
17-
--age ${{ parameters.age }}
18-
${{ parameters.customArgs }}
19-
displayName: Clean ACR Images - ${{ parameters.repo }}
8+
- template: /eng/common/templates/steps/run-imagebuilder.yml@self
9+
parameters:
10+
displayName: Clean ACR Images - ${{ parameters.repo }}
11+
serviceConnection: $(acr.serviceConnectionName)
12+
internalProjectName: ${{ parameters.internalProjectName }}
13+
args: >
14+
cleanAcrImages
15+
${{ parameters.repo }}
16+
$(acr.subscription)
17+
$(acr.resourceGroup)
18+
$(acr.server)
19+
--action ${{ parameters.action }}
20+
--age ${{ parameters.age }}
21+
${{ parameters.customArgs }}

‎eng/common/templates/steps/copy-base-images.yml‎

Lines changed: 22 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,30 @@
11
parameters:
22
additionalOptions: null
33
publicProjectName: null
4+
internalProjectName: null
45
continueOnError: false
56

67
steps:
78
- ${{ if or(eq(variables['System.TeamProject'], parameters.publicProjectName), eq(variables['Build.Reason'], 'PullRequest')) }}:
89
- template: /eng/common/templates/steps/set-dry-run.yml@self
9-
- script: >
10-
$(runImageBuilderCmd)
11-
copyBaseImages
12-
'$(acr.servicePrincipalName)'
13-
'$(acr.servicePrincipalPassword)'
14-
'$(acr.servicePrincipalTenant)'
15-
'$(acr.subscription)'
16-
'$(acr.resourceGroup)'
17-
$(dockerHubRegistryCreds)
18-
$(customCopyBaseImagesArgs)
19-
--repo-prefix $(mirrorRepoPrefix)
20-
--registry-override '$(acr.server)'
21-
--os-type 'linux'
22-
--architecture '*'
23-
$DRYRUNARG
24-
${{ parameters.additionalOptions }}
25-
displayName: Copy Base Images
26-
continueOnError: ${{ parameters.continueOnError }}
10+
- template: /eng/common/templates/steps/run-imagebuilder.yml@self
11+
parameters:
12+
displayName: Copy Base Images
13+
serviceConnection: $(acr.serviceConnectionName)
14+
continueOnError: ${{ parameters.continueOnError }}
15+
internalProjectName: ${{ parameters.internalProjectName }}
16+
# Use environment variable to reference $(dryRunArg). Since $(dryRunArg) might be undefined,
17+
# PowerShell will treat the Azure Pipelines variable macro syntax as a command and throw an
18+
# error
19+
args: >
20+
copyBaseImages
21+
'$(acr.subscription)'
22+
'$(acr.resourceGroup)'
23+
$(dockerHubRegistryCreds)
24+
$(customCopyBaseImagesArgs)
25+
--repo-prefix $(mirrorRepoPrefix)
26+
--registry-override '$(acr.server)'
27+
--os-type 'linux'
28+
--architecture '*'
29+
$env:DRYRUNARG
30+
${{ parameters.additionalOptions }}

‎eng/common/templates/steps/init-docker-linux.yml‎

Lines changed: 38 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -34,16 +34,45 @@ steps:
3434
-f $(engCommonPath)/Dockerfile.WithRepo .
3535
displayName: Build Image for Image Builder
3636
condition: and(succeeded(), ${{ parameters.condition }})
37-
- script: >
38-
echo "##vso[task.setvariable variable=runImageBuilderCmd]
39-
docker run --rm
40-
-v /var/run/docker.sock:/var/run/docker.sock
41-
-v $(Build.ArtifactStagingDirectory):$(artifactsPath)
42-
-w /repo
43-
$(imageBuilderDockerRunExtraOptions)
44-
$(imageNames.imageBuilder.withrepo)"
45-
displayName: Define runImageBuilderCmd Variable
37+
- task: PowerShell@2
38+
displayName: Define ImageBuilder Command Variables
4639
condition: and(succeeded(), ${{ parameters.condition }})
40+
inputs:
41+
targetType: 'inline'
42+
script: |
43+
$tokenHostPath = '$(Agent.TempDirectory)'
44+
$tokenHostFilePath = "${tokenHostPath}/token"
45+
$tokenContainerPath = "/tmp"
46+
$tokenContainerFilePath = "${tokenContainerPath}/token"
47+
48+
$dockerRunBaseCmd = @(
49+
"docker run --rm"
50+
)
51+
52+
$dockerRunArgs = @(
53+
"-v /var/run/docker.sock:/var/run/docker.sock"
54+
"-v $(Build.ArtifactStagingDirectory):$(artifactsPath)"
55+
"-w /repo"
56+
"$(imageBuilderDockerRunExtraOptions)"
57+
"$(imageNames.imageBuilder.withrepo)"
58+
)
59+
60+
$authedDockerRunArgs = @(
61+
'-e AZURE_TENANT_ID=$env:tenantId'
62+
'-e AZURE_CLIENT_ID=$env:servicePrincipalId'
63+
"-e AZURE_FEDERATED_TOKEN_FILE=$tokenContainerFilePath"
64+
"-v ${tokenHostPath}:${tokenContainerPath}"
65+
)
66+
67+
$dockerRunCmd = $dockerRunBaseCmd + $dockerRunArgs
68+
$authedDockerRunCmd = $dockerRunBaseCmd + $authedDockerRunArgs + $dockerRunArgs
69+
70+
$runImageBuilderCmd = $($dockerRunCmd -join ' ')
71+
$runAuthedImageBuilderCmd = $($authedDockerRunCmd -join ' ')
72+
73+
Write-Host "##vso[task.setvariable variable=runImageBuilderCmd]$runImageBuilderCmd"
74+
Write-Host "##vso[task.setvariable variable=runAuthedImageBuilderCmd]$runAuthedImageBuilderCmd"
75+
Write-Host "##vso[task.setvariable variable=tokenHostFilePath]$tokenHostFilePath"
4776
4877
################################################################################
4978
# Setup Test Runner (Optional)

‎eng/common/templates/steps/init-docker-windows.yml‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,26 @@ steps:
3737
displayName: Cleanup Setup Container
3838
condition: and(always(), ${{ parameters.condition }})
3939
continueOnError: true
40-
- powershell: >
41-
echo "##vso[task.setvariable variable=runImageBuilderCmd]
42-
$(Build.BinariesDirectory)\.Microsoft.DotNet.ImageBuilder\Microsoft.DotNet.ImageBuilder.exe"
43-
displayName: Define runImageBuilderCmd Variable
40+
- task: PowerShell@2
41+
displayName: Define runImageBuilderCmd Variables
4442
condition: and(succeeded(), ${{ parameters.condition }})
43+
inputs:
44+
targetType: 'inline'
45+
script: |
46+
$tokenHostPath = '$(Agent.TempDirectory)'
47+
$tokenHostFilePath = "$tokenHostPath\token"
48+
49+
$runImageBuilderCmd = "$(Build.BinariesDirectory)\.Microsoft.DotNet.ImageBuilder\Microsoft.DotNet.ImageBuilder.exe"
50+
51+
$authedImageBuilderCmds = @(
52+
'$env:AZURE_TENANT_ID = $env:tenantId'
53+
'$env:AZURE_CLIENT_ID = $env:servicePrincipalId'
54+
'$env:AZURE_FEDERATED_TOKEN_FILE' + " = $tokenHostFilePath"
55+
$runImageBuilderCmd
56+
)
57+
58+
$runAuthedImageBuilderCmd = $($authedImageBuilderCmds -join "; ")
59+
60+
Write-Host "##vso[task.setvariable variable=runImageBuilderCmd]$runImageBuilderCmd"
61+
Write-Host "##vso[task.setvariable variable=runAuthedImageBuilderCmd]$runAuthedImageBuilderCmd"
62+
Write-Host "##vso[task.setvariable variable=tokenHostFilePath]$tokenHostFilePath"

0 commit comments

Comments
 (0)