Skip to content

Commit d0fdbe8

Browse files
scbeddscbedd
committed
pull across the unified yml changes for 1es. the holistic tests.yml changes will be coming in a follow-up manual commit to main
1 parent 3691357 commit d0fdbe8

27 files changed

+1441
-1233
lines changed

‎eng/CredScanSuppression.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
"sdk/confidentialledger/azure-confidentialledger/tests/_shared/constants.py",
3737
"sdk/keyvault/azure-keyvault-certificates/tests/ca.key",
3838
"sdk/identity/azure-identity/tests/certificate.pfx",
39+
"sdk/identity/azure-identity/tests/certificate.pem",
3940
"sdk/identity/azure-identity/tests/certificate-with-password.pfx",
4041
"sdk/identity/azure-identity/tests/credscan_ignore.py",
4142
"sdk/identity/azure-identity/tests/ec-certificate.pem",

‎eng/common/pipelines/templates/steps/publish-1es-artifact.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,10 @@ steps:
2222
}
2323
condition: and(succeededOrFailed(), ${{ parameters.CustomCondition }})
2424
displayName: Set Artifact Name $(Agent.JobStatus)
25-
2625
- task: 1ES.PublishPipelineArtifact@1
2726
condition: and(succeededOrFailed(), ${{ parameters.CustomCondition }})
2827
displayName: 'Publish ${{ parameters.ArtifactName }} Artifacts'
2928
inputs:
3029
artifactName: '$(PublishArtifactName)'
3130
targetPath: '${{ parameters.ArtifactPath }}'
32-
sbomEnabled: ${{ parameters.SbomEnabled }}
31+
sbomEnabled: ${{ parameters.SbomEnabled }}

‎eng/pipelines/templates/jobs/build-conda-dependencies.yml‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@ jobs:
99
timeoutInMinutes: 90
1010
pool:
1111
name: azsdk-pool-mms-win-2022-general
12-
vmImage: MMS2022
12+
image: azsdk-pool-mms-win-2022-1espt
13+
os: windows
1314
variables:
1415
VS_INSTALLER_URL: "https://aka.ms/vs/17/release/vs_enterprise.exe"
1516
VS_INSTALLER_PATH: "$(Build.BinariesDirectory)/vs_enterprise.exe"
@@ -62,7 +63,7 @@ jobs:
6263
sdk_build_conda -c %arguments%
6364
displayName: Assemble Conda Packages
6465
65-
- template: /eng/common/pipelines/templates/steps/publish-artifact.yml
66+
- template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
6667
parameters:
6768
ArtifactPath: '$(Build.SourcesDirectory)/conda/assembled'
6869
ArtifactName: 'windows_distributions'
@@ -74,7 +75,7 @@ jobs:
7475
inputs:
7576
BuildDropPath: '$(Build.SourcesDirectory)/conda/output'
7677

77-
- template: /eng/common/pipelines/templates/steps/publish-artifact.yml
78+
- template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
7879
parameters:
7980
ArtifactPath: '$(Build.SourcesDirectory)/conda/output'
8081
ArtifactName: 'windows_conda'
@@ -84,7 +85,9 @@ jobs:
8485
timeoutInMinutes: 90
8586
pool:
8687
name: azsdk-pool-mms-ubuntu-2004-general
87-
vmImage: MMSUbuntu20.04
88+
image: azsdk-pool-mms-ubuntu-2004-1espt
89+
os: linux
90+
8891
steps:
8992
- bash: |
9093
sudo apt-get install -y build-essential
@@ -101,6 +104,7 @@ jobs:
101104
pool:
102105
name: Azure Pipelines
103106
vmImage: macos-11
107+
os: macOS
104108
variables:
105109
MacOSXDeploymentTarget: '10.9'
106110
OpenSSLDir: $(Agent.BuildDirectory)/openssl-macosx$(MacOSXDeploymentTarget)

‎eng/pipelines/templates/jobs/ci.tests.yml‎

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,11 @@ parameters:
2929
- name: InjectedPackages
3030
type: string
3131
default: ''
32-
- name: Matrix
33-
type: string
3432
- name: DependsOn
33+
type: object
34+
default: []
35+
- name: Matrix
3536
type: string
36-
default: ''
3737
- name: UsePlatformContainer
3838
type: boolean
3939
default: false
@@ -43,6 +43,8 @@ parameters:
4343
- name: TestProxy
4444
type: boolean
4545
default: false
46+
- name: OSName
47+
type: string
4648

4749
jobs:
4850
- job:
@@ -56,32 +58,35 @@ jobs:
5658
timeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }}
5759

5860
dependsOn:
59-
- ${{ parameters.DependsOn }}
61+
- ${{ parameters.DependsOn }}
6062

6163
strategy:
6264
matrix: $[ ${{ parameters.Matrix }} ]
6365

6466
pool:
6567
name: $(Pool)
66-
vmImage: $(OSVmImage)
67-
68-
${{ if eq(parameters.UsePlatformContainer, 'true') }}:
69-
# Add a default so the job doesn't fail when the matrix is empty
70-
container: $[ variables['Container'] ]
68+
# 1es pipeline templates converts `image` to demands: ImageOverride under the hood
69+
# which is incompatible with image selection in the default non-1es hosted pools
70+
${{ if eq(parameters.OSName, 'macOS') }}:
71+
vmImage: $(OSVmImage)
72+
${{ else }}:
73+
image: $(OSVmImage)
74+
os: ${{ parameters.OSName }}
7175

7276
variables:
7377
- template: ../variables/globals.yml
7478
- name: InjectedPackages
7579
value: ${{ parameters.InjectedPackages }}
7680

81+
7782
steps:
7883
- template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
7984
parameters:
8085
BuildTargetingString: ${{ parameters.BuildTargetingString }}
8186

8287
- template: /eng/common/pipelines/templates/steps/verify-agent-os.yml
8388
parameters:
84-
AgentImage: $(OSVmImage)
89+
AgentImage: ${{ parameters.OSName }}
8590

8691
- task: UsePythonVersion@0
8792
inputs:
@@ -111,7 +116,6 @@ jobs:
111116
ServiceDirectory: ${{ parameters.ServiceDirectory }}
112117
TestMarkArgument: ${{ parameters.TestMarkArgument }}
113118
AdditionalTestArgs: '--wheel_dir="$(Build.ArtifactStagingDirectory)"'
114-
OSVmImage: $(OSVmImage)
115119
CoverageArg: $(CoverageArg)
116120
PythonVersion: $(PythonVersion)
117121
ToxTestEnv: $(toxenv)
@@ -127,7 +131,7 @@ jobs:
127131
- template: ../steps/set-dev-build.yml
128132
parameters:
129133
ServiceDirectory: ${{ parameters.ServiceDirectory }}
130-
134+
131135
- ${{ each step in parameters.BeforeTestSteps }}:
132136
- ${{ step }}
133137
AfterTestSteps: ${{ parameters.AfterTestSteps }}

‎eng/pipelines/templates/jobs/ci.yml‎

Lines changed: 78 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -66,72 +66,104 @@ parameters:
6666
default: false
6767

6868
jobs:
69-
69+
7070
- ${{ if eq(parameters['AdvancedBuild'], false) }}:
7171
- job: 'Build'
7272
timeoutInMinutes: 90
7373

7474
pool:
7575
name: azsdk-pool-mms-ubuntu-2004-general
76-
vmImage: MMSUbuntu20.04
76+
image: azsdk-pool-mms-ubuntu-2004-1espt
77+
os: linux
7778

7879
steps:
79-
- template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
80+
- template: /eng/pipelines/templates/steps/targeting-string-resolve.yml@self
8081
parameters:
8182
BuildTargetingString: ${{ parameters.BuildTargetingString }}
8283

83-
- template: ../steps/build-package-artifacts.yml
84+
- template: ../steps/build-package-artifacts.yml@self
8485
parameters:
8586
ServiceDirectory: ${{ parameters.ServiceDirectory }}
8687
BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
8788
TestPipeline: ${{ parameters.TestPipeline }}
8889
Artifacts: ${{ parameters.Artifacts }}
8990

9091
- ${{ if eq(parameters['AdvancedBuild'], true) }}:
91-
- job: 'Build'
92+
- job: 'Build_Linux'
9293
timeoutInMinutes: 90
9394

94-
strategy:
95-
matrix:
96-
Linux:
97-
imageName: 'MMSUbuntu22.04'
98-
poolName: 'azsdk-pool-mms-ubuntu-2204-general'
99-
ArtifactName: 'linux'
100-
Windows:
101-
imageName: 'MMS2022'
102-
poolName: 'azsdk-pool-mms-win-2022-general'
103-
ArtifactName: 'windows'
104-
Mac:
105-
imageName: 'macos-11'
106-
poolName: 'Azure Pipelines'
107-
ArtifactName: 'mac'
95+
pool:
96+
name: azsdk-pool-mms-ubuntu-2004-general
97+
image: azsdk-pool-mms-ubuntu-2004-1espt
98+
os: linux
99+
100+
steps:
101+
- template: /eng/pipelines/templates/steps/targeting-string-resolve.yml@self
102+
parameters:
103+
BuildTargetingString: ${{ parameters.BuildTargetingString }}
104+
105+
- template: ../steps/build-package-artifacts.yml@self
106+
parameters:
107+
ServiceDirectory: ${{ parameters.ServiceDirectory }}
108+
BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
109+
TestPipeline: ${{ parameters.TestPipeline }}
110+
Artifacts: ${{ parameters.Artifacts }}
111+
ArtifactSuffix: linux
112+
113+
- job: 'Build_Windows'
114+
timeoutInMinutes: 90
108115

109116
pool:
110-
name: $(poolName)
111-
vmImage: $(imageName)
117+
name: azsdk-pool-mms-win-2022-general
118+
image: azsdk-pool-mms-win-2022-1espt
119+
os: windows
112120

113121
steps:
114-
- template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
122+
- template: /eng/pipelines/templates/steps/targeting-string-resolve.yml@self
115123
parameters:
116124
BuildTargetingString: ${{ parameters.BuildTargetingString }}
117125

118-
- template: ../steps/build-package-artifacts.yml
126+
- template: ../steps/build-package-artifacts.yml@self
119127
parameters:
120128
ServiceDirectory: ${{ parameters.ServiceDirectory }}
121129
BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
122130
TestPipeline: ${{ parameters.TestPipeline }}
123131
Artifacts: ${{ parameters.Artifacts }}
124-
ArtifactSuffix: $(ArtifactName)
132+
ArtifactSuffix: windows
133+
134+
- job: 'Build_MacOS'
135+
timeoutInMinutes: 90
136+
137+
pool:
138+
name: 'Azure Pipelines'
139+
vmImage: macos-11
140+
os: macOS
141+
142+
steps:
143+
- template: /eng/pipelines/templates/steps/targeting-string-resolve.yml@self
144+
parameters:
145+
BuildTargetingString: ${{ parameters.BuildTargetingString }}
146+
147+
- template: ../steps/build-package-artifacts.yml@self
148+
parameters:
149+
ServiceDirectory: ${{ parameters.ServiceDirectory }}
150+
BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
151+
TestPipeline: ${{ parameters.TestPipeline }}
152+
Artifacts: ${{ parameters.Artifacts }}
153+
ArtifactSuffix: mac
125154

126155
- job: 'CoalesceBuildArtifacts'
127156
displayName: Combine Built Artifacts
128-
dependsOn:
129-
- 'Build'
157+
dependsOn:
158+
- 'Build_Linux'
159+
- 'Build_Windows'
160+
- 'Build_MacOS'
130161
timeoutInMinutes: 90
131162

132163
pool:
133164
name: azsdk-pool-mms-ubuntu-2004-general
134-
vmImage: MMSUbuntu20.04
165+
image: azsdk-pool-mms-ubuntu-2004-1espt
166+
os: linux
135167

136168
steps:
137169
- task: DownloadPipelineArtifact@2
@@ -149,22 +181,25 @@ jobs:
149181
artifactName: 'packages_linux'
150182
targetPath: $(Build.ArtifactStagingDirectory)/packages
151183

152-
- template: /eng/common/pipelines/templates/steps/publish-artifact.yml
184+
- template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
153185
parameters:
154186
ArtifactPath: '$(Build.ArtifactStagingDirectory)/packages'
155187
ArtifactName: 'packages'
156188

157189
- job: 'Build_Extended'
158190
displayName: Build Extended
159-
dependsOn:
160-
- 'Build'
191+
dependsOn:
161192
- ${{ if eq(parameters['AdvancedBuild'], true) }}:
162193
- 'CoalesceBuildArtifacts'
194+
- ${{ else }}:
195+
- 'Build'
196+
163197
timeoutInMinutes: 90
164198

165199
pool:
166200
name: azsdk-pool-mms-ubuntu-2004-general
167-
vmImage: MMSUbuntu20.04
201+
image: azsdk-pool-mms-ubuntu-2004-1espt
202+
os: linux
168203

169204
steps:
170205
- template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
@@ -189,7 +224,8 @@ jobs:
189224

190225
pool:
191226
name: azsdk-pool-mms-ubuntu-2004-general
192-
vmImage: MMSUbuntu20.04
227+
image: azsdk-pool-mms-ubuntu-2004-1espt
228+
os: linux
193229

194230
steps:
195231
- template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
@@ -220,35 +256,16 @@ jobs:
220256
ValidateFormatting: ${{ parameters.ValidateFormatting }}
221257
GenerateApiReviewForManualOnly: ${{ parameters.GenerateApiReviewForManualOnly }}
222258

223-
- job: Compliance
224-
pool:
225-
name: azsdk-pool-mms-win-2022-general
226-
vmImage: MMS2022
227-
228-
variables:
229-
Codeql.SkipTaskAutoInjection: false
230-
Codeql.Enabled: true
231-
Codeql.Language: python
232-
Codeql.BuildIdentifier: "${{ parameters.ServiceDirectory }}"
233-
Codeql.SourceRoot: "sdk/${{ parameters.ServiceDirectory }}"
234-
235-
# per the guidance of the codeql team:
236-
# https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/codeql/snippets/codeql-3000-other-issues#timeouts
237-
timeoutInMinutes: 360
238-
239-
steps:
240-
- template: /eng/common/pipelines/templates/steps/credscan.yml
241-
parameters:
242-
ServiceDirectory: ${{ parameters.ServiceDirectory }}
243-
BaselineFilePath: $(Build.SourcesDirectory)\eng\python.gdnbaselines
244-
245-
- template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml
259+
- template: /eng/common/pipelines/templates/jobs/generate-job-matrix.yml
246260
parameters:
247261
JobTemplatePath: /eng/pipelines/templates/jobs/ci.tests.yml
262+
OsVmImage: azsdk-pool-mms-ubuntu-2004-1espt
263+
Pool: azsdk-pool-mms-ubuntu-2004-general
248264
DependsOn:
249-
- 'Build'
250265
- ${{ if eq(parameters['AdvancedBuild'], true) }}:
251266
- 'CoalesceBuildArtifacts'
267+
- ${{ else }}:
268+
- 'Build'
252269
MatrixConfigs: ${{ parameters.MatrixConfigs }}
253270
MatrixFilters: ${{ parameters.MatrixFilters }}
254271
MatrixReplace: ${{ parameters.MatrixReplace }}
@@ -267,9 +284,11 @@ jobs:
267284
UnsupportedToxEnvironments: ${{ parameters.UnsupportedToxEnvironments }}
268285
TestProxy: ${{ parameters.TestProxy }}
269286

270-
- template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml
287+
- template: /eng/common/pipelines/templates/jobs/generate-job-matrix.yml
271288
parameters:
272289
JobTemplatePath: /eng/pipelines/templates/jobs/regression.yml
290+
OsVmImage: azsdk-pool-mms-ubuntu-2004-1espt
291+
Pool: azsdk-pool-mms-ubuntu-2004-general
273292
GenerateJobName: generate_regression_matrix
274293
SparseCheckoutPaths: [ "scripts/", "sdk/", "tools/azure-sdk-tools/" ]
275294
MatrixConfigs:
@@ -295,9 +314,10 @@ jobs:
295314
CloudConfig:
296315
Cloud: Public
297316
DependsOn:
298-
- 'Build'
299317
- ${{ if eq(parameters['AdvancedBuild'], true) }}:
300318
- 'CoalesceBuildArtifacts'
319+
- ${{ else }}:
320+
- 'Build'
301321
AdditionalParameters:
302322
BuildTargetingString: ${{ parameters.BuildTargetingString }}
303323
ServiceDirectory: ${{ parameters.ServiceDirectory }}

0 commit comments

Comments
 (0)