Skip to content

Commit 4d8b372

Browse files
azure-sdkbenbp
azure-sdk
and
benbp
authored
Sync eng/common directory with azure-sdk-tools for PR 7758 (#34609)
* Add 1es changes for job/matrix generation and publish * Use more flexible pool filter for prev/next pool matches * consolidate displayname definition * use linux pool variables for generate matrix job * Fix publish artifact * Use single publish task for publish artifact --------- Co-authored-by: Ben Broderick Phillips <[email protected]>
1 parent 605b13a commit 4d8b372

File tree

2 files changed

+170
-0
lines changed

2 files changed

+170
-0
lines changed

‎eng/common/pipelines/templates/jobs/generate-job-matrix.yml‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
parameters:
2+
- name: AdditionalParameters
3+
type: object
4+
- name: DependsOn
5+
type: object
6+
default: null
7+
- name: CloudConfig
8+
type: object
9+
default: {}
10+
- name: MatrixConfigs
11+
type: object
12+
default: []
13+
- name: MatrixFilters
14+
type: object
15+
default: []
16+
- name: MatrixReplace
17+
type: object
18+
default: {}
19+
- name: JobTemplatePath
20+
type: string
21+
# Set this to false to do a full checkout for private repositories with the azure pipelines service connection
22+
- name: SparseCheckout
23+
type: boolean
24+
default: true
25+
- name: SparseCheckoutPaths
26+
type: object
27+
default: []
28+
- name: Pool
29+
type: string
30+
default: $(LINUXPOOL)
31+
- name: OsVmImage
32+
type: string
33+
default: $(LINUXVMIMAGE)
34+
- name: Os
35+
type: string
36+
default: linux
37+
# This parameter is only necessary if there are multiple invocations of this template within the SAME STAGE.
38+
# When that occurs, provide a name other than the default value.
39+
- name: GenerateJobName
40+
type: string
41+
default: 'generate_job_matrix'
42+
- name: PreGenerationSteps
43+
type: stepList
44+
default: []
45+
# Mappings to OS name required at template compile time by 1es pipeline templates
46+
- name: Pools
47+
type: object
48+
default:
49+
- name: Linux
50+
filter: .*Linux.*Pool$
51+
os: linux
52+
- name: Windows
53+
filter: .*Windows.*Pool$
54+
os: windows
55+
- name: Mac
56+
filter: .*MacPool$
57+
os: macOS
58+
59+
jobs:
60+
- job: ${{ parameters.GenerateJobName }}
61+
variables:
62+
- template: /eng/pipelines/templates/variables/image.yml
63+
- name: skipComponentGovernanceDetection
64+
value: true
65+
- name: displayNameFilter
66+
value: $[ coalesce(variables.jobMatrixFilter, '.*') ]
67+
pool:
68+
name: ${{ parameters.Pool }}
69+
vmImage: ${{ parameters.OsVmImage }}
70+
os: ${{ parameters.Os }}
71+
${{ if parameters.DependsOn }}:
72+
dependsOn: ${{ parameters.DependsOn }}
73+
steps:
74+
# Skip sparse checkout for the `azure-sdk-for-<lang>-pr` private mirrored repositories
75+
# as we require the github service connection to be loaded.
76+
- ${{ if and(parameters.SparseCheckout, not(contains(variables['Build.DefinitionName'], '-pr - '))) }}:
77+
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
78+
parameters:
79+
${{ if ne(length(parameters.SparseCheckoutPaths), 0) }}:
80+
Paths: ${{ parameters.SparseCheckoutPaths }}
81+
${{ if and(eq(length(parameters.SparseCheckoutPaths), 0), ne(parameters.AdditionalParameters.ServiceDirectory, '')) }}:
82+
Paths:
83+
- "sdk/${{ parameters.AdditionalParameters.ServiceDirectory }}"
84+
85+
- ${{ parameters.PreGenerationSteps }}
86+
87+
- ${{ each config in parameters.MatrixConfigs }}:
88+
- ${{ each pool in parameters.Pools }}:
89+
- ${{ if eq(config.GenerateVMJobs, 'true') }}:
90+
- task: Powershell@2
91+
inputs:
92+
pwsh: true
93+
filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
94+
arguments: >
95+
-ConfigPath ${{ config.Path }}
96+
-Selection ${{ config.Selection }}
97+
-DisplayNameFilter '$(displayNameFilter)'
98+
-Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
99+
-Replace '${{ join(''',''', parameters.MatrixReplace) }}'
100+
-NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
101+
displayName: Create ${{ pool.name }} Matrix ${{ config.Name }}
102+
name: vm_job_matrix_${{ config.Name }}_${{ pool.name }}
103+
104+
- ${{ if eq(config.GenerateContainerJobs, 'true') }}:
105+
- task: Powershell@2
106+
inputs:
107+
pwsh: true
108+
filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
109+
arguments: >
110+
-ConfigPath ${{ config.Path }}
111+
-Selection ${{ config.Selection }}
112+
-DisplayNameFilter '$(displayNameFilter)'
113+
-Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
114+
-NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
115+
displayName: Create ${{ pool.name }} Container Matrix ${{ config.Name }}
116+
name: container_job_matrix_${{ config.Name }}_${{ pool.name }}
117+
118+
- ${{ each config in parameters.MatrixConfigs }}:
119+
- ${{ each pool in parameters.Pools }}:
120+
- ${{ if eq(config.GenerateVMJobs, 'true') }}:
121+
- template: ${{ parameters.JobTemplatePath }}
122+
parameters:
123+
UsePlatformContainer: false
124+
OSName: ${{ pool.os }}
125+
Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
126+
DependsOn: ${{ parameters.GenerateJobName }}
127+
CloudConfig: ${{ parameters.CloudConfig }}
128+
${{ each param in parameters.AdditionalParameters }}:
129+
${{ param.key }}: ${{ param.value }}
130+
131+
- ${{ if eq(config.GenerateContainerJobs, 'true') }}:
132+
- template: ${{ parameters.JobTemplatePath }}
133+
parameters:
134+
UsePlatformContainer: true
135+
OSName: ${{ pool.os }}
136+
Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
137+
DependsOn: ${{ parameters.GenerateJobName }}
138+
CloudConfig: ${{ parameters.CloudConfig }}
139+
${{ each param in parameters.AdditionalParameters }}:
140+
${{ param.key }}: ${{ param.value }}

‎eng/common/pipelines/templates/steps/publish-1es-artifact.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# This step is used to prevent duplication of artifact publishes when there is an issue that would prevent the overall success of the job.
2+
# Ensuring that we only publish when successful (and two a differently named artifact otherwise) will allow easy retry on a build pipeline
3+
# without running into the "cannot override artifact" failure when we finally do get a passing run.
4+
5+
# ArtifactName - The name of the artifact in the "successful" case.
6+
# ArtifactPath - The path we will be publishing.
7+
# CustomCondition - Used if there is additional logic necessary to prevent attempt of publish.
8+
9+
parameters:
10+
ArtifactName: ''
11+
ArtifactPath: ''
12+
CustomCondition: true
13+
14+
steps:
15+
- pwsh: |
16+
Write-Host "##vso[task.setvariable variable=PublishArtifactName;]${{ parameters.ArtifactName }}"
17+
condition: and(succeeded(), ${{ parameters.CustomCondition }})
18+
displayName: Set Artifact Name
19+
20+
- pwsh: |
21+
Write-Host "##vso[task.setvariable variable=PublishArtifactName;]${{ parameters.ArtifactName }}-FailedAttempt$(System.JobAttempt)"
22+
condition: and(failed(), ${{ parameters.CustomCondition }})
23+
displayName: Set Failed Artifact Name
24+
25+
- task: 1ES.PublishPipelineArtifact@1
26+
condition: and(succeeded(), ${{ parameters.CustomCondition }})
27+
displayName: 'Publish ${{ parameters.ArtifactName }} Artifacts'
28+
inputs:
29+
artifact: '$(PublishArtifactName)'
30+
path: '${{ parameters.ArtifactPath }}'

0 commit comments

Comments
 (0)