Skip to content

Commit 5a03501

Browse files
azure-sdkscbeddweshaggard
authored
Sync eng/common directory with azure-sdk-tools for PR 9601 (Azure#47856)
* updates to dynamic matrix generation. moves indirectly included packages to sparse usage. adds additional parameters to GeneratePRMatrix that enable further filtering of the matrix generated for the indirect packages --------- Co-authored-by: Scott Beddall <[email protected]> Co-authored-by: Wes Haggard <[email protected]>
1 parent b003725 commit 5a03501

File tree

3 files changed

+185
-56
lines changed

3 files changed

+185
-56
lines changed

‎eng/common/pipelines/templates/jobs/generate-job-matrix.yml‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,12 @@ parameters:
4848
- name: PRMatrixSetting
4949
type: string
5050
default: 'ArtifactPackageNames'
51+
- name: PRJobBatchSize
52+
type: number
53+
default: 10
54+
- name: PRMatrixIndirectFilters
55+
type: object
56+
default: []
5157
# Mappings to OS name required at template compile time by 1es pipeline templates
5258
- name: Pools
5359
type: object
@@ -126,7 +132,6 @@ jobs:
126132
- ${{ else }}:
127133
- ${{ each pool in parameters.Pools }}:
128134
- pwsh: |
129-
# dump the conglomerated CI matrix
130135
'${{ convertToJson(parameters.MatrixConfigs) }}' | Set-Content matrix.json
131136
132137
./eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 `
@@ -135,7 +140,9 @@ jobs:
135140
-PRMatrixSetting ${{ parameters.PRMatrixSetting }} `
136141
-DisplayNameFilter '$(displayNameFilter)' `
137142
-Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' `
138-
-Replace '${{ join(''',''', parameters.MatrixReplace) }}'
143+
-IndirectFilters '${{ join(''',''', parameters.PRMatrixIndirectFilters) }}' `
144+
-Replace '${{ join(''',''', parameters.MatrixReplace) }}' `
145+
-PackagesPerPRJob ${{ parameters.PRJobBatchSize }}
139146
displayName: Create ${{ pool.name }} PR Matrix
140147
name: vm_job_matrix_pr_${{ pool.name }}
141148

‎eng/common/pipelines/templates/steps/verify-readmes.yml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,20 @@ parameters:
1313
- name: Condition
1414
type: string
1515
default: succeeded()
16+
- name: IncludeIndirect
17+
type: boolean
18+
default: true
1619

1720
steps:
1821
- pwsh: |
22+
$includeIndirect = $${{ parameters.IncludeIndirect }}
1923
$packageProperties = Get-ChildItem -Recurse "${{ parameters.PackagePropertiesFolder }}" *.json
20-
2124
$paths = @()
2225
26+
if (-not $includeIndirect) {
27+
$packageProperties = $packageProperties | Where-Object { (Get-Content -Raw $_ | ConvertFrom-Json).IncludedForValidation -eq $false }
28+
}
29+
2330
foreach($propertiesFile in $packageProperties) {
2431
$PackageProp = Get-Content -Path $propertiesFile | ConvertFrom-Json
2532

‎eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1‎

Lines changed: 168 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,12 @@ It generates the matrix by the following algorithm:
2323
- add the combined property name to the parameters of the matrix item
2424
- add the matrix item to the overall result
2525
26+
.PARAMETER IndirectFilters
27+
Any array of strings representing filters that will only be applied to the matrix generation for indirect packages. This is useful for
28+
filtering out OTHER parameter settings othan than PRMatrixSetting that are only relevant to direct packages.
29+
30+
For .NET, this value will be AdditionalTestArguments=/p:UseProjectReferenceToAzureClients=true
31+
2632
.EXAMPLE
2733
./eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 `
2834
-PackagePropertiesFolder "path/to/populated/PackageInfo" `
@@ -36,13 +42,158 @@ param (
3642
[Parameter(Mandatory = $true)][string] $PRMatrixSetting,
3743
[Parameter(Mandatory = $False)][string] $DisplayNameFilter,
3844
[Parameter(Mandatory = $False)][array] $Filters,
45+
[Parameter(Mandatory = $False)][array] $IndirectFilters,
3946
[Parameter(Mandatory = $False)][array] $Replace,
47+
[Parameter(Mandatory = $False)][int] $PackagesPerPRJob = 10,
4048
[Parameter()][switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID)
4149
)
4250

51+
Set-StrictMode -Version 4
4352
. $PSScriptRoot/job-matrix-functions.ps1
4453
. $PSScriptRoot/../Helpers/Package-Helpers.ps1
45-
$BATCHSIZE = 10
54+
. $PSScriptRoot/../Package-Properties.ps1
55+
$BATCHSIZE = $PackagesPerPRJob
56+
57+
# this function takes an array of objects, takes a copy of the first item, and moves that item to the back of the array
58+
function QueuePop([ref]$queue) {
59+
60+
if ($queue.Value.Length -eq 1) {
61+
return ($queue.Value[0] | ConvertTo-Json -Depth 100 | ConvertFrom-Json -AsHashtable)
62+
}
63+
64+
# otherwise we can rotate stuff
65+
$first = $queue.Value[0]
66+
$rest = $queue.Value[1..($queue.Value.Length - 1)]
67+
68+
$queue.Value = $rest + $first
69+
70+
return ($first | ConvertTo-Json -Depth 100 | ConvertFrom-Json -AsHashtable)
71+
}
72+
73+
function GeneratePRMatrixForBatch {
74+
param (
75+
[Parameter(Mandatory = $true)][array] $Packages
76+
)
77+
78+
$OverallResult = @()
79+
if (!$Packages) {
80+
Write-Host "Unable to generate matrix for empty package list"
81+
return ,$OverallResult
82+
}
83+
84+
# this check assumes that we have properly separated the direct and indirect package lists
85+
$directBatch = $Packages[0].IncludedForValidation -eq $false
86+
Write-Host "Generating matrix for $($directBatch ? 'direct' : 'indirect') packages"
87+
88+
# The key here is that after we group the packages by the matrix config objects, we can use the first item's MatrixConfig
89+
# to generate the matrix for the group, no reason to have to parse the key value backwards to get the matrix config.
90+
$matrixBatchesByConfig = Group-ByObjectKey $Packages "CIMatrixConfigs"
91+
92+
foreach ($matrixBatchKey in $matrixBatchesByConfig.Keys) {
93+
$matrixBatch = $matrixBatchesByConfig[$matrixBatchKey]
94+
$matrixConfigs = $matrixBatch | Select-Object -First 1 -ExpandProperty CIMatrixConfigs
95+
96+
$matrixResults = @()
97+
foreach ($matrixConfig in $matrixConfigs) {
98+
Write-Host "Generating config for $($matrixConfig.Path)"
99+
100+
$matrixResults = @()
101+
if ($directBatch) {
102+
$matrixResults = GenerateMatrixForConfig `
103+
-ConfigPath $matrixConfig.Path `
104+
-Selection $matrixConfig.Selection `
105+
-DisplayNameFilter $DisplayNameFilter `
106+
-Filters $Filters `
107+
-Replace $Replace
108+
109+
if ($matrixResults) {
110+
Write-Host "We have the following direct matrix results: "
111+
Write-Host ($matrixResults | Out-String)
112+
}
113+
}
114+
else {
115+
$matrixResults = GenerateMatrixForConfig `
116+
-ConfigPath $matrixConfig.Path `
117+
-Selection $matrixConfig.Selection `
118+
-DisplayNameFilter $DisplayNameFilter `
119+
-Filters ($Filters + $IndirectFilters) `
120+
-Replace $Replace
121+
122+
if ($matrixResults) {
123+
Write-Host "We have the following indirect matrix results: "
124+
Write-Host ($matrixResults | Out-String)
125+
}
126+
else {
127+
Write-Host "No indirect matrix results found for $($matrixConfig.Path)"
128+
continue
129+
}
130+
}
131+
132+
$packageBatches = Split-ArrayIntoBatches -InputArray $matrixBatch -BatchSize $BATCHSIZE
133+
134+
# we only need to modify the generated job name if there is more than one matrix config + batch
135+
$matrixSuffixNecessary = $matrixBatchesByConfig.Keys.Count -gt 1
136+
137+
# if we are doing direct packages, we need to walk the batches and duplicate the matrix config for each batch, fully assigning
138+
# the each batch's packages to the matrix config. This will generate a _non-sparse_ matrix for the incoming packages
139+
if ($directBatch) {
140+
$batchSuffixNecessary = $packageBatches.Length -gt 1
141+
$batchCounter = 1
142+
143+
foreach ($batch in $packageBatches) {
144+
$namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join ","
145+
146+
foreach ($matrixOutputItem in $matrixResults) {
147+
# we need to clone this, as each item is an object with possible children
148+
$outputItem = $matrixOutputItem | ConvertTo-Json -Depth 100 | ConvertFrom-Json -AsHashtable
149+
# we just need to iterate across them, grab the parameters hashtable, and add the new key
150+
# if there is more than one batch, we will need to add a suffix including the batch name to the job name
151+
$outputItem["parameters"]["$PRMatrixSetting"] = $namesForBatch
152+
153+
if ($matrixSuffixNecessary) {
154+
$outputItem["name"] = $outputItem["name"] + "_" + $matrixConfig.Name
155+
}
156+
157+
if ($batchSuffixNecessary) {
158+
$outputItem["name"] = $outputItem["name"] + "_b$batchCounter"
159+
}
160+
161+
$OverallResult += $outputItem
162+
}
163+
$batchCounter += 1
164+
}
165+
}
166+
# in the case of indirect packages, instead of walking the batches and duplicating their matrix config entirely,
167+
# we instead will walk each each matrix, create a parameter named for the PRMatrixSetting, and add the targeted packages
168+
# as an array. This will generate a _sparse_ matrix for for whatever the incoming packages are
169+
else {
170+
$batchSuffixNecessary = $packageBatches.Length -gt 0
171+
$batchCounter = 1
172+
foreach ($batch in $packageBatches) {
173+
$namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join ","
174+
$outputItem = QueuePop -queue ([ref]$matrixResults)
175+
176+
$outputItem["parameters"]["$PRMatrixSetting"] = $namesForBatch
177+
178+
if ($matrixSuffixNecessary) {
179+
$outputItem["name"] = $outputItem["name"] + "_" + $matrixConfig.Name
180+
}
181+
182+
if ($batchSuffixNecessary) {
183+
$outputItem["name"] = $outputItem["name"] + "_ib$batchCounter"
184+
}
185+
# now we need to take an item from the front of the matrix results, clone it, and add it to the back of the matrix results
186+
# we will add the cloned version to OverallResult
187+
$OverallResult += $outputItem
188+
$batchCounter += 1
189+
}
190+
}
191+
}
192+
}
193+
194+
195+
return ,$OverallResult
196+
}
46197

47198
if (!(Test-Path $PackagePropertiesFolder)) {
48199
Write-Error "Package Properties folder doesn't exist"
@@ -58,71 +209,35 @@ Write-Host "Generating PR job matrix for $PackagePropertiesFolder"
58209

59210
$configs = Get-Content -Raw $PRMatrixFile | ConvertFrom-Json
60211

61-
# calculate general targeting information and create our batches prior to generating any matrix
212+
# get all the package property objects loaded
62213
$packageProperties = Get-ChildItem -Recurse "$PackagePropertiesFolder" *.json `
63214
| ForEach-Object { Get-Content -Path $_.FullName | ConvertFrom-Json }
64215

65-
# set default matrix config for each package if there isn't an override
216+
# enhance the package props with a default matrix config if one isn't present
66217
$packageProperties | ForEach-Object {
67218
if (-not $_.CIMatrixConfigs) {
68219
$_.CIMatrixConfigs = $configs
69220
}
70221
}
71222

72-
# The key here is that after we group the packages by the matrix config objects, we can use the first item's MatrixConfig
73-
# to generate the matrix for the group, no reason to have to parse the key value backwards to get the matrix config.
74-
$matrixBatchesByConfig = Group-ByObjectKey $packageProperties "CIMatrixConfigs"
223+
$directPackages = $packageProperties | Where-Object { $_.IncludedForValidation -eq $false }
224+
$indirectPackages = $packageProperties | Where-Object { $_.IncludedForValidation -eq $true }
75225

76226
$OverallResult = @()
77-
foreach ($matrixBatchKey in $matrixBatchesByConfig.Keys) {
78-
$matrixBatch = $matrixBatchesByConfig[$matrixBatchKey]
79-
$matrixConfigs = $matrixBatch | Select-Object -First 1 -ExpandProperty CIMatrixConfigs
80-
81-
$matrixResults = @()
82-
foreach ($matrixConfig in $matrixConfigs) {
83-
Write-Host "Generating config for $($matrixConfig.Path)"
84-
$matrixResults = GenerateMatrixForConfig `
85-
-ConfigPath $matrixConfig.Path `
86-
-Selection $matrixConfig.Selection `
87-
-DisplayNameFilter $DisplayNameFilter `
88-
-Filters $Filters `
89-
-Replace $Replace
90-
91-
$packageBatches = Split-ArrayIntoBatches -InputArray $matrixBatch -BatchSize $BATCHSIZE
92-
93-
# we only need to modify the generated job name if there is more than one matrix config + batch
94-
$matrixSuffixNecessary = $matrixBatchesByConfig.Keys.Count -gt 1
95-
$batchSuffixNecessary = $packageBatches.Length -gt 1
96-
$batchCounter = 1
97-
98-
foreach ($batch in $packageBatches) {
99-
$namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join ","
100-
# to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys:
101-
# [
102-
# { "name": "jobname", "parameters": { matrixSetting1: matrixValue1, ...} },
103-
# ]
104-
foreach ($matrixOutputItem in $matrixResults) {
105-
# we need to clone this, as each item is an object with possible children
106-
$outputItem = $matrixOutputItem | ConvertTo-Json -Depth 100 | ConvertFrom-Json -AsHashtable
107-
# we just need to iterate across them, grab the parameters hashtable, and add the new key
108-
# if there is more than one batch, we will need to add a suffix including the batch name to the job name
109-
$outputItem["parameters"]["$PRMatrixSetting"] = $namesForBatch
110-
111-
if ($matrixSuffixNecessary) {
112-
$outputItem["name"] = $outputItem["name"] + "_" + $matrixConfig.Name
113-
}
114-
115-
if ($batchSuffixNecessary) {
116-
$outputItem["name"] = $outputItem["name"] + "_b$batchCounter"
117-
}
118-
119-
$OverallResult += $outputItem
120-
}
121-
$batchCounter += 1
122-
}
227+
if ($directPackages) {
228+
Write-Host "Discovered $($directPackages.Length) direct packages"
229+
foreach($artifact in $directPackages) {
230+
Write-Host "-> $($artifact.ArtifactName)"
123231
}
232+
$OverallResult += GeneratePRMatrixForBatch -Packages $directPackages
233+
}
234+
if ($indirectPackages) {
235+
Write-Host "Discovered $($indirectPackages.Length) indirect packages"
236+
foreach($artifact in $indirectPackages) {
237+
Write-Host "-> $($artifact.ArtifactName)"
238+
}
239+
$OverallResult += GeneratePRMatrixForBatch -Packages $indirectPackages
124240
}
125-
126241
$serialized = SerializePipelineMatrix $OverallResult
127242

128243
Write-Output $serialized.pretty

0 commit comments

Comments
 (0)