Skip to content

Commit c957155

Browse files
azure-sdkscbedd
azure-sdk
and
scbedd
authored
Sync eng/common directory with azure-sdk-tools for PR 9836 (#2171)
* adjustments to iterating across matrix configs for packages that belong to multiple matrices --------- Co-authored-by: Scott Beddall <[email protected]>
1 parent 96c6d4d commit c957155

File tree

1 file changed

+87
-81
lines changed

1 file changed

+87
-81
lines changed

‎eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1‎

Lines changed: 87 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -94,110 +94,116 @@ function GeneratePRMatrixForBatch {
9494
$matrixBatchesByConfig = Group-ByObjectKey $Packages "CIMatrixConfigs"
9595

9696
foreach ($matrixBatchKey in $matrixBatchesByConfig.Keys) {
97+
# recall that while we have grouped the package info by the matrix config object, each package still has knowledge about
98+
# every other matrix that it belongs to.
99+
# so we actually need to get a valid matrix config object from the first package in the batch, but we need to be certain
100+
# that the matrix config object we get is the SAME ONE that we are iterating through
97101
$matrixBatch = $matrixBatchesByConfig[$matrixBatchKey]
98-
$matrixConfigs = $matrixBatch | Select-Object -First 1 -ExpandProperty CIMatrixConfigs
99-
102+
$allPossibleMatrixConfigsForFirstPackage = $matrixBatch | Select-Object -First 1 -ExpandProperty CIMatrixConfigs
103+
$matrixConfig = $allPossibleMatrixConfigsForFirstPackage | Where-Object { (Get-ObjectKey $_) -eq $matrixBatchKey }
100104
$matrixResults = @()
101-
foreach ($matrixConfig in $matrixConfigs) {
102-
Write-Host "Generating config for $($matrixConfig.Path)"
103-
$nonSparse = $matrixConfig.PSObject.Properties['NonSparseParameters'] ? $matrixConfig.NonSparseParameters : @()
104-
105-
$matrixResults = @()
106-
if ($directBatch) {
107-
$matrixResults = GenerateMatrixForConfig `
108-
-ConfigPath $matrixConfig.Path `
109-
-Selection $matrixConfig.Selection `
110-
-DisplayNameFilter $DisplayNameFilter `
111-
-Filters $Filters `
112-
-Replace $Replace `
113-
-NonSparseParameters $nonSparse
114-
115-
if ($matrixResults) {
116-
Write-Host "We have the following direct matrix results: "
117-
Write-Host ($matrixResults | Out-String)
118-
}
105+
106+
if (!$matrixConfig) {
107+
Write-Error "Unable to find matrix config for $matrixBatchKey. Check the package properties for the package $($matrixBatch[0].ArtifactName)."
108+
exit 1
109+
}
110+
111+
Write-Host "Generating config for $($matrixConfig.Path)"
112+
$nonSparse = $matrixConfig.PSObject.Properties['NonSparseParameters'] ? $matrixConfig.NonSparseParameters : @()
113+
114+
if ($directBatch) {
115+
$matrixResults = GenerateMatrixForConfig `
116+
-ConfigPath $matrixConfig.Path `
117+
-Selection $matrixConfig.Selection `
118+
-DisplayNameFilter $DisplayNameFilter `
119+
-Filters $Filters `
120+
-Replace $Replace `
121+
-NonSparseParameters $nonSparse
122+
123+
if ($matrixResults) {
124+
Write-Host "We have the following direct matrix results: "
125+
Write-Host ($matrixResults | Out-String)
126+
}
127+
}
128+
else {
129+
$matrixResults = GenerateMatrixForConfig `
130+
-ConfigPath $matrixConfig.Path `
131+
-Selection $matrixConfig.Selection `
132+
-DisplayNameFilter $DisplayNameFilter `
133+
-Filters ($Filters + $IndirectFilters) `
134+
-Replace $Replace `
135+
-NonSparseParameters $nonSparse
136+
137+
if ($matrixResults) {
138+
Write-Host "We have the following indirect matrix results: "
139+
Write-Host ($matrixResults | Out-String)
119140
}
120141
else {
121-
$matrixResults = GenerateMatrixForConfig `
122-
-ConfigPath $matrixConfig.Path `
123-
-Selection $matrixConfig.Selection `
124-
-DisplayNameFilter $DisplayNameFilter `
125-
-Filters ($Filters + $IndirectFilters) `
126-
-Replace $Replace `
127-
-NonSparseParameters $nonSparse
128-
129-
if ($matrixResults) {
130-
Write-Host "We have the following indirect matrix results: "
131-
Write-Host ($matrixResults | Out-String)
132-
}
133-
else {
134-
Write-Host "No indirect matrix results found for $($matrixConfig.Path)"
135-
continue
136-
}
142+
Write-Host "No indirect matrix results found for $($matrixConfig.Path)"
143+
continue
137144
}
145+
}
138146

139-
$packageBatches = Split-ArrayIntoBatches -InputArray $matrixBatch -BatchSize $BATCHSIZE
140-
141-
# we only need to modify the generated job name if there is more than one matrix config + batch
142-
$matrixSuffixNecessary = $matrixBatchesByConfig.Keys.Count -gt 1
143-
144-
# if we are doing direct packages (or a full indirect matrix), we need to walk the batches and duplicate the matrix config for each batch, fully assigning
145-
# the each batch's packages to the matrix config. This will generate a _non-sparse_ matrix for the incoming packages
146-
if ($directBatch -or $FullSparseMatrix) {
147-
$batchSuffixNecessary = $packageBatches.Length -gt $($directBatch ? 1 : 0)
148-
$batchCounter = 1
149-
150-
foreach ($batch in $packageBatches) {
151-
$namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join ","
152-
153-
foreach ($matrixOutputItem in $matrixResults) {
154-
# we need to clone this, as each item is an object with possible children
155-
$outputItem = $matrixOutputItem | ConvertTo-Json -Depth 100 | ConvertFrom-Json -AsHashtable
156-
# we just need to iterate across them, grab the parameters hashtable, and add the new key
157-
# if there is more than one batch, we will need to add a suffix including the batch name to the job name
158-
$outputItem["parameters"]["$PRMatrixSetting"] = $namesForBatch
147+
$packageBatches = Split-ArrayIntoBatches -InputArray $matrixBatch -BatchSize $BATCHSIZE
159148

160-
if ($matrixSuffixNecessary) {
161-
$outputItem["name"] = $outputItem["name"] + "_" + $matrixConfig.Name
162-
}
149+
# we only need to modify the generated job name if there is more than one matrix config + batch
150+
$matrixSuffixNecessary = $matrixBatchesByConfig.Keys.Count -gt 1
163151

164-
if ($batchSuffixNecessary) {
165-
$outputItem["name"] = $outputItem["name"] + "$batchNamePrefix$batchCounter"
166-
}
152+
# if we are doing direct packages (or a full indirect matrix), we need to walk the batches and duplicate the matrix config for each batch, fully assigning
153+
# the each batch's packages to the matrix config. This will generate a _non-sparse_ matrix for the incoming packages
154+
if ($directBatch -or $FullSparseMatrix) {
155+
$batchSuffixNecessary = $packageBatches.Length -gt $($directBatch ? 1 : 0)
156+
$batchCounter = 1
167157

168-
$OverallResult += $outputItem
169-
}
170-
$batchCounter += 1
171-
}
172-
}
173-
# in the case of indirect packages, instead of walking the batches and duplicating their matrix config entirely,
174-
# we instead will walk each each matrix, create a parameter named for the PRMatrixSetting, and add the targeted packages
175-
# as an array. This will generate a _sparse_ matrix for for whatever the incoming packages are
176-
else {
177-
$batchSuffixNecessary = $packageBatches.Length -gt 0
178-
$batchCounter = 1
179-
foreach ($batch in $packageBatches) {
180-
$namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join ","
181-
$outputItem = QueuePop -queue ([ref]$matrixResults)
158+
foreach ($batch in $packageBatches) {
159+
$namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join ","
182160

161+
foreach ($matrixOutputItem in $matrixResults) {
162+
# we need to clone this, as each item is an object with possible children
163+
$outputItem = $matrixOutputItem | ConvertTo-Json -Depth 100 | ConvertFrom-Json -AsHashtable
164+
# we just need to iterate across them, grab the parameters hashtable, and add the new key
165+
# if there is more than one batch, we will need to add a suffix including the batch name to the job name
183166
$outputItem["parameters"]["$PRMatrixSetting"] = $namesForBatch
184167

185168
if ($matrixSuffixNecessary) {
186169
$outputItem["name"] = $outputItem["name"] + "_" + $matrixConfig.Name
187170
}
188171

189172
if ($batchSuffixNecessary) {
190-
$outputItem["name"] = $outputItem["name"] + "_$batchNamePrefix$batchCounter"
173+
$outputItem["name"] = $outputItem["name"] + "$batchNamePrefix$batchCounter"
191174
}
192-
# now we need to take an item from the front of the matrix results, clone it, and add it to the back of the matrix results
193-
# we will add the cloned version to OverallResult
175+
194176
$OverallResult += $outputItem
195-
$batchCounter += 1
196177
}
178+
$batchCounter += 1
197179
}
198180
}
199-
}
181+
# in the case of indirect packages, instead of walking the batches and duplicating their matrix config entirely,
182+
# we instead will walk each each matrix, create a parameter named for the PRMatrixSetting, and add the targeted packages
183+
# as an array. This will generate a _sparse_ matrix for for whatever the incoming packages are
184+
else {
185+
$batchSuffixNecessary = $packageBatches.Length -gt 0
186+
$batchCounter = 1
187+
foreach ($batch in $packageBatches) {
188+
$namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join ","
189+
$outputItem = QueuePop -queue ([ref]$matrixResults)
190+
191+
$outputItem["parameters"]["$PRMatrixSetting"] = $namesForBatch
192+
193+
if ($matrixSuffixNecessary) {
194+
$outputItem["name"] = $outputItem["name"] + "_" + $matrixConfig.Name
195+
}
200196

197+
if ($batchSuffixNecessary) {
198+
$outputItem["name"] = $outputItem["name"] + "_$batchNamePrefix$batchCounter"
199+
}
200+
# now we need to take an item from the front of the matrix results, clone it, and add it to the back of the matrix results
201+
# we will add the cloned version to OverallResult
202+
$OverallResult += $outputItem
203+
$batchCounter += 1
204+
}
205+
}
206+
}
201207

202208
return ,$OverallResult
203209
}

0 commit comments

Comments
 (0)