Skip to content

Commit bfafdb3

Browse files
alzimmermsftalzimmermsft
authored
Parallelize code generation validation scripts (Azure#45892)
* Parallelize Swagger validation * Finalize Swagger update script changes * First TypeSpec update * Finalize TypeSpec update script * Finalize TypeSpec update script * Small script update * Mock Core release update PR * Merge scripts together * Uncomment cleanup code * Few fixes * Revert testing changes
1 parent fddb6b1 commit bfafdb3

File tree

4 files changed

+222
-200
lines changed

4 files changed

+222
-200
lines changed

‎eng/pipelines/templates/jobs/ci.yml‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -401,20 +401,13 @@ jobs:
401401
displayName: Use Node.js 22.x
402402

403403
- task: PowerShell@2
404-
displayName: Verify Code Generation for Pull Request
404+
displayName: Verify Swagger and TypeSpec Code Generation
405405
inputs:
406406
pwsh: true
407407
filePath: $(Build.SourcesDirectory)/eng/scripts/Compare-CurrentToCodegeneration.ps1
408408
arguments: >
409409
-ServiceDirectories '$(PRServiceDirectories)'
410-
411-
- task: PowerShell@2
412-
displayName: Verify TypeSpec Code Generation
413-
inputs:
414-
pwsh: true
415-
filePath: $(Build.SourcesDirectory)/eng/scripts/TypeSpec-Compare-CurrentToCodegeneration.ps1
416-
arguments: >
417-
-ServiceDirectories '$(PRServiceDirectories)'
410+
-RegenerationType 'All'
418411
419412
- template: /eng/pipelines/templates/steps/run-and-validate-linting.yml
420413
parameters:
@@ -459,7 +452,10 @@ jobs:
459452
${{ if eq(parameters.ServiceDirectory, 'auto') }}:
460453
EnablePRGeneration: true
461454
PRMatrixSparseIndirect: true
462-
PRJobBatchSize: 20
455+
# Use a PR job batch size of 50 as the design of Java's CI doesn't increase CI times linear for each library
456+
# added. So, using a smaller batch size means more CI jobs that are slightly faster than fewer larger jobs.
457+
# Overall, using a large batch size reduces overall CI time.
458+
PRJobBatchSize: 50
463459
# This key will use the PackageInfo.Name to populate the ArtifactPackageNames
464460
PRMatrixKey: 'Name'
465461
SparseCheckoutPaths:

‎eng/scripts/Compare-CurrentToCodegeneration.ps1‎

Lines changed: 216 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,85 +1,246 @@
11
<#
22
.SYNOPSIS
3-
Invokes all 'Update-Codegeneration.ps1' scripts found within the specified directory and compares it against current
4-
code.
3+
Runs code generation for either Swagger or TypeSpec, based on configuration, and compares the generated code against
4+
the state of the current codebase.
55
66
.DESCRIPTION
7-
Invokes all 'Update-Codegeneration.ps1' scripts found within the specified directory and compares it against current
8-
code.
7+
Runs code generation for either Swagger or TypeSpec, based on configuration, and compares the generated code against
8+
the state of the current codebase.
99
10-
If the regenerated code is different than the current code this will tell the differences, the files the differences
11-
are in, and exit with a failure status.
10+
If the regenerated code is different than the current codebase this will report the differences and exit with a failure
11+
status.
1212
13-
.PARAMETER Directory
14-
The directory that will be searched for 'Update-Codegeneration.ps1' scripts. The default is the root directory of the
15-
Azure SDK for Java repository. CI jobs should use the 'ServiceDirectory', such as /sdk/storage.
13+
.PARAMETER ServiceDirectories
14+
The service directories that will be searched for either 'Update-Codegeneration.ps1' or 'tsp-location.yaml' files to
15+
run code regeneration. If this parameter is not specified, the script will not check any directories and will exit
16+
with a success status.
17+
18+
.PARAMETER RegenerationType
19+
The type of regeneration to perform. This can be 'All', 'Swagger', or 'TypeSpec'. If not specified, the script will use
20+
'All' as the default, which means it will run both Swagger and TypeSpec code generation.
21+
22+
.PARAMETER Parallelization
23+
The number of parallel jobs to run. The default is the number of processors on the machine. If unspecified or
24+
less than 1, it will default to 1.
1625
#>
1726

1827
param(
1928
[Parameter(Mandatory = $false)]
20-
[string]$ServiceDirectories
29+
[string]$ServiceDirectories,
30+
31+
[Parameter(Mandatory = $false)]
32+
[ValidateSet('All', 'Swagger', 'TypeSpec')]
33+
[string]$RegenerationType = 'All',
34+
35+
[Parameter(Mandatory = $false)]
36+
[int]$Parallelization = [Environment]::ProcessorCount
2137
)
2238

23-
$SeparatorBars = "==========================================================================="
39+
class GenerationInformation {
40+
# The directory where the library is located. Used for logging and validation.
41+
[string]$LibraryFolder
2442

25-
# Returns true if there's an error, false otherwise
26-
function Compare-CurrentToCodegeneration {
27-
param(
28-
[Parameter(Mandatory=$true)]
29-
$ServiceDirectory
30-
)
43+
# The path to the script that will perform the code generation.
44+
# This can be 'Update-Codegeneration.ps1' for Swagger or 'tsp-location.yaml' for TypeSpec.
45+
[string]$ScriptPath
46+
47+
# The type of code generation this script performs, either 'Swagger' or 'TypeSpec'.
48+
# This is used to determine actions to take based on the type of code generation.
49+
[ValidateSet('Swagger', 'TypeSpec')]
50+
[string]$Type
3151

32-
$swaggers = Get-ChildItem -Path $ServiceDirectory -Filter "Update-Codegeneration.ps1" -Recurse
33-
if ($swaggers.Count -eq 0) {
34-
Write-Host "$SeparatorBars"
35-
Write-Host "No Swagger files to regenerate for $ServiceDirectory"
36-
Write-Host "$SeparatorBars"
37-
return $false
52+
GenerationInformation([string]$libraryFolder, [string]$scriptPath, [string]$type) {
53+
$this.LibraryFolder = $libraryFolder
54+
$this.ScriptPath = $scriptPath
55+
$this.Type = $type
3856
}
57+
}
3958

59+
function Find-GenerationInformation {
60+
param (
61+
[System.Collections.ArrayList]$GenerationInformations,
62+
[string]$ServiceDirectory,
63+
[string]$RegenerationType
64+
)
4065

41-
Write-Host "$SeparatorBars"
42-
Write-Host "Invoking Autorest code regeneration for $ServiceDirectory"
43-
Write-Host "$SeparatorBars"
66+
$path = "sdk/$ServiceDirectory"
67+
if ($RegenerationType -eq 'Swagger' -or $RegenerationType -eq 'All') {
68+
# Search for 'Update-Codegeneration.ps1' script in the specified service directory.
69+
Get-ChildItem -Path $path -Filter "Update-Codegeneration.ps1" -Recurse | ForEach-Object {
70+
$GenerationInformations.Add([GenerationInformation]::new($path, $_, 'Swagger')) | Out-Null
71+
}
72+
}
4473

45-
foreach ($script in $swaggers) {
46-
Write-Host "Calling Invoke-Expression $($script.FullName)"
47-
(& $script.FullName) | Write-Host
74+
if ($RegenerationType -eq 'TypeSpec' -or $RegenerationType -eq 'All') {
75+
# Search for 'tsp-location.yaml' script in the specified service directory.
76+
Get-ChildItem -Path $path -Filter "tsp-location.yaml" -Recurse | ForEach-Object {
77+
$GenerationInformations.Add([GenerationInformation]::new($path, $_, 'TypeSpec')) | Out-Null
78+
}
4879
}
80+
}
4981

50-
Write-Host "$SeparatorBars"
51-
Write-Host "Verify no diff for $ServiceDirectory"
52-
Write-Host "$SeparatorBars"
82+
# No ServiceDirectories specified, no validation will be performed.
83+
if (-not $ServiceDirectories) {
84+
Write-Host "No ServiceDirectories specified, no validation will be performed."
85+
exit 0
86+
}
5387

54-
# prevent warning related to EOL differences which triggers an exception for some reason
55-
& git -c core.safecrlf=false diff --ignore-space-at-eol --exit-code -- "*.java"
88+
# If Parallelization is not specified or less than 1, set it to 1.
89+
if ($Parallelization -lt 1) {
90+
$Parallelization = 1
91+
}
5692

57-
if ($LastExitCode -ne 0) {
58-
$status = git status -s | Out-String
59-
Write-Host "The following files in $ServiceDirectory are out of date:"
60-
Write-Host "$status"
61-
return $true
93+
# Split the ServiceDirectories by comma and trim whitespace.
94+
# Then search for 'Update-Codegeneration.ps1' or 'tsp-location.yaml' scripts in those directories,
95+
# based on RegenerationType, and store the results in a list of GenerationInformation objects.
96+
$generationInformations = New-Object 'Collections.ArrayList'
97+
foreach ($serviceDirectory in $ServiceDirectories.Split(',')) {
98+
$serviceDirectory = $serviceDirectory.Trim()
99+
if ($serviceDirectory -match '\w+/\w+') {
100+
# The service directory is a specific library, e.g., "communication/azure-communication-chat"
101+
# Search the directory directly for an "Update-Codegeneration.ps1" script.
102+
Find-GenerationInformation $generationInformations $serviceDirectory $RegenerationType
103+
} else {
104+
# The service directory is a top-level service, e.g., "storage"
105+
# Search for all libraries under the service directory.
106+
foreach ($libraryFolder in Get-ChildItem -Path "sdk/$serviceDirectory" -Directory) {
107+
Find-GenerationInformation $generationInformations "$serviceDirectory/$($libraryFolder.Name)" $RegenerationType
108+
}
62109
}
63-
return $false
64110
}
65111

66-
$hasError = $false
112+
if ($generationInformations.Count -eq 0) {
113+
$kind = $RegenerationType -eq 'All' ? 'Swagger or TypeSpec' : $RegenerationType
114+
Write-Host @"
115+
======================================
116+
No $kind generation files to regenerate in directories: $ServiceDirectories.
117+
======================================
118+
"@
119+
exit 0
120+
}
67121

68-
# If a list of ServiceDirectories was passed in, process the entire list otherwise
69-
# pass in an empty string to verify everything
70-
if ($ServiceDirectories) {
71-
foreach ($ServiceDirectory in $ServiceDirectories.Split(',')) {
72-
$path = "sdk/$ServiceDirectory"
73-
$result = Compare-CurrentToCodegeneration $path
74-
if ($result) {
75-
$hasError = $true
76-
}
122+
if ($RegenerationType -eq 'Swagger' -or $RegenerationType -eq 'All') {
123+
# Ensure Autorest is installed.
124+
$output = (& npm install -g autorest 2>&1)
125+
if ($LASTEXITCODE -ne 0) {
126+
Write-Error "Failed to install Autorest for Swagger regeneration."
127+
Write-Error $output
128+
exit 1
77129
}
78-
} else {
79-
Write-Host "The service directory list was empty for this PR, no Swagger files check"
80130
}
81131

82-
if ($hasError) {
83-
exit 1
132+
if ($RegenerationType -eq 'TypeSpec' -or $RegenerationType -eq 'All') {
133+
$output = (& npm install -g @azure-tools/typespec-client-generator-cli 2>&1)
134+
if ($LASTEXITCODE -ne 0) {
135+
Write-Error "Error installing @azure-tools/typespec-client-generator-cli"
136+
Write-Error "$output"
137+
exit 1
138+
}
84139
}
85-
exit 0
140+
141+
$generateScript = {
142+
$directory = $_.LibraryFolder
143+
$updateCodegenScript = $_.ScriptPath
144+
145+
if ($_.Type -eq 'Swagger') {
146+
# 6>&1 redirects Write-Host calls in the script to the output stream, so we can capture it.
147+
$generateOutput = (& $updateCodegenScript 6>&1)
148+
149+
if ($LastExitCode -ne 0) {
150+
Write-Host @"
151+
======================================
152+
Error running Swagger regeneration $updateCodegenScript
153+
======================================
154+
$([String]::Join("`n", $generateOutput))
155+
"@
156+
throw
157+
} else {
158+
# prevent warning related to EOL differences which triggers an exception for some reason
159+
(& git -c core.safecrlf=false diff --ignore-space-at-eol --exit-code -- "$directory/*.java") | Out-Null
160+
161+
if ($LastExitCode -ne 0) {
162+
$status = (git status -s "$directory" | Out-String)
163+
Write-Host @"
164+
======================================
165+
The following Swagger generated files in directoy $directory are out of date:
166+
======================================
167+
$status
168+
"@
169+
throw
170+
} else {
171+
Write-Host @"
172+
======================================
173+
Successfully ran Swagger regneration with no diff $updateCodegenScript
174+
======================================
175+
"@
176+
}
177+
}
178+
} elseif ($_.Type -eq 'TypeSpec') {
179+
Push-Location $Directory
180+
try {
181+
$generateOutput = (& tsp-client update 2>&1)
182+
if ($LastExitCode -ne 0) {
183+
Write-Host @"
184+
======================================
185+
Error running TypeSpec regeneration in directory $Directory
186+
======================================
187+
$([String]::Join("`n", $generateOutput))
188+
"@
189+
throw
190+
}
191+
192+
# Update code snippets before comparing the diff
193+
$mvnOutput = (& mvn --no-transfer-progress codesnippet:update-codesnippet 2>&1)
194+
if ($LastExitCode -ne 0) {
195+
Write-Host @"
196+
======================================
197+
Error updating TypeSpec codesnippets in directory $Directory
198+
======================================
199+
$([String]::Join("`n", $mvnOutput))
200+
"@
201+
throw
202+
}
203+
} finally {
204+
Pop-Location
205+
}
206+
207+
# prevent warning related to EOL differences which triggers an exception for some reason
208+
(& git -c core.safecrlf=false diff --ignore-space-at-eol --exit-code -- "$Directory/*.java" ":(exclude)**/src/test/**" ":
209+
(exclude)**/src/samples/**" ":(exclude)**/src/main/**/implementation/**" ":(exclude)**/src/main/**/resourcemanager/**/*Manager.java") | Out-Null
210+
211+
if ($LastExitCode -ne 0) {
212+
$status = (git status -s "$Directory" | Out-String)
213+
Write-Host @"
214+
======================================
215+
The following TypeSpec files in directoy $Directory are out of date:
216+
======================================
217+
$status
218+
"@
219+
throw
220+
} else {
221+
Write-Host @"
222+
======================================
223+
Successfully ran TypeSpec update in directory with no diff $Directory
224+
======================================
225+
"@
226+
}
227+
} else {
228+
Write-Error "Unknown generation type: $($_.Type), directory: $directory"
229+
throw
230+
}
231+
}
232+
233+
# Timeout is set to 60 seconds per script.
234+
$timeout = 60 * $generationInformations.Count
235+
236+
$job = $generationInformations | ForEach-Object -Parallel $generateScript -ThrottleLimit $Parallelization -AsJob
237+
238+
# Out-Null to suppress output information from the job and 2>$null to suppress any error messages from Receive-Job.
239+
$job | Wait-Job -Timeout $timeout | Out-Null
240+
$job | Receive-Job 2>$null | Out-Null
241+
242+
# Clean up generated code, so that next step will not be affected.
243+
git reset --hard | Out-Null
244+
git clean -fd . | Out-Null
245+
246+
exit $job.State -eq 'Failed'

‎eng/scripts/Invoke-Codegeneration.ps1‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,6 @@ param(
2222
[string]$AutorestOptions
2323
)
2424

25-
# Ensure Autorest is installed.
26-
npm install --error -g autorest
27-
if ($LASTEXITCODE -ne 0) {
28-
Write-Error "Failed to install Autorest."
29-
exit 1
30-
}
31-
3225
try {
3326
Push-Location $Directory
3427
if ($AutorestOptions) {

0 commit comments

Comments
 (0)