Skip to content

Commit 30809b2

Browse files
JimSuplizioJimSuplizio
authored
Update the *Compare-CurrentToCodegeneration scripts to work in the PR pipeline (Azure#44795)
* Update the *Compare-CurrentToCodegeneration scripts to work in the PR pipeline * Actually return a return value if everything succeeds * Single quote the $(PRServiceDirectories) variable * these will be reverted before merge * Change the Invoke-Expression command * Update the script for changes made yesterday in another PR * Add healthdataaiservices, which still needs to be fixed on the typespec side, and some diagnostics * Remove appconfig and storage pom changes. The Compare-CurrentToCodegeneration.ps1 updates work * Remove the healthdataservices change to test something * Making the temp changes to azure-storage-blob's pom, locked the version to the release version, this should remove my changes entirely * EOF on next line * trying to figure out why the compare function is returing pass and an error is still being detected * redirect output of tsp-client update to null so it isn't returned * more command redirection * Remove the diagnostics output, add healthdataaiservices back into PR, which is failing right now * refactor PRServiceDirectories usage based upon feedback * remove IgnoreVerifyTypeSpecCodeGenerationError from processing * revert pom changes which were for testing only, forcing changes so they happened in a PR build
1 parent bfb7087 commit 30809b2

File tree

6 files changed

+154
-138
lines changed

6 files changed

+154
-138
lines changed

‎eng/pipelines/templates/jobs/ci.yml‎

Lines changed: 14 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,6 @@ parameters:
5252
- name: JavaBuildVersion
5353
type: string
5454
default: $(JavaBuildVersion)
55-
- name: IgnoreVerifyTypeSpecCodeGenerationError
56-
type: boolean
57-
default: false
5855
- name: TimeoutInMinutes
5956
type: number
6057
default: 60
@@ -349,6 +346,7 @@ jobs:
349346
CheckLinkGuidance: $true
350347

351348
- ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}:
349+
# NOTE: The PRServiceDirectories string is used in the two Code Generation steps below
352350
- pwsh: |
353351
$diffJson = Get-Content '$(Build.ArtifactStagingDirectory)/diff/diff.json' -Raw
354352
$diff = ConvertFrom-Json $diffJson
@@ -357,15 +355,17 @@ jobs:
357355
$changedServicesString = $diff.ChangedServices -join ","
358356
}
359357
Write-Host "changedServicesString='$changedServicesString'"
360-
Write-Host "##vso[task.setvariable variable=SamplesServiceDirectories;]$changedServicesString"
361-
displayName: Get samples service directories from PR diff
362-
- template: /eng/common/pipelines/templates/steps/verify-samples.yml
363-
parameters:
364-
ServiceDirectories: $(SamplesServiceDirectories)
358+
Write-Host "##vso[task.setvariable variable=PRServiceDirectories;]$changedServicesString"
359+
displayName: Set PRServiceDirectories from PR diff
365360
- ${{ else }}:
366-
- template: /eng/common/pipelines/templates/steps/verify-samples.yml
367-
parameters:
368-
ServiceDirectory: ${{ parameters.ServiceDirectory }}
361+
- pwsh: |
362+
Write-Host "changedServicesString='${{ parameters.ServiceDirectory }}'"
363+
Write-Host "##vso[task.setvariable variable=PRServiceDirectories;]$changedServicesString"
364+
displayName: Set PRServiceDirectories to ServiceDirectory parameter
365+
366+
- template: /eng/common/pipelines/templates/steps/verify-samples.yml
367+
parameters:
368+
ServiceDirectories: $(PRServiceDirectories)
369369

370370
# Use BasePathLength of 38 instead of the default 49 as some released files fail when the number is higher.
371371
- template: /eng/common/pipelines/templates/steps/verify-path-length.yml
@@ -394,21 +394,20 @@ jobs:
394394
JdkVersion: ${{ parameters.JavaBuildVersion }}
395395

396396
- task: PowerShell@2
397-
displayName: Verify Code Generation
397+
displayName: Verify Code Generation for Pull Request
398398
inputs:
399399
pwsh: true
400400
filePath: $(Build.SourcesDirectory)/eng/scripts/Compare-CurrentToCodegeneration.ps1
401401
arguments: >
402-
-Directory sdk/${{ parameters.ServiceDirectory }}
402+
-ServiceDirectories '$(PRServiceDirectories)'
403403
404404
- task: PowerShell@2
405405
displayName: Verify TypeSpec Code Generation
406-
continueOnError: ${{ eq(parameters.IgnoreVerifyTypeSpecCodeGenerationError, 'true') }}
407406
inputs:
408407
pwsh: true
409408
filePath: $(Build.SourcesDirectory)/eng/scripts/TypeSpec-Compare-CurrentToCodegeneration.ps1
410409
arguments: >
411-
-Directory sdk/${{ parameters.ServiceDirectory }}
410+
-ServiceDirectories '$(PRServiceDirectories)'
412411
413412
- template: /eng/pipelines/templates/steps/run-and-validate-linting.yml
414413
parameters:

‎eng/pipelines/templates/stages/archetype-sdk-client.yml‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,6 @@ parameters:
5252
- name: ReleaseDependsOnLiveTests
5353
type: string
5454
default: not-specified
55-
- name: IgnoreVerifyTypeSpecCodeGenerationError
56-
type: boolean
57-
default: false
5855
- name: TimeoutInMinutes
5956
type: number
6057
default: 60
@@ -90,7 +87,6 @@ extends:
9087
Artifacts: ${{ parameters.Artifacts }}
9188
ExcludePaths: ${{parameters.ExcludePaths}}
9289
TimeoutInMinutes: ${{ parameters.TimeoutInMinutes }}
93-
IgnoreVerifyTypeSpecCodeGenerationError: ${{ parameters.IgnoreVerifyTypeSpecCodeGenerationError }}
9490
ReleaseArtifacts:
9591
- ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(variables['Build.Reason'], 'Manual')) }}:
9692
- ${{ each artifact in parameters.Artifacts }}:

‎eng/scripts/Compare-CurrentToCodegeneration.ps1‎

Lines changed: 58 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -17,55 +17,69 @@ Azure SDK for Java repository. CI jobs should use the 'ServiceDirectory', such a
1717

1818
param(
1919
[Parameter(Mandatory = $false)]
20-
[string]$Directory
20+
[string]$ServiceDirectories
2121
)
2222

23-
$path = ""
24-
if ($Directory) {
25-
$path = $Directory
23+
$SeparatorBars = "==========================================================================="
24+
25+
# Returns true if there's an error, false otherwise
26+
function Compare-CurrentToCodegeneration {
27+
param(
28+
[Parameter(Mandatory=$true)]
29+
$ServiceDirectory
30+
)
31+
32+
$swaggers = Get-ChildItem -Path $ServiceDirectory -Filter "Update-Codegeneration.ps1" -Recurse
33+
if ($swaggers.Count -eq 0) {
34+
Write-Host "$SeparatorBars"
35+
Write-Host "No Swagger files to regenerate for $ServiceDirectory"
36+
Write-Host "$SeparatorBars"
37+
return $false
38+
}
39+
40+
41+
Write-Host "$SeparatorBars"
42+
Write-Host "Invoking Autorest code regeneration for $ServiceDirectory"
43+
Write-Host "$SeparatorBars"
44+
45+
foreach ($script in $swaggers) {
46+
Write-Host "Calling Invoke-Expression $($script.FullName)"
47+
(& $script.FullName) | Write-Host
48+
}
49+
50+
Write-Host "$SeparatorBars"
51+
Write-Host "Verify no diff for $ServiceDirectory"
52+
Write-Host "$SeparatorBars"
53+
54+
# prevent warning related to EOL differences which triggers an exception for some reason
55+
& git -c core.safecrlf=false diff --ignore-space-at-eol --exit-code -- "*.java"
56+
57+
if ($LastExitCode -ne 0) {
58+
$status = git status -s | Out-String
59+
Write-Host "The following files in $ServiceDirectory are out of date:"
60+
Write-Host "$status"
61+
return $true
62+
}
63+
return $false
2664
}
2765

28-
$swaggers = Get-ChildItem -Path $path -Filter "Update-Codegeneration.ps1" -Recurse
29-
if ($swaggers.Count -eq 0) {
30-
Write-Host "
31-
32-
===========================================
33-
No Swagger files to regenerate
34-
===========================================
35-
36-
"
37-
exit 0
38-
}
39-
40-
41-
Write-Host "
42-
43-
===================================
44-
Invoking Autorest code regeneration
45-
===================================
46-
47-
"
48-
49-
foreach ($script in $swaggers) {
50-
Invoke-Expression $script.FullName
66+
$hasError = $false
67+
68+
# If a list of ServiceDirectories was passed in, process the entire list otherwise
69+
# pass in an empty string to verify everything
70+
if ($ServiceDirectories) {
71+
foreach ($ServiceDirectory in $ServiceDirectories.Split(',')) {
72+
$path = "sdk/$ServiceDirectory"
73+
$result = Compare-CurrentToCodegeneration $path
74+
if ($result) {
75+
$hasError = $true
76+
}
77+
}
78+
} else {
79+
Write-Host "The service directory list was empty for this PR, no Swagger files check"
5180
}
5281

53-
Write-Host "
54-
55-
==============
56-
Verify no diff
57-
==============
58-
59-
"
60-
61-
# prevent warning related to EOL differences which triggers an exception for some reason
62-
& git -c core.safecrlf=false diff --ignore-space-at-eol --exit-code -- "*.java"
63-
64-
if ($LastExitCode -ne 0) {
65-
$status = git status -s | Out-String
66-
Write-Host "
67-
The following files are out of date:
68-
$status
69-
"
82+
if ($hasError) {
7083
exit 1
7184
}
85+
exit 0

‎eng/scripts/TypeSpec-Compare-CurrentToCodegeneration.ps1‎

Lines changed: 82 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -15,92 +15,107 @@ Azure SDK for Java repository. One can also input service directory like: /sdk/s
1515

1616
param(
1717
[Parameter(Mandatory = $false)]
18-
[string]$Directory
18+
[string]$ServiceDirectories
1919
)
2020

21+
$SeparatorBars = "==========================================================================="
22+
2123
function Reset-Repository {
2224
# Clean up generated code, so that next step will not be affected.
2325
git reset --hard
2426
git clean -fd .
2527
}
2628

27-
$tspYamls = Get-ChildItem -Path $Directory -Filter "tsp-location.yaml" -Recurse
28-
if ($tspYamls.Count -eq 0) {
29-
Write-Host "
30-
31-
===========================================
32-
No TypeSpec files to regenerate
33-
===========================================
34-
35-
"
36-
exit 0
37-
}
38-
39-
Write-Host "
40-
41-
===========================================
42-
Installing typespec-client-generator-cli
43-
===========================================
29+
function Install-typespec-client-generator-cli {
30+
Write-Host "$SeparatorBars"
31+
Write-Host "Installing typespec-client-generator-cli"
32+
Write-Host "npm install -g @azure-tools/typespec-client-generator-cli"
33+
Write-Host "$SeparatorBars"
4434

45-
"
46-
47-
npm install -g @azure-tools/typespec-client-generator-cli
48-
49-
Write-Host "
50-
51-
===========================================
52-
Invoking tsp-client update
53-
===========================================
54-
55-
"
56-
57-
$failedSdk = $null
58-
foreach ($tspLocationPath in $tspYamls) {
59-
$sdkPath = (get-item $tspLocationPath).Directory.FullName
60-
Write-Host "Generate SDK for $sdkPath"
61-
Push-Location
62-
Set-Location -Path $sdkPath
63-
tsp-client update
35+
$output = npm install -g @azure-tools/typespec-client-generator-cli | Out-String
6436
if ($LastExitCode -ne 0) {
65-
$failedSdk += $sdkPath
37+
Write-Host "Error installing @azure-tools/typespec-client-generator-cli"
38+
Write-Host "$output"
39+
exit 1
6640
}
67-
# Update code snippets before comparing the diff
68-
Write-Host "Update code snippets"
69-
mvn --no-transfer-progress codesnippet:update-codesnippet
70-
Pop-Location
7141
}
7242

73-
if ($failedSdk.Length -gt 0) {
74-
Write-Host "Code generation failed for following modules: $failedSdk"
75-
Reset-Repository
76-
exit 1
77-
}
43+
# Returns true if there's an error, false otherwise
44+
function TypeSpec-Compare-CurrentToCodegeneration {
45+
param(
46+
[Parameter(Mandatory=$true)]
47+
$ServiceDirectory
48+
)
49+
50+
$tspYamls = Get-ChildItem -Path $ServiceDirectory -Filter "tsp-location.yaml" -Recurse
51+
if ($tspYamls.Count -eq 0) {
52+
Write-Host "$SeparatorBars"
53+
Write-Host "No TypeSpec files to regenerate for $ServiceDirectory"
54+
Write-Host "$SeparatorBars"
55+
return $false
56+
}
7857

79-
Write-Host "
58+
Write-Host "$SeparatorBars"
59+
Write-Host "Invoking tsp-client update for tsp-location.yaml files in $ServiceDirectory"
60+
Write-Host "$SeparatorBars"
61+
62+
$failedSdk = $null
63+
foreach ($tspLocationPath in $tspYamls) {
64+
$sdkPath = (get-item $tspLocationPath).Directory.FullName
65+
Write-Host "Generate SDK for $sdkPath"
66+
Push-Location
67+
Set-Location -Path $sdkPath
68+
tsp-client update | Out-Null
69+
if ($LastExitCode -ne 0) {
70+
$failedSdk += $sdkPath
71+
}
72+
# Update code snippets before comparing the diff
73+
Write-Host "Update code snippets"
74+
mvn --no-transfer-progress codesnippet:update-codesnippet | Out-Null
75+
Pop-Location
76+
}
77+
if ($failedSdk.Length -gt 0) {
78+
Write-Host "Code generation failed for following modules: $failedSdk"
79+
return $true
80+
}
8081

81-
==============
82-
Verify no diff
83-
==============
82+
Write-Host "$SeparatorBars"
83+
Write-Host "Verify no diff for TypeSpec generated files in $ServiceDirectory"
84+
Write-Host "$SeparatorBars"
8485

85-
"
86+
# prevent warning related to EOL differences which triggers an exception for some reason
87+
git -c core.safecrlf=false diff --ignore-space-at-eol --exit-code -- "*.java" ":(exclude)**/src/test/**" ":
88+
(exclude)**/src/samples/**" ":(exclude)**/src/main/**/implementation/**" ":(exclude)**/src/main/**/resourcemanager/**/*Manager.java"
8689

87-
# prevent warning related to EOL differences which triggers an exception for some reason
88-
git -c core.safecrlf=false diff --ignore-space-at-eol --exit-code -- "*.java" ":(exclude)**/src/test/**" ":
89-
(exclude)**/src/samples/**" ":(exclude)**/src/main/**/implementation/**" ":(exclude)**/src/main/**/resourcemanager/**/*Manager.java"
90+
if ($LastExitCode -ne 0) {
91+
$status = git status -s | Out-String
92+
Write-Host "The following files are out of date:"
93+
Write-Host "$status"
94+
return $true
95+
}
9096

91-
if ($LastExitCode -ne 0) {
92-
$status = git status -s | Out-String
93-
Write-Host "
94-
The following files are out of date:
95-
$status
96-
"
97-
Reset-Repository
98-
exit 1
97+
# Delete out TypeSpec temporary folders if they still exist.
98+
Get-ChildItem -Path $ServiceDirectory -Filter TempTypeSpecFiles -Recurse -Directory | ForEach-Object {
99+
Remove-Item -Path $_.FullName -Recurse -Force
100+
}
101+
return $false
99102
}
100103

101-
# Delete out TypeSpec temporary folders if they still exist.
102-
Get-ChildItem -Path $Directory -Filter TempTypeSpecFiles -Recurse -Directory | ForEach-Object {
103-
Remove-Item -Path $_.FullName -Recurse -Force
104+
$hasError = $false
105+
if ($ServiceDirectories) {
106+
Install-typespec-client-generator-cli
107+
foreach ($ServiceDirectory in $ServiceDirectories.Split(',')) {
108+
$path = "sdk/$ServiceDirectory"
109+
$result = TypeSpec-Compare-CurrentToCodegeneration $path
110+
if ($result) {
111+
$hasError = $true
112+
}
113+
}
114+
} else {
115+
Write-Host "The service directory list was empty for this PR, no TypeSpec files to regenerate"
104116
}
105-
106117
Reset-Repository
118+
if ($hasError) {
119+
exit 1
120+
}
121+
exit 0

‎sdk/healthdataaiservices/ci.yml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,13 +39,10 @@ parameters:
3939
type: boolean
4040
default: false
4141

42-
# Pending for PR https://github.com/Azure/azure-sdk-for-java/pull/42783 to be merged.
43-
# Then we can remove IgnoreVerifyTypeSpecCodeGenerationError.
4442
extends:
4543
template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml
4644
parameters:
4745
ServiceDirectory: healthdataaiservices
48-
IgnoreVerifyTypeSpecCodeGenerationError: true
4946
Artifacts:
5047
- name: azure-health-deidentification
5148
groupId: com.azure

0 commit comments

Comments
 (0)