Skip to content

Commit cbd4df9

Browse files
scbeddscbedd
authored
Handle AOTCompat and BuildSnippets in net - pullrequest (Azure#48536)
* add parsing of ci.yml parameters BuildSnippet, CheckAOTCompat, and AOTTestInputs to each package's CIParameters dictionary * conditionally activate snippet build and aotcompat during pullrequest builds using new per-package metadata * reducing build and analyze PR batches to only packages that were DIRECTLY changed in the given PR * fix issue with invalid artifact name for APIView artifact during "Detect API Changes"
1 parent 917d827 commit cbd4df9

File tree

10 files changed

+222
-55
lines changed

10 files changed

+222
-55
lines changed

‎eng/pipelines/templates/steps/analyze.yml‎

Lines changed: 10 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -96,33 +96,18 @@ steps:
9696
parameters:
9797
LogFilePath: $(Build.ArtifactStagingDirectory)/rebuild.binlog
9898

99-
- pwsh: |
100-
$failed = $false
101-
if ("$(ProjectNames)") {
102-
$services = "$(ChangedServices)" -split ","
103-
104-
foreach ($service in $services) {
105-
Write-Host "building $service"
106-
dotnet build eng/service.proj -warnaserror `
107-
/t:rebuild `
108-
/p:DebugType=none `
109-
/p:SDKType=${{ parameters.SDKType }} `
110-
/p:ServiceDirectory=$service `
111-
/p:IncludePerf=false `
112-
/p:IncludeStress=false `
113-
/p:PublicSign=false `
114-
/p:Configuration=$(BuildConfiguration) `
115-
/p:EnableSourceLink=false `
116-
/p:ProjectListOverrideFile="" `
117-
/p:BuildSnippets=true `
118-
$(DiagnosticArguments)
119-
}
120-
}
121-
else {
122-
Write-Host "No services were directly changed. Skipping snippet build."
123-
}
99+
- task: Powershell@2
124100
displayName: Build snippets
125101
condition: and(succeeded(), eq(${{ parameters.BuildSnippets }}, true))
102+
inputs:
103+
filePath: $(Build.SourcesDirectory)/eng/scripts/Build-Snippets.ps1
104+
arguments: >
105+
-PackageInfoFolder "$(Build.ArtifactStagingDirectory)/PackageInfo"
106+
-SdkType "${{ parameters.SDKType }}"
107+
-ProjectNames "$(ProjectNames)"
108+
-DiagnosticArguments "$(DiagnosticArguments)"
109+
-BuildConfiguration "$(BuildConfiguration)"
110+
pwsh: true
126111

127112
- ${{ if eq(parameters.ServiceDirectory, 'auto') }}:
128113
- template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml

‎eng/pipelines/templates/steps/aot-compatibility.yml‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,29 @@
11
parameters:
22
ServiceDirectory: ''
3+
PackageInfoFolder: ''
34
AOTTestInputs: []
45

56
steps:
6-
- ${{ each aotTestInput in parameters.AOTTestInputs }}:
7+
8+
- ${{ if eq(parameters.ServiceDirectory, 'auto') }}:
79
- task: Powershell@2
8-
displayName: Check for AOT compatibility regressions in ${{ aotTestInput.ArtifactName }}
10+
displayName: Check for AOT compatibility regressions for PR
911
inputs:
1012
targetType: filepath
11-
filePath: $(Build.SourcesDirectory)/eng/scripts/compatibility/Check-AOT-Compatibility.ps1
13+
filePath: $(Build.SourcesDirectory)/eng/scripts/compatibility/Check-AOT-Compatibility-For-PR.ps1
1214
arguments: >-
13-
-ServiceDirectory ${{ parameters.ServiceDirectory }}
14-
-PackageName ${{ aotTestInput.ArtifactName }}
15-
-ExpectedWarningsFilePath ${{ aotTestInput.ExpectedWarningsFilePath }}
16-
workingDirectory: $(Build.SourcesDirectory)/eng/scripts/compatibility
15+
-PackageInfoFolder "${{ parameters.PackageInfoFolder }}"
16+
-ProjectNames "$(ProjectNames)"
17+
workingDirectory: $(Build.SourcesDirectory)/eng/scripts/compatibility
18+
- ${{ else }}:
19+
- ${{ each aotTestInput in parameters.AOTTestInputs }}:
20+
- task: Powershell@2
21+
displayName: Check for AOT compatibility regressions in ${{ aotTestInput.ArtifactName }}
22+
inputs:
23+
targetType: filepath
24+
filePath: $(Build.SourcesDirectory)/eng/scripts/compatibility/Check-AOT-Compatibility.ps1
25+
arguments: >-
26+
-ServiceDirectory ${{ parameters.ServiceDirectory }}
27+
-PackageName ${{ aotTestInput.ArtifactName }}
28+
-ExpectedWarningsFilePath ${{ aotTestInput.ExpectedWarningsFilePath }}
29+
workingDirectory: $(Build.SourcesDirectory)/eng/scripts/compatibility

‎eng/pipelines/templates/steps/build.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,7 @@ steps:
153153
parameters:
154154
ServiceDirectory: ${{ parameters.ServiceDirectory }}
155155
AOTTestInputs: ${{ parameters.AOTTestInputs }}
156+
PackageInfoFolder: '$(Build.ArtifactStagingDirectory)/PackageInfo'
156157

157158
- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
158159
displayName: "Component Detection"
@@ -180,6 +181,8 @@ steps:
180181
- template: /eng/common/pipelines/templates/steps/detect-api-changes.yml
181182
parameters:
182183
Artifacts: ${{ parameters.Artifacts }}
184+
${{ if eq(parameters.ServiceDirectory, 'auto') }}:
185+
ArtifactName: 'packages_$(System.JobName)'
183186

184187
- template: /eng/common/pipelines/templates/steps/validate-all-packages.yml
185188
parameters:

‎eng/pipelines/templates/steps/pr-matrix-presteps.yml‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,31 @@ steps:
3434
ArtifactPath: '$(Build.ArtifactStagingDirectory)/PackageInfoPublishing'
3535

3636
- pwsh: |
37-
$packageInfos = Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)/PackageInfo" -Filter "*.json" -Recurse
38-
$packageInfos | ForEach-Object {
39-
$content = Get-Content -Raw -Path $_.FullName | ConvertFrom-Json
40-
$content.IncludedForValidation = $false
41-
$content | ConvertTo-Json -Depth 32 | Out-File $_.FullName
37+
$packageInfos = Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)/PackageInfo" -Filter "*.json" -Recurse `
38+
| ForEach-Object { Get-Content $_.FullName | ConvertFrom-Json }
39+
40+
$directPackages = $packageInfos | Where-Object { $_.IncludedForValidation -eq $false }
41+
42+
if (!$directPackages) {
43+
Write-Host "Falling back to Azure.Template from the original package list. There are no other direct packages for assembly."
44+
45+
# grab just the Azure.Template package from the original list, set it to direct
46+
Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)/PackageInfo" -Filter "Azure.Template.json" -Recurse |
47+
ForEach-Object {
48+
$json = Get-Content $_.FullName | ConvertFrom-Json
49+
$json.IncludedForValidation = $false
50+
$json | ConvertTo-Json -Depth 100 | Set-Content $_.FullName
51+
}
4252
}
43-
displayName: Set all packages to direct
53+
54+
# remove all the indirect packages from the list of packages that we want to generate matrix for
55+
Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)/PackageInfo" -Filter "*.json" -Recurse |
56+
ForEach-Object {
57+
$json = Get-Content $_.FullName | ConvertFrom-Json
58+
if ($json.IncludedForValidation -eq $true) {
59+
Remove-Item $_.FullName -Force
60+
}
61+
}
62+
displayName: Only target direct packages for build and test
4463
condition: and(succeededOrFailed(), eq(${{ parameters.ForceDirect }}, true))
4564

‎eng/scripts/Build-Snippets.ps1‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
param(
2+
[Parameter(Mandatory=$true)]
3+
[string] $PackageInfoFolder,
4+
[Parameter(Mandatory=$true)]
5+
[string] $SdkType,
6+
[Parameter(Mandatory=$true)]
7+
[string] $ProjectNames,
8+
[Parameter(Mandatory=$false)]
9+
[string] $BuildConfiguration,
10+
[Parameter(Mandatory=$false)]
11+
[string] $DiagnosticArguments
12+
)
13+
. $PSScriptRoot/splittestdependencies/generate-dependency-functions.ps1
14+
15+
$ErrorActionPreference = "Stop"
16+
Set-StrictMode -Version 4
17+
18+
$TargetProjects = $ProjectNames -split ","
19+
$RepoRoot = Resolve-Path (Join-Path "$PSScriptRoot" ".." "..")
20+
21+
$snippetEnabledProjects = Get-ChildItem -Recurse "$PackageInfoFolder" *.json `
22+
| Foreach-Object { Get-Content -Raw -Path $_.FullName | ConvertFrom-Json } `
23+
| Where-Object { $_.ArtifactName -in $TargetProjects -and $_.CIParameters.BuildSnippets -eq $true }
24+
25+
$scopedFile = Write-PkgInfoToDependencyGroupFile -OutputPath "$RepoRoot" -PackageInfoFolder $PackageInfoFolder -ProjectNames $TargetProjects
26+
27+
Write-Host "Writing project list to $scopedFile"
28+
Write-Host (Get-Content -Raw -Path $scopedFile)
29+
30+
if ($snippetEnabledProjects) {
31+
dotnet build eng/service.proj -warnaserror `
32+
/t:rebuild `
33+
/p:DebugType=none `
34+
/p:SDKType=$SdkType `
35+
/p:ServiceDirectory=* `
36+
/p:IncludePerf=false `
37+
/p:IncludeStress=false `
38+
/p:PublicSign=false `
39+
/p:Configuration="$BuildConfiguration" `
40+
/p:EnableSourceLink=false `
41+
/p:BuildSnippets=true `
42+
/p:ProjectListOverrideFile="$scopedFile" `
43+
$DiagnosticArguments
44+
}
45+
else {
46+
Write-Host "There are no projects with BuildSnippets set to true. Evaluated projects: $ProjectNames"
47+
}
48+

‎eng/scripts/Language-Settings.ps1‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,42 @@ function Get-AllPackageInfoFromRepo($serviceDirectory)
116116
}
117117
}
118118

119+
$ciProps = $pkgProp.GetCIYmlForArtifact()
120+
121+
if ($ciProps) {
122+
# CheckAOTCompat is opt _in_, so we should default to false if not specified
123+
$shouldAot = GetValueSafelyFrom-Yaml $ciProps.ParsedYml @("extends", "parameters", "CheckAOTCompat")
124+
if ($shouldAot) {
125+
$parsedBool = $null
126+
if ([bool]::TryParse($shouldAot, [ref]$parsedBool)) {
127+
$pkgProp.CIParameters["CheckAOTCompat"] = $parsedBool
128+
}
129+
130+
# when AOTCompat is true, there is an additional parameter we need to retrieve
131+
$aotArtifacts = GetValueSafelyFrom-Yaml $ciProps.ParsedYml @("extends", "parameters", "AOTTestInputs")
132+
if ($aotArtifacts) {
133+
$aotArtifacts = $aotArtifacts | Where-Object { $_.ArtifactName -eq $pkgProp.ArtifactName }
134+
$pkgProp.CIParameters["AOTTestInputs"] = $aotArtifacts
135+
}
136+
}
137+
else {
138+
$pkgProp.CIParameters["CheckAOTCompat"] = $false
139+
$pkgProp.CIParameters["AOTTestInputs"] = @()
140+
}
141+
142+
# BuildSnippets is opt _out_, so we should default to true if not specified
143+
$shouldSnippet = GetValueSafelyFrom-Yaml $ciProps.ParsedYml @("extends", "parameters", "BuildSnippets")
144+
if ($null -ne $shouldSnippet) {
145+
$parsedBool = $null
146+
if ([bool]::TryParse($shouldSnippet, [ref]$parsedBool)) {
147+
$pkgProp.CIParameters["BuildSnippets"] = $parsedBool
148+
}
149+
}
150+
else {
151+
$pkgProp.CIParameters["BuildSnippets"] = $true
152+
}
153+
}
154+
119155
$allPackageProps += $pkgProp
120156
}
121157

‎eng/scripts/compatibility/Check-AOT-Compatibility-For-PR.ps1‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
<#
2+
.DESCRIPTION
3+
This script checks AOT compatibility packages within the specified folder, limited to just the projects named in the ProjectNames param.
4+
5+
.PARAMETER PackageInfoFolder
6+
The package info folder containing the JSON files with package metadata.
7+
8+
.PARAMETER ProjectNames
9+
The comma-separated list of project names targeted for the current batch.
10+
11+
#>
12+
param(
13+
[Parameter(Mandatory=$true)]
14+
[string]$PackageInfoFolder,
15+
[string]$ProjectNames
16+
)
17+
18+
if (-not (Test-Path $PackageInfoFolder)) {
19+
Write-Error "Package info folder '$PackageInfoFolder' does not exist."
20+
exit 1
21+
}
22+
23+
$projectNamesArray = @()
24+
if ($ProjectNames) {
25+
$projectNamesArray = $ProjectNames.Split(',') | ForEach-Object { $_.Trim() }
26+
}
27+
else {
28+
Write-Error "ProjectNames parameter doesn't target any packages. Please provide a comma-separated list of project names."
29+
exit 0
30+
}
31+
32+
$filteredPackages = Get-ChildItem -Path $PackageInfoFolder -Filter "*.json" -File `
33+
| ForEach-Object { Get-Content -Raw $_.FullName | ConvertFrom-Json } `
34+
| Where-Object { $projectNamesArray.Contains($_.ArtifactName) }
35+
36+
$failedAotChecks = $false
37+
foreach ($package in $filteredPackages) {
38+
if ($package.CIParameters.CheckAOTCompat -and $package.CIParameters.AOTTestInputs) {
39+
Write-Host "Running Check-AOT-Compatibility.ps1 for Package: $($package.ArtifactName) Service $($package.ServiceDirectory)"
40+
$aotDetails = $package.CIParameters.AOTTestInputs
41+
& $PSScriptRoot/Check-AOT-Compatibility.ps1 `
42+
-PackageName $package.ArtifactName `
43+
-ServiceDirectory $package.ServiceDirectory `
44+
-ExpectedWarningsFilePath $aotDetails.ExpectedWarningsFilePath
45+
46+
if ($LASTEXITCODE -ne 0) {
47+
$failedAotChecks = $true
48+
}
49+
}
50+
}
51+
52+
if ($failedAotChecks) {
53+
Write-Error "AOT compatibility check failed for one or more packages."
54+
exit 1
55+
}

‎eng/scripts/splittestdependencies/generate-dependency-functions.ps1‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,21 @@ function Split-Project-File-To-Groups($ProjectFile, $NumberOfTestsPerJob, $Exclu
2323
return ,$projGroup
2424
}
2525

26+
function Write-PkgInfoToDependencyGroupFile([string]$OutputPath, [string]$PackageInfoFolder, [string[]]$ProjectNames) {
27+
$packageProperties = Get-ChildItem -Recurse "$PackageInfoFolder" *.json `
28+
| Foreach-Object { Get-Content -Raw -Path $_.FullName | ConvertFrom-Json }
29+
$changedProjects = $packageProperties | Where-Object { $ProjectNames -contains $_.ArtifactName }
30+
| ForEach-Object { "$($_.DirectoryPath)/**/*.csproj"; }
31+
32+
$projectsForGeneration = ($changedProjects | ForEach-Object { "`$(RepoRoot)$_" } | Sort-Object)
33+
34+
$projectGroups = @()
35+
$projectGroups += ,$projectsForGeneration
36+
37+
$outputFile = (Write-Test-Dependency-Group-To-Files -ProjectFileConfigName "packages" -ProjectGroups $projectGroups -MatrixOutputFolder $OutputPath)[0]
38+
39+
return $outputFile
40+
}
2641

2742
# Take the group contains the project list. Write each group into a project file which can be used by dotnet test command.
2843
# Return the array of project files

‎eng/scripts/splittestdependencies/set-artifact-packages.ps1‎

Lines changed: 8 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -8,35 +8,27 @@ param (
88
. $PSScriptRoot/generate-dependency-functions.ps1
99

1010
$RepoRoot = Resolve-Path (Join-Path "$PSScriptRoot" ".." ".." "..")
11+
$packageSet = "$ProjectNames" -split ","
1112

12-
# set changed services given the set of changed packages, this will mean that
13-
# ChangedServices will be appropriate for the batched set of packages if that is indeed how
14-
# we set the targeted artifacts
13+
# retrieve the package info files
1514
$packageProperties = Get-ChildItem -Recurse "$PackageInfoFolder" *.json `
1615
| Foreach-Object { Get-Content -Raw -Path $_.FullName | ConvertFrom-Json }
1716

18-
$packageSet = "$ProjectNames" -split ","
19-
20-
$changedServicesArray = $packageProperties | Where-Object { $packageSet -contains $_.ArtifactName }
21-
| ForEach-Object { $_.ServiceDirectory } | Get-Unique
17+
# filter the package info files to only those that are part of the targeted batch (present in $ProjectNames arg)
18+
# so that we can accurate determine the affected services for the current batch
19+
$changedServicesArray = $packageProperties | Where-Object { $packageSet -contains $_.ArtifactName } `
20+
| ForEach-Object { $_.ServiceDirectory } | Get-Unique
2221
$changedServices = $changedServicesArray -join ","
2322

24-
$changedProjects = $packageProperties | Where-Object { $packageSet -contains $_.ArtifactName }
25-
| ForEach-Object { "$($_.DirectoryPath)/**/*.csproj"; }
26-
27-
$projectsForGeneration = ($changedProjects | ForEach-Object { "`$(RepoRoot)$_" } | Sort-Object)
28-
$projectGroups = @()
29-
$projectGroups += ,$projectsForGeneration
30-
3123
if ($SetOverrideFile) {
32-
$outputFile = (Write-Test-Dependency-Group-To-Files -ProjectFileConfigName "packages" -ProjectGroups $projectGroups -MatrixOutputFolder $OutputPath)[0]
24+
$outputFile = Write-PkgInfoToDependencyGroupFile -OutputPath $OutputPath -PackageInfoFolder $PackageInfoFolder -ProjectNames $packageSet
3325
Get-ChildItem -Recurse $OutputPath | ForEach-Object { Write-Host "Dumping $($_.FullName)"; Get-Content -Raw -Path $_.FullName | Write-Host }
3426
# the projectlistoverride file must be provided as a relative path
3527
$relativeOutputPath = [System.IO.Path]::GetRelativePath($RepoRoot, "$OutputPath/$outputFile")
3628
Write-Host "##vso[task.setvariable variable=ProjectListOverrideFile;]$relativeOutputPath"
3729
}
3830

39-
# Filter JSON files to only keep those matching package names in packageSet
31+
# remove any package.json files that are not part of the targeted batch
4032
Get-ChildItem -Recurse "$PackageInfoFolder" *.json | ForEach-Object {
4133
$fileContent = Get-Content -Raw -Path $_.FullName | ConvertFrom-Json
4234
if ($packageSet -notcontains $fileContent.Name) {

‎sdk/pullrequest.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,4 +22,5 @@ extends:
2222
template: /eng/pipelines/templates/stages/archetype-sdk-client.yml
2323
parameters:
2424
ServiceDirectory: ${{ parameters.Service }}
25-
BuildSnippets: false # setting this to false until we can handle ci.yml level parameter customizations
25+
CheckAOTCompat: true
26+
BuildSnippets: true

0 commit comments

Comments
 (0)