Skip to content

Commit fd4959d

Browse files
JimSuplizioJimSuplizio
authored
Updates in preparation for PR pipeline (Azure#43904)
* Updates in preparation for PR pipeline * Fix variable name * Remove my initials from a comment and complete the comment
1 parent 6781272 commit fd4959d

File tree

6 files changed

+125
-24
lines changed

6 files changed

+125
-24
lines changed

‎eng/pipelines/templates/jobs/ci.tests.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@ parameters:
55
- name: ServiceDirectory
66
type: string
77
default: 'not-specified' # Set a default that breaks in obvious ways.
8+
- name: ExcludePaths
9+
type: object
10+
default: []
811
- name: TestPipeline
912
type: boolean
1013
default: false
@@ -104,6 +107,8 @@ jobs:
104107
- template: /eng/pipelines/templates/steps/initialize-test-environment.yml
105108
parameters:
106109
CheckoutRecordings: true
110+
ServiceDirectory: ${{parameters.ServiceDirectory}}
111+
ExcludePaths: ${{parameters.ExcludePaths}}
107112

108113
- template: /eng/common/testproxy/test-proxy-tool.yml
109114
parameters:

‎eng/pipelines/templates/jobs/ci.yml‎

Lines changed: 47 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -95,18 +95,11 @@ jobs:
9595
- '!sdk/**/test-recordings'
9696
- '!sdk/**/session-records'
9797

98-
# This is the case for the PullReqest pipeline which has a ServiceDirectory of 'auto'.
99-
# For the PullRequest pipeline, the packageInfo files generated from the diff end up
100-
# being the source of truth for the artifacts list.
101-
- ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}:
102-
- template: /eng/common/pipelines/templates/steps/save-package-properties.yml
103-
parameters:
104-
ServiceDirectory: ${{parameters.ServiceDirectory}}
105-
ExcludePaths: ${{parameters.ExcludePaths}}
106-
10798
- template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml
10899
parameters:
109100
JobType: 'Build'
101+
ServiceDirectory: ${{parameters.ServiceDirectory}}
102+
ExcludePaths: ${{parameters.ExcludePaths}}
110103

111104
- task: UsePythonVersion@0
112105
displayName: 'Use Python $(PythonVersion)'
@@ -203,8 +196,11 @@ jobs:
203196
arguments: >
204197
-BuildOutputDirectory $(System.DefaultWorkingDirectory)/build
205198
-ArtifactsList ('$(ArtifactsJson)' | ConvertFrom-Json | Select-Object name, groupId)
199+
-PackageInfoDir $(Build.ArtifactStagingDirectory)/PackageInfo
206200
-InformationAction Continue
207201
202+
# This only copies the ReleaseArtifacts but for the PR pipeline just copy
203+
# all of the built artifacts.
208204
- task: PowerShell@2
209205
displayName: 'Copy artifacts to staging'
210206
inputs:
@@ -215,6 +211,7 @@ jobs:
215211
-SourceDirectory $(System.DefaultWorkingDirectory)/build
216212
-TargetDirectory $(Build.ArtifactStagingDirectory)
217213
-Artifacts ('${{ replace(convertToJson(parameters.ReleaseArtifacts), '''', '`''') }}' | ConvertFrom-Json | Where-Object -Not skipPublishPackage )
214+
-PackageInfoDir $(Build.ArtifactStagingDirectory)/PackageInfo
218215
-InformationAction Continue
219216
220217
- task: Powershell@2
@@ -303,6 +300,8 @@ jobs:
303300
- template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml
304301
parameters:
305302
JobType: 'Analyze'
303+
ServiceDirectory: ${{parameters.ServiceDirectory}}
304+
ExcludePaths: ${{parameters.ExcludePaths}}
306305

307306
- task: PowerShell@2
308307
displayName: Generate directories variable for sparse checkout
@@ -332,9 +331,14 @@ jobs:
332331
ServiceDirectory: "template"
333332
TestPipeline: ${{ parameters.TestPipeline }}
334333

335-
- template: /eng/common/pipelines/templates/steps/verify-readme.yml
336-
parameters:
337-
ScanPath: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}
334+
- ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}:
335+
- template: /eng/common/pipelines/templates/steps/verify-readmes.yml
336+
parameters:
337+
PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
338+
- ${{ else }}:
339+
- template: /eng/common/pipelines/templates/steps/verify-readme.yml
340+
parameters:
341+
ScanPath: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}
338342

339343
- template: /eng/common/pipelines/templates/steps/verify-links.yml
340344
parameters:
@@ -345,9 +349,24 @@ jobs:
345349
Directory: sdk/${{ parameters.ServiceDirectory }}
346350
CheckLinkGuidance: $true
347351

348-
- template: /eng/common/pipelines/templates/steps/verify-samples.yml
349-
parameters:
350-
ServiceDirectory: ${{ parameters.ServiceDirectory }}
352+
- ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}:
353+
- pwsh: |
354+
$diffJson = Get-Content '$(Build.ArtifactStagingDirectory)/diff/diff.json' -Raw
355+
$diff = ConvertFrom-Json $diffJson
356+
$changedServicesString = ''
357+
if ($diff.ChangedServices) {
358+
$changedServicesString = $diff.ChangedServices -join ","
359+
}
360+
Write-Host "changedServicesString='$changedServicesString'"
361+
Write-Host "##vso[task.setvariable variable=SamplesServiceDirectories;]$changedServicesString"
362+
displayName: Get samples service directories from PR diff
363+
- template: /eng/common/pipelines/templates/steps/verify-samples.yml
364+
parameters:
365+
ServiceDirectories: $(SamplesServiceDirectories)
366+
- ${{ else }}:
367+
- template: /eng/common/pipelines/templates/steps/verify-samples.yml
368+
parameters:
369+
ServiceDirectory: ${{ parameters.ServiceDirectory }}
351370

352371
# Use BasePathLength of 38 instead of the default 49 as some released files fail when the number is higher.
353372
- template: /eng/common/pipelines/templates/steps/verify-path-length.yml
@@ -356,13 +375,18 @@ jobs:
356375
BasePathLength: 38
357376

358377
- ${{if ne(parameters.SDKType, 'data')}}:
359-
- ${{ each artifact in parameters.Artifacts }}:
360-
- ${{if ne(artifact.skipVerifyChangelog, 'true')}}:
361-
- template: /eng/common/pipelines/templates/steps/verify-changelog.yml
362-
parameters:
363-
PackageName: ${{artifact.name}}
364-
ServiceName: ${{parameters.ServiceDirectory}}
365-
ForRelease: false
378+
- ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}:
379+
- template: /eng/common/pipelines/templates/steps/verify-changelogs.yml
380+
parameters:
381+
PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
382+
- ${{ else }}:
383+
- ${{ each artifact in parameters.Artifacts }}:
384+
- ${{if ne(artifact.skipVerifyChangelog, 'true')}}:
385+
- template: /eng/common/pipelines/templates/steps/verify-changelog.yml
386+
parameters:
387+
PackageName: ${{artifact.name}}
388+
ServiceName: ${{parameters.ServiceDirectory}}
389+
ForRelease: false
366390

367391
# We `install` the code quality reports tooling into our local m2 cache separately from building the Maven project
368392
# reports. This means it is available as part of that, but also so that this is not documented in the project report.
@@ -434,6 +458,7 @@ jobs:
434458
AdditionalParameters:
435459
SDKType: ${{ parameters.SDKType }}
436460
ServiceDirectory: ${{ parameters.ServiceDirectory }}
461+
ExcludePaths: ${{ parameters.ExcludePaths }}
437462
TestPipeline: ${{ parameters.TestPipeline }}
438463
PreTestSteps: ${{ parameters.PreTestSteps }}
439464
PreBuildSteps: ${{ parameters.PreBuildSteps }}

‎eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,24 @@ parameters:
1515
- name: UseCache
1616
type: boolean
1717
default: true
18+
- name: ServiceDirectory
19+
type: string
20+
default: 'not-specified' # Set a default that breaks in obvious ways.
21+
- name: ExcludePaths
22+
type: object
23+
default: []
1824

1925
steps:
26+
27+
# This is the case for the PullReqest pipeline which has a ServiceDirectory of 'auto'.
28+
# For the PullRequest pipeline, the packageInfo files generated from the diff end up
29+
# being the source of truth for the artifacts list.
30+
- ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.ServiceDirectory, 'auto')) }}:
31+
- template: /eng/common/pipelines/templates/steps/save-package-properties.yml
32+
parameters:
33+
ServiceDirectory: ${{parameters.ServiceDirectory}}
34+
ExcludePaths: ${{parameters.ExcludePaths}}
35+
2036
# Create the ProjectList and ProjectListSha256 variables.
2137
# The ProjectList is used by subsequent steps and tasks to run CI.
2238
# The ProjectListSha256 is used by the cache key, previously the cache key was using ProjectList but it was found

‎eng/pipelines/templates/steps/initialize-test-environment.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,12 @@ parameters:
55
- name: OS
66
type: string
77
default: $(OSVmImage)
8+
- name: ServiceDirectory
9+
type: string
10+
default: 'not-specified' # Set a default that breaks in obvious ways.
11+
- name: ExcludePaths
12+
type: object
13+
default: []
814

915
steps:
1016
# Skip sparse checkout for the `azure-sdk-for-<lang>-pr` private mirrored repositories
@@ -14,6 +20,7 @@ steps:
1420
parameters:
1521
Paths:
1622
- '**/*.xml'
23+
- '**/ci*.yml' # necessary for save-package-properties.yml
1724
- ${{ if not(parameters.CheckoutRecordings) }}:
1825
- '!sdk/**/test-recordings'
1926
- '!sdk/**/session-records'
@@ -44,6 +51,8 @@ steps:
4451
- template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml
4552
parameters:
4653
JobType: 'test'
54+
ServiceDirectory: ${{parameters.ServiceDirectory}}
55+
ExcludePaths: ${{parameters.ExcludePaths}}
4756

4857
# This call is used generate the sparse checkout variables for non-FromSource runs.
4958
- task: PowerShell@2

‎eng/scripts/Stage-MavenPackageArtifacts.ps1‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
param(
22
[Parameter(Mandatory=$true)][string]$SourceDirectory,
33
[Parameter(Mandatory=$true)][string]$TargetDirectory,
4-
[Parameter(Mandatory=$true)][array]$Artifacts
4+
[Parameter(Mandatory=$false)][array]$Artifacts,
5+
[Parameter(Mandatory=$false)][string] $PackageInfoDir = $null
56
)
67

78
$ErrorActionPreference = "Stop"
@@ -11,6 +12,28 @@ Write-host "Target Directory is: $TargetDirectory"
1112

1213
. $PSScriptRoot\MavenPackaging.ps1
1314

15+
if ($Artifacts -eq $null) {
16+
$Artifacts = @()
17+
}
18+
19+
if ($Artifacts.Count -eq 0) {
20+
if (-not $PackageInfoDir -or (-not (Test-Path -Path $PackageInfoDir))) {
21+
LogError "Artifacts list was empty and PackageInfoDir was null or incorrect."
22+
exit(1)
23+
}
24+
Write-Host "Artifacts List was empty, getting Artifacts from PackageInfoDir=$PackageInfoDir"
25+
[array]$packageInfoFiles = Get-ChildItem -Path $PackageInfoDir "*.json"
26+
foreach($packageInfoFile in $packageInfoFiles) {
27+
$packageInfoJson = Get-Content $packageInfoFile -Raw
28+
$packageInfo = ConvertFrom-Json $packageInfoJson
29+
$Artifacts += New-Object PSObject -Property @{
30+
groupId = $packageInfo.Group
31+
name = $packageInfo.ArtifactName
32+
}
33+
34+
}
35+
}
36+
1437
Write-Host "Searching for packages in: $SourceDirectory"
1538
$packageDetails = Get-MavenPackageDetails -ArtifactDirectory $SourceDirectory
1639
Write-Host "Found $($packageDetails.Count) packages in: $SourceDirectory"

‎eng/scripts/Verify-Required-Maven-Artifacts.ps1‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,37 @@
99
param(
1010
[Parameter(Mandatory=$true)][string]$BuildOutputDirectory,
1111
# ArtifactsList will be using ('${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json | Select-Object name, groupId)
12-
[Parameter(Mandatory=$true)][array] $ArtifactsList
12+
[Parameter(Mandatory=$false)][array] $ArtifactsList,
13+
[Parameter(Mandatory=$false)][string] $PackageInfoDir = $null
1314
)
1415
. "${PSScriptRoot}/../common/scripts/common.ps1"
1516

1617
Write-Host "BuildOutputDirectory=$($BuildOutputDirectory)"
1718
Write-Host "BuildOutputDirectory contents"
1819
Get-ChildItem -Path $BuildOutputDirectory -Recurse -Name
1920

21+
if (-not $ArtifactsList) {
22+
$ArtifactsList = @()
23+
}
24+
25+
if ($ArtifactsList.Count -eq 0) {
26+
if (-not $PackageInfoDir -or (-not (Test-Path -Path $PackageInfoDir))) {
27+
LogError "ArtifactsList was empty and PackageInfoDir was null or incorrect."
28+
exit(1)
29+
}
30+
Write-Host "Artifacts List was empty, getting Artifacts from PackageInfoDir=$PackageInfoDir"
31+
[array]$packageInfoFiles = Get-ChildItem -Path $PackageInfoDir "*.json"
32+
foreach($packageInfoFile in $packageInfoFiles) {
33+
$packageInfoJson = Get-Content $packageInfoFile -Raw
34+
$packageInfo = ConvertFrom-Json $packageInfoJson
35+
$ArtifactsList += New-Object PSObject -Property @{
36+
groupId = $packageInfo.Group
37+
name = $packageInfo.ArtifactName
38+
}
39+
40+
}
41+
}
42+
2043
Write-Host ""
2144
Write-Host "ArtifactsList:"
2245
$ArtifactsList | Format-Table -Property GroupId, Name | Out-String | Write-Host

0 commit comments

Comments
 (0)