Skip to content

Commit 12f3d2f

Browse files
authored
Implemented the validation of test coverage in CI when submitting PR (#21339)
1 parent efbc051 commit 12f3d2f

File tree

3 files changed

+83
-11
lines changed

3 files changed

+83
-11
lines changed

.azure-pipelines/util/test-steps.yml

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ steps:
5454

5555
- pwsh: |
5656
Install-Module -Name Pester -RequiredVersion 4.10.1 -Force
57-
Install-Module -Name Az.Accounts -Scope CurrentUser -Force
57+
Import-Module (Join-Path -Path . -ChildPath Az.Accounts | Join-Path -ChildPath Az.Accounts.psd1) -Force
5858
if ($IsWindows) { $sp = ";" } else { $sp = ":" }
5959
$env:PSModulePath = $env:PSModulePath + $sp + (pwd).Path
6060
$rootFolder = (Get-item $PWD).Parent.Parent
@@ -70,6 +70,13 @@ steps:
7070
env:
7171
PowerShellPlatform: ${{ parameters.powerShellPlatform }}
7272

73+
- task: PowerShell@2
74+
displayName: Analyze Test Coverage
75+
inputs:
76+
pwsh: true
77+
targetType: filePath
78+
filePath: ./tools/TestFx/Coverage/ValidateTestCoverage.ps1
79+
7380
- pwsh: |
7481
$PipelineResult = Get-Content PipelineResult.json | ConvertFrom-Json
7582
$FailedModuleList = $PipelineResult.test.Details[0].Modules | Where-Object { $_.Status -eq "Failed" } | ForEach-Object { $_.Module }
@@ -88,4 +95,4 @@ steps:
8895

8996
- template: publish-artifacts-steps.yml
9097
parameters:
91-
artifactName: test-${{ parameters.testFramework }}-${{ parameters.osName }}
98+
artifactName: test-${{ parameters.testFramework }}-${{ parameters.osName }}

tools/TestFx/Coverage/AnalyzeTestCoverage.ps1

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ foreach ($moduleName in $allModules) {
170170
$cvgParameter = ($totalTestedParametersCount / $totalParametersCount).ToString("P2")
171171

172172
$cvgReport = [PSCustomObject]@{
173-
Module = $moduleName
173+
Module = $simpleModuleName
174174
TotalCommands = $totalCommandsCount
175175
TestedCommands = $totalTestedCommandsCount
176176
CommandCoverage = $cvgCommand
@@ -185,7 +185,7 @@ foreach ($moduleName in $allModules) {
185185

186186
if ($CalcBaseline.IsPresent) {
187187
$cvgBaseline = [PSCustomObject]@{
188-
Module = $moduleName
188+
Module = $simpleModuleName
189189
CommandCoverage = $cvgCommand
190190
}
191191
$cvgBaseline | Export-Csv -Path $cvgBaselineCsv -Encoding utf8 -NoTypeInformation -Append -Force
@@ -198,14 +198,16 @@ foreach ($moduleName in $allModules) {
198198
Write-Host
199199
}
200200

201-
$cvgOverall = ($overallTestedCommandsCount / $overallCommandsCount).ToString("P2")
202-
$cvgReportOverall = [PSCustomObject]@{
203-
Module = "Total"
204-
TotalCommands = $overallCommandsCount
205-
TestedCommands = $overallTestedCommandsCount
206-
CommandCoverage = $cvgOverall
201+
if ($CalcBaseline.IsPresent) {
202+
$cvgOverall = ($overallTestedCommandsCount / $overallCommandsCount).ToString("P2")
203+
$cvgReportOverall = [PSCustomObject]@{
204+
Module = "Total"
205+
TotalCommands = $overallCommandsCount
206+
TestedCommands = $overallTestedCommandsCount
207+
CommandCoverage = $cvgOverall
208+
}
209+
$cvgReportOverall | Export-Csv -Path $cvgReportCsv -Encoding utf8 -NoTypeInformation -Append -Force
207210
}
208-
$cvgReportOverall | Export-Csv -Path $cvgReportCsv -Encoding utf8 -NoTypeInformation -Append -Force
209211

210212
Write-Host "##[section]Overall commands # : $overallCommandsCount" -ForegroundColor Magenta
211213
Write-Host "##[section]Overall tested commands # : $overallTestedCommandsCount" -ForegroundColor Magenta
Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
& (Join-Path -Path $PSScriptRoot -ChildPath "AnalyzeTestCoverage.ps1")
2+
3+
$cvgRootDir = (Get-AzConfig -TestCoverageLocation).Value
4+
$cvgResultsDir = Join-Path -Path $cvgRootDir -ChildPath "TestCoverageAnalysis" | Join-Path -ChildPath "Results"
5+
$rptCsv = Join-Path -Path $cvgResultsDir -ChildPath "Report.csv"
6+
if (!(Test-Path -Path $rptCsv -PathType Leaf)) {
7+
Write-Warning "No test coverage analysis result was found!"
8+
return
9+
}
10+
11+
$blCsv = Join-Path -Path $PSScriptRoot -ChildPath "Baseline.csv"
12+
if (!(Test-Path -Path $blCsv -PathType Leaf)) {
13+
Write-Warning "No test coverage baseline was found!"
14+
return
15+
}
16+
17+
$repoRootDir = $PSScriptRoot | Split-Path | Split-Path | Split-Path
18+
$cipJson = Join-Path -Path $repoRootDir -ChildPath "artifacts" | Join-Path -ChildPath "PipelineResult" | Join-Path -ChildPath "CIPlan.json"
19+
if (!(Test-Path -Path $cipJson -PathType Leaf)) {
20+
Write-Warning "No CI plan was found!"
21+
return
22+
}
23+
24+
$cipJson = Get-Content -Path $cipJson -Raw | ConvertFrom-Json
25+
$testedModules = $cipJson.test | Where-Object { $_ -ne "Accounts" }
26+
27+
$toolsDir = $PSScriptRoot | Split-Path | Split-Path
28+
$funcTestStatus = Join-Path -Path $toolsDir -ChildPath "ExecuteCIStep.ps1"
29+
. $funcTestStatus
30+
31+
$rptData = Import-Csv -Path $rptCsv
32+
$blData = Import-Csv -Path $blCsv
33+
34+
$cvgMessageHeader50 = "|Type|Title|Current Coverage|Description|`n|---|---|---|---|`n"
35+
$cvgMessageHeader80 = "|Type|Title|Current Coverage|Last Coverage|Description|`n|---|---|---|---|---|`n"
36+
37+
$rptData | Where-Object Module -in $testedModules | ForEach-Object {
38+
$module = $_.Module
39+
Write-Host "##[section]Validating test coverage for module $module..."
40+
41+
$cmdCvg = $_.CommandCoverage
42+
$cmdCvgD = [decimal]$cmdCvg.TrimEnd("%") / 100
43+
44+
Write-Host "Test coverage for module $module is $cmdCvg."
45+
if ($cmdCvgD -lt 0.5) {
46+
Write-Warning "Test coverage for module $module is less than 50% !"
47+
$cvgMessageBody50 = "|⚠️|Test Coverage Less Than 50%|$cmdCvg|Test coverage for the module cannot be lower than 50%.|`n"
48+
Set-ModuleTestStatusInPipelineResult -ModuleName "Az.$module" -Status Warning -Content ($cvgMessageHeader50 + $cvgMessageBody50)
49+
}
50+
elseif ($cmdCvgD -lt 0.8) {
51+
$blCvgRow = $blData | Where-Object Module -eq $module
52+
$blCvg = $blCvgRow.CommandCoverage
53+
$blCvgD = [decimal]$blCvg.TrimEnd("%") / 100
54+
Write-Host "Last release test coverage for module $module is $blCvg."
55+
if ($cmdCvgD -lt $blCvgD) {
56+
Write-Warning "Test coverage for module $module is less than 80% and lower than the last release !"
57+
$cvgMessageBody80 = "|⚠️|Test Coverage Less Than 80%|$cmdCvg|$blCvg|Test coverage cannot be lower than the number of the last release.|`n"
58+
Set-ModuleTestStatusInPipelineResult -ModuleName "Az.$module" -Status Warning -Content ($cvgMessageHeader80 + $cvgMessageBody80)
59+
}
60+
}
61+
62+
Write-Host
63+
}

0 commit comments

Comments
 (0)