Implemented ci for calculating test coverage baseline (#21331)

vidai-msft · web-flow · commit 2b936ff067bb · 2023-03-27T14:08:40.000+08:00
diff --git a/.azure-pipelines/test-coverage.yml b/.azure-pipelines/test-coverage.yml
@@ -0,0 +1,140 @@
+parameters:
+- name: windowsImage
+  displayName: Windows Image
+  type: string
+  default: windows-2019
+- name: dotnet_sdk_3_1
+  displayName: .NET Core 3.1
+  default: 3.1.x
+- name: dotnet_sdk_6
+  displayName: .NET 6
+  type: string
+  default: 6.0.x
+- name: dotnet_sdk_7
+  displayName: .NET 7
+  type: string
+  default: 7.0.x
+- name: testFramework
+  displayName: Test Framework
+  type: string
+  default: netcoreapp3.1
+
+variables:
+  EnableTestCoverage: true
+  TestCoverageLocation: $(Pipeline.Workspace)/artifacts
+
+pr: none
+trigger: none
+
+jobs:
+- job: TestCoverage
+  timeoutInMinutes: 600
+  pool:
+    vmImage: ${{ parameters.windowsImage }}
+
+  steps:
+  - task: UseDotNet@2
+    displayName: 'Install .NET Core 3.1 SDK'
+    inputs:
+      packageType: sdk
+      version: ${{ parameters.dotnet_sdk_3_1 }}
+
+  - task: PowerShell@2
+    displayName: Initialize Environment
+    inputs:
+      pwsh: true
+      targetType: filePath
+      filePath: ./tools/TestFx/Utilities/CommonUtility.ps1
+
+  - task: DotNetCoreCLI@2
+    displayName: Build Azure PowerShell Solution
+    inputs:
+      command: custom
+      custom: msbuild
+      arguments: 'build.proj -t:"Clean;Build" -p:"Configuration=Debug"'
+
+  - task: DotNetCoreCLI@2
+    displayName: Test SDK-Based Cmdlets
+    condition: succeeded()
+    inputs:
+      command: custom
+      custom: msbuild
+      arguments: 'build.proj -t:"Test" -p:"Configuration=Debug;TestFramework=${{ parameters.testFramework }}"'
+
+  - task: PowerShell@2
+    displayName: Test AutoRest-Based Cmdlets
+    condition: succeeded()
+    inputs:
+      pwsh: true
+      workingDirectory: 'artifacts/Debug'
+      targetType: inline
+      script: |
+        Install-Module -Name Pester -RequiredVersion 4.10.1 -Force
+        Import-Module (Join-Path -Path . -ChildPath Az.Accounts | Join-Path -ChildPath Az.Accounts.psd1) -Force
+
+        $PreErrorActionPreference = $ErrorActionPreference
+        $ErrorActionPreference = "SilentlyContinue"
+
+        Get-ChildItem -Path . -Filter test-module.ps1 -File -Recurse | ForEach-Object {
+          try {
+            . $_ -Isolated
+          }
+          catch {
+            Write-Host $_.Exception
+          }
+        }
+
+        $ErrorActionPreference = $PreErrorActionPreference
+
+  - task: PowerShell@2
+    displayName: Analyze Test Coverage
+    condition: succeeded()
+    inputs:
+      pwsh: true
+      targetType: filePath
+      filePath: ./tools/TestFx/Coverage/AnalyzeTestCoverage.ps1
+      arguments: -CalcBaseline
+
+  - task: PowerShell@2
+    displayName: Save Test Coverage Results
+    condition: succeeded()
+    inputs:
+      pwsh: true
+      targetType: filePath
+      filePath: ./tools/TestFx/Coverage/SaveTestCoverageResult.ps1
+      arguments: CITest $(Build.BuildId) $(KustoServicePrincipalTenantId) $(KustoServicePrincipalId) $(KustoServicePrincipalSecret) $(KustoClusterName) $(KustoClusterRegion) $(TestCoverageDatabaseName) $(TestCoverageTableName) $(TestCoverageLocation)
+
+  - task: PowerShell@2
+    displayName: Update Test Coverage Baseline
+    condition: succeeded()
+    inputs:
+      pwsh: true
+      targetType: inline
+      script: |
+        $blDir = "./tools/TestFx/Coverage"
+        $blCsv = Join-Path -Path $blDir -ChildPath "Baseline.csv"
+        if (Test-Path -Path $blCsv) {
+          Remove-Item -Path $blCsv -Force
+        }
+
+        git checkout -b testcoverage-baseline
+        Copy-Item -Path "$(Pipeline.Workspace)/artifacts/TestCoverageAnalysis/Results/Baseline.csv" -Destination $blDir
+        git config user.email "65331932+azure-powershell-bot@users.noreply.github.com"
+        git config user.name "azure-powershell-bot"
+        git add $blCsv
+        git commit -m "Update with the latest test coverage data"
+        git remote set-url origin https://azure-powershell-bot:$(BotAccessToken)@github.com/Azure/azure-powershell.git
+        git push origin testcoverage-baseline --force
+
+        $title = "Update test coverage baseline"
+        $headBranch = "testcoverage-baseline"
+        $targetBranch = "main"
+        $description = "Update with the latest test coverage baseline data"
+        ./tools/Github/CreatePR.ps1 -Title $title -HeadBranch $headBranch -BaseBranch $targetBranch -BotAccessToken $(BotAccessToken) -Description $description
+
+  - task: PublishPipelineArtifact@1
+    displayName: Publish Test Coverage Result
+    condition: always()
+    inputs:
+      artifact: testcoverage
+      targetPath: $(TestCoverageLocation)
diff --git a/src/Accounts/Authentication/TestCoverage.cs b/src/Accounts/Authentication/TestCoverage.cs
@@ -105,7 +105,7 @@ public void LogRawData(AzurePSQoSEvent qos)
         {
 #if DEBUG || TESTCOVERAGE
             string moduleName = qos.ModuleName;
-            string commandName = qos.CommandName;
+            string commandName = ProcessCommandName(qos.CommandName);
             string sourceScriptPath = qos.SourceScript;
             string sourceScriptName = Path.GetFileName(sourceScriptPath);
 
@@ -150,5 +150,14 @@ public void LogRawData(AzurePSQoSEvent qos)
             }
 #endif
         }
+
+        private string ProcessCommandName(string commandName)
+        {
+            if (commandName.Contains("_"))
+            {
+                commandName = commandName.Substring(0, commandName.IndexOf("_"));
+            }
+            return commandName;
+        }
     }
 }
diff --git a/src/MySql/test/AzMySqlFlexibleServer.Tests.ps1 b/src/MySql/test/AzMySqlFlexibleServer.Tests.ps1
@@ -15,41 +15,42 @@ Describe 'AzMySqlFlexibleServer' {
     It 'List1' {
         {
             $servers = Get-AzMySqlFlexibleServer
-            $servers.Count | Should -BeGreaterOrEqual 1  
+            $servers.Count | Should -BeGreaterOrEqual 1
             $servers = Get-AzMySqlFlexibleServer -ResourceGroupName $env.resourceGroup
-            $servers.Count | Should -Be 1  
+            $servers.Count | Should -Be 1
         } | Should -Not -Throw
     }
 
     It 'ViaName'  {
         {
             $servers = Get-AzMySqlFlexibleServer -ResourceGroupName $env.resourceGroup -Name $env.flexibleServerName
             $servers.Name | Should -Be $env.flexibleServerName
-            
-            $server = Update-AzMySqlFlexibleServer -ResourceGroupName $env.resourceGroup -Name $env.flexibleServerName -BackupRetentionDay 12 
+
+            $server = Update-AzMySqlFlexibleServer -ResourceGroupName $env.resourceGroup -Name $env.flexibleServerName -BackupRetentionDay 12
             $server.BackupRetentionDay | Should -Be 12
 
             $server = Update-AzMySqlFlexibleServer -ResourceGroupName $env.resourceGroup -Name $env.flexibleServerName -MaintenanceWindow "Mon:1:30"
             $server.MaintenanceWindowCustomWindow | Should -Be 'Enabled'
             $server.MaintenanceWindowDayOfWeek | Should -Be '1'
             $server.MaintenanceWindowStartHour | Should -Be '1'
             $server.MaintenanceWindowStartMinute | Should -Be '30'
-            
+
             Stop-AzMySqlFlexibleServer -ResourceGroupName $env.resourceGroup -Name $env.flexibleServerName
-            
+
             Start-AzMySqlFlexibleServer -ResourceGroupName $env.resourceGroup -Name $env.flexibleServerName
-            
+
             Restart-AzMySqlFlexibleServer -ResourceGroupName $env.resourceGroup -Name $env.flexibleServerName
-        
-            $replica = Get-AzMySqlFlexibleServer -ResourceGroupName $env.resourceGroup -ServerName $env.flexibleServerName | New-AzMySqlFlexibleServerReplica -Replica $env.replicaName -ResourceGroupName $env.resourceGroup 
+
+            $flxServer = Get-AzMySqlFlexibleServer -ResourceGroupName $env.resourceGroup -ServerName $env.flexibleServerName
+            $replica = New-AzMySqlFlexibleServerReplica -InputObject $flxServer -Replica $env.replicaName -ResourceGroupName $env.resourceGroup
             $replica.Name | Should -Be $env.replicaName
             $replica.SkuName | Should -Be $env.flexibleSku
 
             $replica = Get-AzMySqlFlexibleServerReplica -ResourceGroupName $env.resourceGroup -ServerName $env.flexibleServerName
             $replica.Count | Should -Be 1
 
             Remove-AzMySqlFlexibleServer -ResourceGroupName $env.resourceGroup -Name $env.replicaName
-        
+
         } | Should -Not -Throw
     }
 
@@ -68,18 +69,18 @@ Describe 'AzMySqlFlexibleServer' {
 
             $ID = "/subscriptions/$($env.SubscriptionId)/resourceGroups/$($env.resourceGroup)/providers/Microsoft.DBforMySQL/flexibleServers/$($env.flexibleServerName)/stop"
             Stop-AzMySqlFlexibleServer -InputObject $ID
-            
+
             $ID = "/subscriptions/$($env.SubscriptionId)/resourceGroups/$($env.resourceGroup)/providers/Microsoft.DBforMySQL/flexibleServers/$($env.flexibleServerName)/start"
             Start-AzMySqlFlexibleServer -InputObject $ID
-            
+
             $ID = "/subscriptions/$($env.SubscriptionId)/resourceGroups/$($env.resourceGroup)/providers/Microsoft.DBforMySQL/flexibleServers/$($env.flexibleServerName)/restart"
             Restart-AzMySqlFlexibleServer -InputObject $ID
-            
+
             $restorePointInTime = (Get-Date).AddMinutes(-10)
             $restoredServer = Restore-AzMySqlFlexibleServer -Name $env.restoreName -ResourceGroupName $env.resourceGroup -RestorePointInTime $restorePointInTime -InputObject $server
-            
+
             Remove-AzMySqlFlexibleServer -ResourceGroupName $env.resourceGroup -Name $restoredServer.Name
 
         } | Should -Not -Throw
     }
-}
+}
diff --git a/src/MySql/test/Get-AzMySqlReplica.Tests.ps1 b/src/MySql/test/Get-AzMySqlReplica.Tests.ps1
@@ -13,7 +13,8 @@ while(-not $mockingPath) {
 
 Describe 'Get-AzMySqlReplica' {
     It 'List' {
-        Get-AzMySqlServer -ResourceGroupName $env.resourceGroup -ServerName $env.serverName | New-AzMySqlReplica -Replica $env.replicaName -ResourceGroupName $env.resourceGroup
+        $server = Get-AzMySqlServer -ResourceGroupName $env.resourceGroup -ServerName $env.serverName
+        New-AzMySqlReplica -InputObject $server -Replica $env.replicaName -ResourceGroupName $env.resourceGroup
         $replica = Get-AzMySqlReplica -ResourceGroupName $env.resourceGroup -ServerName $env.serverName
         $replica.Count | Should -Be 1
         Remove-AzMySqlServer -InputObject $replica
diff --git a/src/MySql/test/New-AzMySqlReplica.Tests.ps1 b/src/MySql/test/New-AzMySqlReplica.Tests.ps1
@@ -13,7 +13,8 @@ while(-not $mockingPath) {
 
 Describe 'New-AzMySqlReplica' {
     It 'CreateExpanded' {
-        $replica = Get-AzMySqlServer -ResourceGroupName $env.resourceGroup -ServerName $env.serverName | New-AzMySqlReplica -Replica $env.replicaName -ResourceGroupName $env.resourceGroup 
+        $server = Get-AzMySqlServer -ResourceGroupName $env.resourceGroup -ServerName $env.serverName
+        $replica = New-AzMySqlReplica -InputObject $server -Replica $env.replicaName -ResourceGroupName $env.resourceGroup
         $replica.Name | Should -Be $env.replicaName
         $replica.SkuName | Should -Be $env.Sku
         $replica.Location | Should -Be eastus
diff --git a/src/MySql/test/Restore-AzMySqlServer.Tests.ps1 b/src/MySql/test/Restore-AzMySqlServer.Tests.ps1
@@ -15,15 +15,17 @@ while(-not $mockingPath) {
 # See https://github.com/Azure/autorest.powershell/issues/580
 Describe 'Restore-AzMySqlServer' {
     It 'GeoRestore' {
-        $replica = Get-AzMySqlServer -ResourceGroupName $env.resourceGroup -ServerName $env.serverName | New-AzMySqlReplica -Replica $env.replicaName -ResourceGroupName $env.resourceGroup
-        $restoreServer = Restore-AzMySqlServer -Name $env.serverName -ResourceGroupName $env.resourceGroup -InputObject $replica -UseGeoRestore 
+        $server = Get-AzMySqlServer -ResourceGroupName $env.resourceGroup -ServerName $env.serverName
+        $replica = New-AzMySqlReplica -InputObject $server -Replica $env.replicaName -ResourceGroupName $env.resourceGroup
+        $restoreServer = Restore-AzMySqlServer -Name $env.serverName -ResourceGroupName $env.resourceGroup -InputObject $replica -UseGeoRestore
         $restoreServer.Name | Should -Be $env.serverName
         $restoreServer.SkuName | Should -Be $env.Sku
     }
 
     It 'PointInTimeRestore' {
+        $server = Get-AzMySqlServer -ResourceGroupName $env.resourceGroup -ServerName $env.serverName
         $restorePointInTime = (Get-Date).AddMinutes(-10)
-        $restoreServer = Get-AzMySqlServer -ResourceGroupName $env.resourceGroup -ServerName $env.serverName | Restore-AzMySqlServer -Name $env.restoreName2 -ResourceGroupName $env.resourceGroup -RestorePointInTime $restorePointInTime -UsePointInTimeRestore
+        $restoreServer = Restore-AzMySqlServer -InputObject $server -Name $env.restoreName2 -ResourceGroupName $env.resourceGroup -RestorePointInTime $restorePointInTime -UsePointInTimeRestore
         $restoreServer.Name | Should -Be $env.restoreName2
         $restoreServer.SkuName | Should -Be $env.Sku
     }
diff --git a/tools/TestFx/Coverage/AnalyzeTestCoverage.ps1 b/tools/TestFx/Coverage/AnalyzeTestCoverage.ps1
diff --git a/tools/TestFx/Coverage/SaveTestCoverageResult.ps1 b/tools/TestFx/Coverage/SaveTestCoverageResult.ps1
diff --git a/tools/TestFx/Utilities/CommonUtility.ps1 b/tools/TestFx/Utilities/CommonUtility.ps1

Original file line number	Diff line number	Diff line change
`@@ -105,7 +105,7 @@ public void LogRawData(AzurePSQoSEvent qos)`
`105`	`105`	`{`
`106`	`106`	`#if DEBUG \|\| TESTCOVERAGE`
`107`	`107`	`string moduleName = qos.ModuleName;`
`108`		`- string commandName = qos.CommandName;`
	`108`	`+ string commandName = ProcessCommandName(qos.CommandName);`
`109`	`109`	`string sourceScriptPath = qos.SourceScript;`
`110`	`110`	`string sourceScriptName = Path.GetFileName(sourceScriptPath);`
`111`	`111`
`@@ -150,5 +150,14 @@ public void LogRawData(AzurePSQoSEvent qos)`
`150`	`150`	`}`
`151`	`151`	`#endif`
`152`	`152`	`}`
	`153`	`+`
	`154`	`+ private string ProcessCommandName(string commandName)`
	`155`	`+ {`
	`156`	`+ if (commandName.Contains("_"))`
	`157`	`+ {`
	`158`	`+ commandName = commandName.Substring(0, commandName.IndexOf("_"));`
	`159`	`+ }`
	`160`	`+ return commandName;`
	`161`	`+ }`
`153`	`162`	`}`
`154`	`163`	`}`