Add benchmark result comparison support (#10753)

* Tag a build as baseline. Download a baseline and do a comparison if one is present.

* fix whitespace

* Change branch to release/4.x

* Optimizations based on PR feedback.

Author: kshyju

eng/ci/host.benchmarks.yml

@@ -22,6 +22,11 @@ resources:
     type: git
     name: engineering
     ref: refs/tags/release
+  pipelines:
+  - pipeline: BaselineResult
+    source: host.benchmarks
+    tags:
+    - AzFunc.Perf.Baseline
 
 variables:
   - template: /eng/ci/templates/variables/benchmarks.yml@self

eng/ci/templates/official/jobs/run-benchmarks.yml

@@ -16,20 +16,29 @@ jobs:
   variables:
     runDescription: ${{ parameters.description }}
     functionApp: ${{ parameters.functionAppName }}
+    benchmarkArtifactName: benchmark_results_$(functionApp)
     functionAppOutputPath: $(Build.ArtifactStagingDirectory)/FunctionApps/$(functionApp)
-    benchmarkResultsJsonPath: "$(Build.ArtifactStagingDirectory)/BenchmarkResults/$(buildNumber)_$(functionApp).json"
+    benchmarkResultsJsonPath: $(Build.ArtifactStagingDirectory)/BenchmarkResults/$(Build.BuildNumber)_$(functionApp).json
     functionsWorkerRuntime: 'dotnet-isolated'
-    crankAgentUrl: "http://localhost:5010"  # Default crank agent URL.
+    crankAgentUrl: "http://localhost:5010" # Default crank agent URL.
     configFilePath: "./eng/perf/http.benchmarks.yml"
     hostLocation: "./../../"
+    baselineBenchmarkResultFilePath: ''
+    baselineBenchmarkResultsDownloadDir: $(Pipeline.Workspace)/BenchmarkBaselineResult
 
   templateContext:
+    inputs:
+    - input: pipelineArtifact
+      artifactName: $(benchmarkArtifactName)
+      pipeline: BaselineResult
+      targetPath: $(baselineBenchmarkResultsDownloadDir)
+
     outputParentDirectory: $(Build.ArtifactStagingDirectory)
     outputs:
     - output: pipelineArtifact
       displayName: Publish benchmark results
       path: $(benchmarkResultsJsonPath)
-      artifact: 'BenchmarkResults_$(functionApp)'
+      artifact: $(benchmarkArtifactName)
 
   steps:
 
@@ -46,12 +55,8 @@ jobs:
   - script: dotnet tool install -g Microsoft.Crank.Agent --version "0.2.0-*"
     displayName: Install Microsoft.Crank.Agent tool
 
-  - task: PowerShell@2
-    displayName: Start crank-agent
-    inputs:
-      targetType: 'inline'
-      script: |
-        Start-Process powershell -ArgumentList '-NoExit', '-Command', 'crank-agent'
+  - pwsh: Start-Process powershell -ArgumentList '-NoExit', '-Command', 'crank-agent'
+    displayName: 'Start crank-agent'
 
   - task: CopyFiles@2
     displayName: Copy benchmark apps to temp location
@@ -75,12 +80,8 @@ jobs:
   - script: dotnet tool install -g Microsoft.Crank.Controller --version "0.2.0-*"
     displayName: Install Microsoft.Crank.Controller
 
-  - task: PowerShell@2
-    displayName: Run crank-controller
-    inputs:
-      targetType: 'inline'
-      script: |
-        $crankArgs = "--config $(configFilePath) --scenario hellohttp --profile win2022 --load.options.reuseBuild true --description `"$(runDescription)`" --command-line-property --no-metadata --no-measurements --json $(benchmarkResultsJsonPath) --property sourceVersion=$(sourceVersion) --property buildNumber=$(buildNumber) --property buildId=$(buildId) --variable FunctionsWorkerRuntime=$(functionsWorkerRuntime) --variable HostLocation=$(hostLocation) --variable FunctionAppPath=$(functionAppOutputPath)"
+  - pwsh: |
+        $crankArgs = "--config $(configFilePath) --scenario hellohttp --profile win2022 --load.options.reuseBuild true --description `"$(runDescription)`" --command-line-property --no-measurements --json $(benchmarkResultsJsonPath) --property sourceVersion=$(Build.SourceVersion) --property buildNumber=$(Build.BuildNumber) --property buildId=$(Build.BuildId) --property sourceBranch=$(Build.SourceBranch) --variable FunctionsWorkerRuntime=$(functionsWorkerRuntime) --variable HostLocation=$(hostLocation) --variable FunctionAppPath=$(functionAppOutputPath)"
         $crankArgs += " ${{ parameters.additionalCrankArgs }}"
         $command = "crank $crankArgs"
 
@@ -90,13 +91,40 @@ jobs:
 
         Write-Host "Running command: $command"
         Invoke-Expression $command
+    displayName: 'Run Benchmark'
 
-  - task: PowerShell@2
-    displayName: Functions host logs
-    inputs:
-      targetType: 'inline'
-      script: |
-        $url = "$(crankAgentUrl)/jobs/1/output"
-        Write-Host "Making GET request to: $url to get logs"
-        $response = Invoke-WebRequest -Uri $url -Method GET -UseBasicParsing
-        Write-Host $response.Content
+  # Retrieve function host logs
+  - pwsh: |
+      $url = "$(crankAgentUrl)/jobs/1/output"
+      Write-Host "Fetching logs from: $url"
+      $response = Invoke-WebRequest -Uri $url -Method GET -UseBasicParsing
+      Write-Host $response.Content
+    displayName: 'Fetch Function Host Logs'
+
+  # Tag the build as a baseline if it originates from the specified branch.
+  # Baseline builds serve as reference points for performance comparisons in future builds.
+  # The tag added here will help identify these builds in the pipeline.
+  - pwsh: |
+      Write-Host "##vso[build.addbuildtag]$(benchmarkBaselineTagName)"
+    condition: and(succeeded(), eq(variables['Build.SourceBranch'], variables['benchmarkBaselineBranch']))
+    displayName: 'Tag Build as Baseline'
+
+  # Locate baseline benchmark result file from the downloaded baseline artifact.
+  - pwsh: |
+      $baselineDir = "$(baselineBenchmarkResultsDownloadDir)"
+      $fileNamePattern = "*_$(functionApp).json"
+      $baselineFile = Get-ChildItem -Path $baselineDir -Filter $fileNamePattern | Select-Object -First 1
+
+      if ($baselineFile) {
+        Write-Host "Found baseline benchmark result file: $($baselineFile.FullName)"
+        Write-Host "##vso[task.setvariable variable=baselineBenchmarkResultFilePath]$($baselineFile.FullName)"
+      } else {
+        Write-Host "No baseline benchmark result file found."
+      }
+    displayName: 'Set Baseline Benchmark Result File Path'
+
+  # Compare results with baseline
+  - pwsh: |
+      crank compare "$(baselineBenchmarkResultFilePath)" "$(benchmarkResultsJsonPath)"
+    condition: and(succeeded(), ne(variables['baselineBenchmarkResultFilePath'], ''))
+    displayName: 'Compare Results with Baseline'

eng/ci/templates/variables/benchmarks.yml

@@ -1,9 +1,7 @@
 variables:
-- name: buildId
-  value: $(Build.BuildId)
-- name: buildNumber
-  value: $(Build.BuildNumber)
-- name: sourceVersion
-  value: $(Build.SourceVersion)
 - name: storeBenchmarkResultsInDatabase
-  value: ${{ eq(variables['Build.Reason'], 'Schedule') }}
+  value: ${{ eq(variables['Build.Reason'], 'Schedule') }}
+- name: benchmarkBaselineBranch
+  value: refs/heads/release/4.x
+- name: benchmarkBaselineTagName
+  value: AzFunc.Perf.Baseline