Add support for running benchmarks in Linux environment. (#10782)

kshyju · web-flow · commit e57d37f1190c · 2025-02-05T17:06:23.000-08:00
* Add support for running benchmarks in Linux environment.

* Remove verbose output for publish benchmark app step

* Using template if else instead of condition on task. Using "Invoke-WebRequest" for both windows and Linux.

* Fixed displayName to be consistent.

* printing the error message conditionally.

* remove `condition` for install libssl1.1 step
diff --git a/eng/ci/host.benchmarks.yml b/eng/ci/host.benchmarks.yml
@@ -44,30 +44,46 @@ extends:
     - stage: Setup
       displayName: Setup & start agents
       jobs:
+      # WINDOWS
+      - template: /eng/ci/templates/official/jobs/setup-benchmark-agents.yml@self
+        parameters:
+          functionAppName: HelloHttpNet9
+      - template: /eng/ci/templates/official/jobs/setup-benchmark-agents.yml@self
+        parameters:
+          functionAppName: HelloHttpNet9NoProxy
+
+      # LINUX
       - template: /eng/ci/templates/official/jobs/setup-benchmark-agents.yml@self
         parameters:
           functionAppName: HelloHttpNet9 
-          agentName: HelloHttpNet9_APP
+          os: Linux
       - template: /eng/ci/templates/official/jobs/setup-benchmark-agents.yml@self
         parameters:
-          agentName: HelloHttpNet9NoProxy_APP
-          functionAppName: HelloHttpNet9NoProxy 
+          functionAppName: HelloHttpNet9NoProxy
+          os: Linux
 
     - stage: Run
       displayName: Run Benchmarks
       dependsOn: [] # Force this stage to run independently and in parallel with other stages.
       jobs:
+      # WINDOWS
       - template: /eng/ci/templates/official/jobs/run-benchmarks.yml@self
         parameters:
           description: .NET9 Web Application
           functionAppName: HelloHttpNet9  # App with ASP.NET Integration
-          additionalCrankArgs: $(AdditionalCrankArguments)  # Pipeline variable to pass additional arguments to crank command.
-          storeResultsInDatabase: ${{ variables.storeBenchmarkResultsInDatabase }}
-          agentName: HelloHttpNet9_APP
       - template: /eng/ci/templates/official/jobs/run-benchmarks.yml@self
         parameters:
           description: .NET9 Worker Application
           functionAppName: HelloHttpNet9NoProxy # App without ASP.NET Integration
-          additionalCrankArgs: $(AdditionalCrankArguments)
-          storeResultsInDatabase: ${{ variables.storeBenchmarkResultsInDatabase }}
-          agentName: HelloHttpNet9NoProxy_APP
+
+      # LINUX
+      - template: /eng/ci/templates/official/jobs/run-benchmarks.yml@self
+        parameters:
+          description: .NET9 Web Application
+          functionAppName: HelloHttpNet9  
+          os: Linux
+      - template: /eng/ci/templates/official/jobs/run-benchmarks.yml@self
+        parameters:
+          description: .NET9 Worker Application
+          functionAppName: HelloHttpNet9NoProxy 
+          os: Linux
diff --git a/eng/ci/templates/official/jobs/run-benchmarks.yml b/eng/ci/templates/official/jobs/run-benchmarks.yml
@@ -3,31 +3,40 @@ parameters:
   type: string
 - name: description
   type: string
-- name: storeResultsInDatabase
-  type: boolean
-  default: false
-- name: additionalCrankArgs
-  type: string
-  default: ''
-- name: agentName
+- name: os
   type: string
+  default: Windows
+  values:
+    - Windows
+    - Linux
 
 jobs:
-- job: ${{ parameters.functionAppName }}
-
+- job: ${{ parameters.functionAppName }}_${{ parameters.os }}
+
+  ${{ if eq(parameters.os, 'Linux') }}:
+    pool:
+      name: 1es-pool-azfunc-benchmarking-large
+      image: 1es-ubuntu-22.04-benchmark-runner-vanilla
+      os: linux 
+  ${{ else }}:
+    pool:
+      name: 1es-pool-azfunc-benchmarking-large
+      image: 1es-windows-2022-benchmark-runner-vanilla
+      os: windows
+  
   variables:
-    agentName: ${{ parameters.agentName }}
+    agentId: ${{ parameters.functionAppName }}${{ parameters.os }}
     runDescription: ${{ parameters.description }}
     functionApp: ${{ parameters.functionAppName }}
-    benchmarkArtifactName: benchmark_results_$(functionApp)
+    benchmarkArtifactName: benchmark_results_$(Agent.OS)_$(functionApp)
     functionAppOutputPath: $(Build.ArtifactStagingDirectory)/FunctionApps/$(functionApp)
     benchmarkResultsJsonPath: $(Build.ArtifactStagingDirectory)/BenchmarkResults/$(Build.BuildNumber)_$(functionApp).json
     functionsWorkerRuntime: 'dotnet-isolated'
     configFilePath: "./eng/perf/http.benchmarks.yml"
     hostLocation: "./../../"
     baselineBenchmarkResultFilePath: ''
     baselineBenchmarkResultsDownloadDir: $(Pipeline.Workspace)/BenchmarkBaselineResult
-    appAgentIpAddress: ''
+    appAgentHostName: ''
 
   templateContext:
     inputs:
@@ -46,7 +55,7 @@ jobs:
   steps:
 
   - task: AzureKeyVault@2
-    condition: and(succeeded(), eq('${{ parameters.storeResultsInDatabase }}', true))
+    condition: and(succeeded(), eq(variables['storeBenchmarkResultsInDatabase'], true))
     inputs:
       azureSubscription: Azure-Functions-Host-CI-internal
       KeyVaultName: functions-perf-crank-kv
@@ -61,27 +70,13 @@ jobs:
   - script: dotnet tool install -g Microsoft.Crank.Controller --version "0.2.0-*"
     displayName: Install Microsoft.Crank.Controller tool
 
-  - pwsh: Start-Process powershell -ArgumentList '-NoExit', '-Command', 'crank-agent'
-    displayName: Start crank-agent
-
-  - task: CopyFiles@2
-    displayName: Copy benchmark apps to temp location
-    inputs:
-      SourceFolder: '$(Build.SourcesDirectory)/test/Performance/Apps'
-      Contents: '**/*'
-      TargetFolder: '$(Build.ArtifactStagingDirectory)/PerformanceTestApps'
-      CleanTargetFolder: true
-
-  - task: DotNetCoreCLI@2
-    displayName: Publish $(functionApp) app
-    inputs:
-      command: publish
-      publishWebProjects: false
-      zipAfterPublish: false
-      modifyOutputPath: false
-      projects: '$(Build.ArtifactStagingDirectory)/PerformanceTestApps/$(functionApp)/HelloHttp.csproj'
-      arguments: -c Release -o $(functionAppOutputPath) -f net9.0
-      workingDirectory: $(Build.ArtifactStagingDirectory)/PerformanceTestApps/$(functionApp)
+  - ${{ if eq(parameters.os, 'Windows') }}:
+    - pwsh: Start-Process powershell -ArgumentList '-NoExit', '-Command', 'crank-agent'
+      displayName: Start crank-agent
+  - ${{ else }}:
+    - script: |
+        nohup crank-agent &
+      displayName: Start crank-agent
 
   - task: AzureCLI@2
     displayName: Get Remote Agent IP Address
@@ -99,12 +94,12 @@ jobs:
             --auth-mode login `
             --table-name $(BenchmarkAgentInfoTableName) `
             --partition-key $(Build.BuildNumber) `
-            --row-key $(agentName) `
-            --select AgentIPAddress
+            --row-key $(agentId) `
+            --select HostName
 
           if ($Entity) {
-            $AgentIPAddress = ($Entity | ConvertFrom-Json).AgentIPAddress
-            Write-Host "##vso[task.setvariable variable=appAgentIpAddress]$AgentIPAddress"
+            $HostName = ($Entity | ConvertFrom-Json).HostName
+            Write-Host "##vso[task.setvariable variable=appAgentHostName]$HostName"
             break
           } else {
             Write-Host "Entity not found. Retrying in 30 seconds..."
@@ -120,25 +115,24 @@ jobs:
 
   - pwsh: |
         $crankArgs = "--config $(configFilePath) --scenario hellohttp --profile win2022 --load.options.reuseBuild true --description `"$(runDescription)`" --command-line-property --no-measurements --json $(benchmarkResultsJsonPath) --property sourceVersion=$(Build.SourceVersion) --property buildNumber=$(Build.BuildNumber) --property buildId=$(Build.BuildId) --property sourceBranch=$(Build.SourceBranch) --variable FunctionsWorkerRuntime=$(functionsWorkerRuntime) --variable HostLocation=$(hostLocation) --variable FunctionAppPath=$(functionAppOutputPath)"
-        $crankArgs += " --variable CrankAgentAppVm=$(appAgentIpAddress) --variable CrankAgentLoadVm=localhost --variable AspNetUrls=http://$(appAgentIpAddress):5000"
-        $crankArgs += " ${{ parameters.additionalCrankArgs }}"
+        $crankArgs += " --variable CrankAgentAppVm=$(appAgentHostName) --variable CrankAgentLoadVm=localhost --variable AspNetUrls=http://$(appAgentHostName):5000"
+        $crankArgs += " $(AdditionalCrankArguments)"
         $command = "crank $crankArgs"
 
-        if ('${{ parameters.storeResultsInDatabase }}' -eq 'true') {
+        if ('${{ variables['storeBenchmarkResultsInDatabase'] }}' -eq 'true') {
             $command += " --table HttpBenchmarks --sql `"$(BenchmarkResultsSqlConnectionString)`""
         }
 
         Write-Host "Running command: $command"
         Invoke-Expression $command
-    displayName: 'Run Benchmark'
+    displayName: Run Benchmark
 
-  # Retrieve function host logs
   - pwsh: |
-      $url = "http://$(appAgentIpAddress):5010/jobs/1/output"
+      $url = "http://$(appAgentHostName):5010/jobs/1/output"
       Write-Host "Fetching logs from: $url"
       $response = Invoke-WebRequest -Uri $url -Method GET -UseBasicParsing
       Write-Host $response.Content
-    displayName: 'Fetch Function Host Logs'
+    displayName: Fetch Function Host Logs
 
   # Tag the build as a baseline if it originates from the specified branch.
   # Baseline builds serve as reference points for performance comparisons in future builds.
@@ -152,13 +146,18 @@ jobs:
   - pwsh: |
       $baselineDir = "$(baselineBenchmarkResultsDownloadDir)"
       $fileNamePattern = "*_$(functionApp).json"
-      $baselineFile = Get-ChildItem -Path $baselineDir -Filter $fileNamePattern | Select-Object -First 1
 
-      if ($baselineFile) {
-        Write-Host "Found baseline benchmark result file: $($baselineFile.FullName)"
-        Write-Host "##vso[task.setvariable variable=baselineBenchmarkResultFilePath]$($baselineFile.FullName)"
+      if (Test-Path $baselineDir -PathType Container) {
+          $baselineFile = Get-ChildItem -Path $baselineDir -Filter $fileNamePattern | Select-Object -First 1
+
+          if ($baselineFile) {
+              Write-Host "Found baseline benchmark result file: $($baselineFile.FullName)"
+              Write-Host "##vso[task.setvariable variable=baselineBenchmarkResultFilePath]$($baselineFile.FullName)"
+          } else {
+              Write-Host "No baseline benchmark result file matching the pattern '$fileNamePattern' found in directory '$baselineDir'."
+          }
       } else {
-        Write-Host "No baseline benchmark result file found."
+          Write-Host "The specified directory '$baselineDir' does not exist."
       }
     displayName: 'Set Baseline Benchmark Result File Path'
 
diff --git a/eng/ci/templates/official/jobs/setup-benchmark-agents.yml b/eng/ci/templates/official/jobs/setup-benchmark-agents.yml
@@ -1,29 +1,53 @@
 parameters:
 - name: functionAppName
   type: string
-- name: agentName
+- name: os
   type: string
+  default: Windows
+  values:
+    - Windows
+    - Linux
+ 
 jobs:
-- job: ${{ parameters.functionAppName }}
+- job: ${{ parameters.functionAppName }}_${{ parameters.os }}
+
+  ${{ if eq(parameters.os, 'Linux') }}:
+    pool:
+      name: 1es-pool-azfunc-benchmarking-large
+      image: 1es-ubuntu-22.04-benchmark-runner-vanilla
+      os: linux 
+  ${{ else }}:
+    pool:
+      name: 1es-pool-azfunc-benchmarking-large
+      image: 1es-windows-2022-benchmark-runner-vanilla
+      os: windows
 
   variables:
-    agentName: ${{ parameters.agentName }}
+    agentId: ${{ parameters.functionAppName }}${{ parameters.os }}
     functionApp: ${{ parameters.functionAppName }}
     functionAppOutputPath: $(Build.ArtifactStagingDirectory)/FunctionApps/$(functionApp)
 
   steps:
 
   - template: /eng/ci/templates/install-dotnet.yml@self
 
+  - ${{ if eq(parameters.os, 'Linux') }}:
+    # Adds the Ubuntu 20.04 security repository to Ubuntu 22.04 and installs libssl1.1, which is unavailable by default.
+    - script: |
+        echo "deb http://security.ubuntu.com/ubuntu focal-security main" | sudo tee /etc/apt/sources.list.d/focal-security.list
+        sudo apt update && sudo apt install -y libssl1.1
+      displayName: "Install libssl1.1 on Ubuntu 22.04"
+
   - script: dotnet tool install -g Microsoft.Crank.Agent --version "0.2.0-*"
     displayName: Install Microsoft.Crank.Agent tool
 
-  - pwsh: Start-Process powershell -ArgumentList '-NoExit', '-Command', 'crank-agent'
-    displayName: Start crank agent
-
-  - pwsh: |
-      dotnet --info
-    displayName: Print .NET info
+  - ${{ if eq(parameters.os, 'Windows') }}:
+    - pwsh: Start-Process powershell -ArgumentList '-NoExit', '-Command', 'crank-agent'
+      displayName: Start crank-agent
+  - ${{ else }}:
+    - script: |
+        nohup crank-agent &
+      displayName: Start crank-agent
 
   - task: CopyFiles@2
     displayName: Copy benchmark apps to temp location
@@ -41,22 +65,20 @@ jobs:
       zipAfterPublish: false
       modifyOutputPath: false
       projects: '$(Build.ArtifactStagingDirectory)/PerformanceTestApps/$(functionApp)/HelloHttp.csproj'
-      arguments: -c Release -o $(functionAppOutputPath) -f net9.0 -v n
+      arguments: -c Release -o $(functionAppOutputPath) -f net9.0
       workingDirectory: $(Build.ArtifactStagingDirectory)/PerformanceTestApps/$(functionApp)
 
-  - script: |
-      netsh advfirewall firewall add rule name="Open Port 5000" dir=in action=allow protocol=TCP localport=5000
-      netsh advfirewall firewall add rule name="Open Port 5010" dir=in action=allow protocol=TCP localport=5010
-    displayName: Open port 5000 and 5010
+  - ${{ if eq(parameters.os, 'Windows') }}:
+    - script: |
+        netsh advfirewall firewall add rule name="Open Port 5000" dir=in action=allow protocol=TCP localport=5000
+        netsh advfirewall firewall add rule name="Open Port 5010" dir=in action=allow protocol=TCP localport=5010
+      displayName: Open port 5000 and 5010
 
-  - task: PowerShell@2
-    displayName: Get Agent IP Address
-    inputs:
-      targetType: 'inline'
-      script: |
-        $ipAddress = (Test-Connection -ComputerName (hostname) -Count 1).IPv4Address.IPAddressToString
-        Write-Host "IP Address: $ipAddress"
-        Write-Host "##vso[task.setvariable variable=agentIp]$ipAddress"
+  - pwsh: |
+      $HostName = [System.Net.Dns]::GetHostName()
+      Write-Host "##vso[task.setvariable variable=machineHostName]$HostName"
+      Write-Host "HostName: $HostName"
+    displayName: Get Machine Info
 
   - task: AzureCLI@2
     displayName: Persist Agent IP Address
@@ -69,7 +91,7 @@ jobs:
           --auth-mode login `
           --account-name $(StorageAccount) `
           --table-name $(BenchmarkAgentInfoTableName) `
-          --entity PartitionKey=$(Build.BuildNumber) RowKey=$(agentName) AgentName=$(agentName) AgentIPAddress=$(agentIp)
+          --entity PartitionKey=$(Build.BuildNumber) RowKey=$(agentId) AgentId=$(agentId) HostName=$(machineHostName)
 
   - pwsh: |
       $url = "http://localhost:5010/jobs/all"
@@ -80,9 +102,9 @@ jobs:
       while ($attempt -lt $maxAttempts) {
           $response = Invoke-WebRequest -Uri $url -Method Get -ErrorAction Stop
           $data = $response.Content | ConvertFrom-Json
-          $completedJobCount = ($data | Where-Object { $_.state -eq "Deleted" }).Count 
+          $completedJobCount = ($data | Where-Object { $_.state -eq "Deleted" }).Count
 
-          if ($completedJobCount -gt 0) { 
+          if ($completedJobCount -gt 0) {
               Write-Host "Found at least 1 job with 'state' = 'Deleted'. Exiting task."
               exit 0
           }
@@ -108,4 +130,4 @@ jobs:
           --auth-mode login `
           --account-name $(StorageAccount) `
           --table-name $(BenchmarkAgentInfoTableName) `
-          --partition-key $(Build.BuildNumber) --row-key=$(agentName)
+          --partition-key $(Build.BuildNumber) --row-key=$(agentId)
