Skip to content

Commit cfa63f6

Browse files
kshyjukshyju
authored
Run benchmarking in dedicated agents (application and load) (#10773)
* Running application and load generators in separate agents. Using MMS image. * specify ASPNETCORE_URLS env var * Minor cleanup * Minor cleanup * Fix trigger * use 1es-windows-2022-benchmark-runner-vanilla image * Improvements based on PR feedback. Added retry timeout. * Added back linebreaks.
1 parent 82791d6 commit cfa63f6

File tree

4 files changed

+178
-11
lines changed

4 files changed

+178
-11
lines changed

‎eng/ci/host.benchmarks.yml

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,23 +36,38 @@ extends:
3636
template: v1/1ES.Unofficial.PipelineTemplate.yml@1es
3737
parameters:
3838
pool:
39-
name: 1es-pool-azfunc-large
40-
image: 1es-windows-2022
39+
name: 1es-pool-azfunc-benchmarking-large
40+
image: 1es-windows-2022-benchmark-runner-vanilla
4141
os: windows
4242

4343
stages:
44-
- stage: HttpApps
44+
- stage: Setup
45+
displayName: Setup & start agents
46+
jobs:
47+
- template: /eng/ci/templates/official/jobs/setup-benchmark-agents.yml@self
48+
parameters:
49+
functionAppName: HelloHttpNet9
50+
agentName: HelloHttpNet9_APP
51+
- template: /eng/ci/templates/official/jobs/setup-benchmark-agents.yml@self
52+
parameters:
53+
agentName: HelloHttpNet9NoProxy_APP
54+
functionAppName: HelloHttpNet9NoProxy
55+
56+
- stage: Run
4557
displayName: Run Benchmarks
58+
dependsOn: [] # Force this stage to run independently and in parallel with other stages.
4659
jobs:
4760
- template: /eng/ci/templates/official/jobs/run-benchmarks.yml@self
4861
parameters:
4962
description: .NET9 Web Application
5063
functionAppName: HelloHttpNet9 # App with ASP.NET Integration
5164
additionalCrankArgs: $(AdditionalCrankArguments) # Pipeline variable to pass additional arguments to crank command.
5265
storeResultsInDatabase: ${{ variables.storeBenchmarkResultsInDatabase }}
66+
agentName: HelloHttpNet9_APP
5367
- template: /eng/ci/templates/official/jobs/run-benchmarks.yml@self
5468
parameters:
5569
description: .NET9 Worker Application
5670
functionAppName: HelloHttpNet9NoProxy # App without ASP.NET Integration
5771
additionalCrankArgs: $(AdditionalCrankArguments)
5872
storeResultsInDatabase: ${{ variables.storeBenchmarkResultsInDatabase }}
73+
agentName: HelloHttpNet9NoProxy_APP

‎eng/ci/templates/official/jobs/run-benchmarks.yml

Lines changed: 44 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,22 +9,25 @@ parameters:
99
- name: additionalCrankArgs
1010
type: string
1111
default: ''
12+
- name: agentName
13+
type: string
1214

1315
jobs:
1416
- job: ${{ parameters.functionAppName }}
1517

1618
variables:
19+
agentName: ${{ parameters.agentName }}
1720
runDescription: ${{ parameters.description }}
1821
functionApp: ${{ parameters.functionAppName }}
1922
benchmarkArtifactName: benchmark_results_$(functionApp)
2023
functionAppOutputPath: $(Build.ArtifactStagingDirectory)/FunctionApps/$(functionApp)
2124
benchmarkResultsJsonPath: $(Build.ArtifactStagingDirectory)/BenchmarkResults/$(Build.BuildNumber)_$(functionApp).json
2225
functionsWorkerRuntime: 'dotnet-isolated'
23-
crankAgentUrl: "http://localhost:5010" # Default crank agent URL.
2426
configFilePath: "./eng/perf/http.benchmarks.yml"
2527
hostLocation: "./../../"
2628
baselineBenchmarkResultFilePath: ''
2729
baselineBenchmarkResultsDownloadDir: $(Pipeline.Workspace)/BenchmarkBaselineResult
30+
appAgentIpAddress: ''
2831

2932
templateContext:
3033
inputs:
@@ -55,8 +58,11 @@ jobs:
5558
- script: dotnet tool install -g Microsoft.Crank.Agent --version "0.2.0-*"
5659
displayName: Install Microsoft.Crank.Agent tool
5760

61+
- script: dotnet tool install -g Microsoft.Crank.Controller --version "0.2.0-*"
62+
displayName: Install Microsoft.Crank.Controller tool
63+
5864
- pwsh: Start-Process powershell -ArgumentList '-NoExit', '-Command', 'crank-agent'
59-
displayName: 'Start crank-agent'
65+
displayName: Start crank-agent
6066

6167
- task: CopyFiles@2
6268
displayName: Copy benchmark apps to temp location
@@ -77,11 +83,44 @@ jobs:
7783
arguments: -c Release -o $(functionAppOutputPath) -f net9.0
7884
workingDirectory: $(Build.ArtifactStagingDirectory)/PerformanceTestApps/$(functionApp)
7985

80-
- script: dotnet tool install -g Microsoft.Crank.Controller --version "0.2.0-*"
81-
displayName: Install Microsoft.Crank.Controller
86+
- task: AzureCLI@2
87+
displayName: Get Remote Agent IP Address
88+
inputs:
89+
azureSubscription: $(ServiceConnection)
90+
scriptType: pscore
91+
scriptLocation: inlineScript
92+
inlineScript: |
93+
$maxAttempts = 60
94+
$attempt = 0
95+
96+
do {
97+
$Entity = az storage entity show `
98+
--account-name $(StorageAccount) `
99+
--auth-mode login `
100+
--table-name $(BenchmarkAgentInfoTableName) `
101+
--partition-key $(Build.BuildNumber) `
102+
--row-key $(agentName) `
103+
--select AgentIPAddress
104+
105+
if ($Entity) {
106+
$AgentIPAddress = ($Entity | ConvertFrom-Json).AgentIPAddress
107+
Write-Host "##vso[task.setvariable variable=appAgentIpAddress]$AgentIPAddress"
108+
break
109+
} else {
110+
Write-Host "Entity not found. Retrying in 30 seconds..."
111+
Start-Sleep -Seconds 30
112+
$attempt++
113+
}
114+
115+
if ($attempt -ge $maxAttempts) {
116+
Write-Host "Maximum attempts reached. Exiting..."
117+
exit 1
118+
}
119+
} while (-not $Entity)
82120
83121
- pwsh: |
84122
$crankArgs = "--config $(configFilePath) --scenario hellohttp --profile win2022 --load.options.reuseBuild true --description `"$(runDescription)`" --command-line-property --no-measurements --json $(benchmarkResultsJsonPath) --property sourceVersion=$(Build.SourceVersion) --property buildNumber=$(Build.BuildNumber) --property buildId=$(Build.BuildId) --property sourceBranch=$(Build.SourceBranch) --variable FunctionsWorkerRuntime=$(functionsWorkerRuntime) --variable HostLocation=$(hostLocation) --variable FunctionAppPath=$(functionAppOutputPath)"
123+
$crankArgs += " --variable CrankAgentAppVm=$(appAgentIpAddress) --variable CrankAgentLoadVm=localhost --variable AspNetUrls=http://$(appAgentIpAddress):5000"
85124
$crankArgs += " ${{ parameters.additionalCrankArgs }}"
86125
$command = "crank $crankArgs"
87126
@@ -95,7 +134,7 @@ jobs:
95134
96135
# Retrieve function host logs
97136
- pwsh: |
98-
$url = "$(crankAgentUrl)/jobs/1/output"
137+
$url = "http://$(appAgentIpAddress):5010/jobs/1/output"
99138
Write-Host "Fetching logs from: $url"
100139
$response = Invoke-WebRequest -Uri $url -Method GET -UseBasicParsing
101140
Write-Host $response.Content

‎eng/ci/templates/official/jobs/setup-benchmark-agents.yml

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
parameters:
2+
- name: functionAppName
3+
type: string
4+
- name: agentName
5+
type: string
6+
jobs:
7+
- job: ${{ parameters.functionAppName }}
8+
9+
variables:
10+
agentName: ${{ parameters.agentName }}
11+
functionApp: ${{ parameters.functionAppName }}
12+
functionAppOutputPath: $(Build.ArtifactStagingDirectory)/FunctionApps/$(functionApp)
13+
14+
steps:
15+
16+
- template: /eng/ci/templates/install-dotnet.yml@self
17+
18+
- script: dotnet tool install -g Microsoft.Crank.Agent --version "0.2.0-*"
19+
displayName: Install Microsoft.Crank.Agent tool
20+
21+
- pwsh: Start-Process powershell -ArgumentList '-NoExit', '-Command', 'crank-agent'
22+
displayName: Start crank agent
23+
24+
- pwsh: |
25+
dotnet --info
26+
displayName: Print .NET info
27+
28+
- task: CopyFiles@2
29+
displayName: Copy benchmark apps to temp location
30+
inputs:
31+
SourceFolder: '$(Build.SourcesDirectory)/test/Performance/Apps'
32+
Contents: '**/*'
33+
TargetFolder: '$(Build.ArtifactStagingDirectory)/PerformanceTestApps'
34+
CleanTargetFolder: true
35+
36+
- task: DotNetCoreCLI@2
37+
displayName: Publish benchmark app
38+
inputs:
39+
command: publish
40+
publishWebProjects: false
41+
zipAfterPublish: false
42+
modifyOutputPath: false
43+
projects: '$(Build.ArtifactStagingDirectory)/PerformanceTestApps/$(functionApp)/HelloHttp.csproj'
44+
arguments: -c Release -o $(functionAppOutputPath) -f net9.0 -v n
45+
workingDirectory: $(Build.ArtifactStagingDirectory)/PerformanceTestApps/$(functionApp)
46+
47+
- script: |
48+
netsh advfirewall firewall add rule name="Open Port 5000" dir=in action=allow protocol=TCP localport=5000
49+
netsh advfirewall firewall add rule name="Open Port 5010" dir=in action=allow protocol=TCP localport=5010
50+
displayName: Open port 5000 and 5010
51+
52+
- task: PowerShell@2
53+
displayName: Get Agent IP Address
54+
inputs:
55+
targetType: 'inline'
56+
script: |
57+
$ipAddress = (Test-Connection -ComputerName (hostname) -Count 1).IPv4Address.IPAddressToString
58+
Write-Host "IP Address: $ipAddress"
59+
Write-Host "##vso[task.setvariable variable=agentIp]$ipAddress"
60+
61+
- task: AzureCLI@2
62+
displayName: Persist Agent IP Address
63+
inputs:
64+
azureSubscription: $(ServiceConnection)
65+
scriptType: 'pscore'
66+
scriptLocation: 'inlineScript'
67+
inlineScript: |
68+
az storage entity insert `
69+
--auth-mode login `
70+
--account-name $(StorageAccount) `
71+
--table-name $(BenchmarkAgentInfoTableName) `
72+
--entity PartitionKey=$(Build.BuildNumber) RowKey=$(agentName) AgentName=$(agentName) AgentIPAddress=$(agentIp)
73+
74+
- pwsh: |
75+
$url = "http://localhost:5010/jobs/all"
76+
Write-Host "Calling $url to check benchmark job status"
77+
$maxAttempts = 60
78+
$attempt = 0
79+
80+
while ($attempt -lt $maxAttempts) {
81+
$response = Invoke-WebRequest -Uri $url -Method Get -ErrorAction Stop
82+
$data = $response.Content | ConvertFrom-Json
83+
$completedJobCount = ($data | Where-Object { $_.state -eq "Deleted" }).Count
84+
85+
if ($completedJobCount -gt 0) {
86+
Write-Host "Found at least 1 job with 'state' = 'Deleted'. Exiting task."
87+
exit 0
88+
}
89+
90+
Write-Host "No completed jobs found. Polling again in 30 seconds..."
91+
Start-Sleep -Seconds 30
92+
$attempt++
93+
}
94+
95+
Write-Host "Maximum attempts reached. Exiting..."
96+
exit 1
97+
displayName: Wait for job completion
98+
99+
- task: AzureCLI@2
100+
displayName: Clean up storage
101+
condition: always()
102+
inputs:
103+
azureSubscription: $(ServiceConnection)
104+
scriptType: 'pscore'
105+
scriptLocation: 'inlineScript'
106+
inlineScript: |
107+
az storage entity delete `
108+
--auth-mode login `
109+
--account-name $(StorageAccount) `
110+
--table-name $(BenchmarkAgentInfoTableName) `
111+
--partition-key $(Build.BuildNumber) --row-key=$(agentName)

‎eng/perf/http.benchmarks.yml

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ jobs:
1212
environmentVariables:
1313
FUNCTIONS_WORKER_RUNTIME: '{{FunctionsWorkerRuntime}}'
1414
AzureWebJobsScriptRoot: '{{FunctionAppPath}}'
15+
ASPNETCORE_URLS: '{{ AspNetUrls }}'
1516

1617
scenarios:
1718
hellohttp:
@@ -27,11 +28,12 @@ scenarios:
2728
profiles:
2829
win2022:
2930
variables:
30-
serverAddress: localhost
31+
serverUri: http://{{ CrankAgentAppVm }}
32+
serverPort: 5000
3133
jobs:
3234
hostruntime:
3335
endpoints:
34-
- http://localhost:5010
36+
- http://{{ CrankAgentAppVm }}:5010
3537
load:
3638
endpoints:
37-
- http://localhost:5010
39+
- http://{{ CrankAgentLoadVm }}:5010

0 commit comments

Comments
 (0)