Skip to content

Commit 09f4edd

Browse files
kshyjukshyju
authored
Add benchmarking pipeline (#10738)
* Adding benchmark pipeline changes. * update schedule to midnight. * Improvements * Optimizations
1 parent 969f761 commit 09f4edd

File tree

13 files changed

+317
-0
lines changed

13 files changed

+317
-0
lines changed

‎eng/ci/host.benchmarks.yml

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# No triggers for code push to any branch.
2+
trigger: none
3+
4+
# No PR triggers for now
5+
pr: none
6+
7+
schedules:
8+
- cron: "0 0 * * *"
9+
displayName: Nightly Build
10+
branches:
11+
include:
12+
- dev
13+
always: true
14+
15+
resources:
16+
repositories:
17+
- repository: 1es
18+
type: git
19+
name: 1ESPipelineTemplates/1ESPipelineTemplates
20+
ref: refs/tags/release
21+
- repository: eng
22+
type: git
23+
name: engineering
24+
ref: refs/tags/release
25+
26+
variables:
27+
- template: /eng/ci/templates/variables/benchmarks.yml@self
28+
- template: /ci/variables/cfs.yml@eng
29+
30+
extends:
31+
template: v1/1ES.Unofficial.PipelineTemplate.yml@1es
32+
parameters:
33+
pool:
34+
name: 1es-pool-azfunc-large
35+
image: 1es-windows-2022
36+
os: windows
37+
38+
stages:
39+
- stage: HttpApps
40+
displayName: Run Benchmarks
41+
jobs:
42+
- template: /eng/ci/templates/official/jobs/run-benchmarks.yml@self
43+
parameters:
44+
description: .NET9 Web Application
45+
functionAppName: HelloHttpNet9 # App with ASP.NET Integration
46+
additionalCrankArgs: $(AdditionalCrankArguments) # Pipeline variable to pass additional arguments to crank command.
47+
storeResultsInDatabase: ${{ variables.storeBenchmarkResultsInDatabase }}
48+
- template: /eng/ci/templates/official/jobs/run-benchmarks.yml@self
49+
parameters:
50+
description: .NET9 Worker Application
51+
functionAppName: HelloHttpNet9NoProxy # App without ASP.NET Integration
52+
additionalCrankArgs: $(AdditionalCrankArguments)
53+
storeResultsInDatabase: ${{ variables.storeBenchmarkResultsInDatabase }}

‎eng/ci/templates/official/jobs/run-benchmarks.yml

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
parameters:
2+
- name: functionAppName
3+
type: string
4+
- name: description
5+
type: string
6+
- name: storeResultsInDatabase
7+
type: boolean
8+
default: false
9+
- name: additionalCrankArgs
10+
type: string
11+
default: ''
12+
13+
jobs:
14+
- job: ${{ parameters.functionAppName }}
15+
16+
variables:
17+
runDescription: ${{ parameters.description }}
18+
functionApp: ${{ parameters.functionAppName }}
19+
functionAppOutputPath: $(Build.ArtifactStagingDirectory)/FunctionApps/$(functionApp)
20+
benchmarkResultsJsonPath: "$(Build.ArtifactStagingDirectory)/BenchmarkResults/$(buildNumber)_$(functionApp).json"
21+
functionsWorkerRuntime: 'dotnet-isolated'
22+
crankAgentUrl: "http://localhost:5010" # Default crank agent URL.
23+
configFilePath: "./eng/perf/http.benchmarks.yml"
24+
hostLocation: "./../../"
25+
26+
templateContext:
27+
outputParentDirectory: $(Build.ArtifactStagingDirectory)
28+
outputs:
29+
- output: pipelineArtifact
30+
displayName: Publish benchmark results
31+
path: $(benchmarkResultsJsonPath)
32+
artifact: 'BenchmarkResults_$(functionApp)'
33+
34+
steps:
35+
36+
- task: AzureKeyVault@2
37+
condition: and(succeeded(), eq('${{ parameters.storeResultsInDatabase }}', true))
38+
inputs:
39+
azureSubscription: Azure-Functions-Host-CI-internal
40+
KeyVaultName: functions-perf-crank-kv
41+
SecretsFilter: BenchmarkResultsSqlConnectionString
42+
RunAsPreJob: false
43+
44+
- template: /eng/ci/templates/install-dotnet.yml@self
45+
46+
- script: dotnet tool install -g Microsoft.Crank.Agent --version "0.2.0-*"
47+
displayName: Install Microsoft.Crank.Agent tool
48+
49+
- task: PowerShell@2
50+
displayName: Start crank-agent
51+
inputs:
52+
targetType: 'inline'
53+
script: |
54+
Start-Process powershell -ArgumentList '-NoExit', '-Command', 'crank-agent'
55+
56+
- task: CopyFiles@2
57+
displayName: Copy benchmark apps to temp location
58+
inputs:
59+
SourceFolder: '$(Build.SourcesDirectory)/test/Performance/Apps'
60+
Contents: '**/*'
61+
TargetFolder: '$(Build.ArtifactStagingDirectory)/PerformanceTestApps'
62+
CleanTargetFolder: true
63+
64+
- task: DotNetCoreCLI@2
65+
displayName: Publish $(functionApp) app
66+
inputs:
67+
command: publish
68+
publishWebProjects: false
69+
zipAfterPublish: false
70+
modifyOutputPath: false
71+
projects: '$(Build.ArtifactStagingDirectory)/PerformanceTestApps/$(functionApp)/HelloHttp.csproj'
72+
arguments: -c Release -o $(functionAppOutputPath) -f net9.0
73+
workingDirectory: $(Build.ArtifactStagingDirectory)/PerformanceTestApps/$(functionApp)
74+
75+
- script: dotnet tool install -g Microsoft.Crank.Controller --version "0.2.0-*"
76+
displayName: Install Microsoft.Crank.Controller
77+
78+
- task: PowerShell@2
79+
displayName: Run crank-controller
80+
inputs:
81+
targetType: 'inline'
82+
script: |
83+
$crankArgs = "--config $(configFilePath) --scenario hellohttp --profile win2022 --load.options.reuseBuild true --description `"$(runDescription)`" --command-line-property --no-metadata --no-measurements --json $(benchmarkResultsJsonPath) --property sourceVersion=$(sourceVersion) --property buildNumber=$(buildNumber) --property buildId=$(buildId) --variable FunctionsWorkerRuntime=$(functionsWorkerRuntime) --variable HostLocation=$(hostLocation) --variable FunctionAppPath=$(functionAppOutputPath)"
84+
$crankArgs += " ${{ parameters.additionalCrankArgs }}"
85+
$command = "crank $crankArgs"
86+
87+
if ('${{ parameters.storeResultsInDatabase }}' -eq 'true') {
88+
$command += " --table HttpBenchmarks --sql `"$(BenchmarkResultsSqlConnectionString)`""
89+
}
90+
91+
Write-Host "Running command: $command"
92+
Invoke-Expression $command
93+
94+
- task: PowerShell@2
95+
displayName: Functions host logs
96+
inputs:
97+
targetType: 'inline'
98+
script: |
99+
$url = "$(crankAgentUrl)/jobs/1/output"
100+
Write-Host "Making GET request to: $url to get logs"
101+
$response = Invoke-WebRequest -Uri $url -Method GET -UseBasicParsing
102+
Write-Host $response.Content

‎eng/ci/templates/variables/benchmarks.yml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
variables:
2+
- name: buildId
3+
value: $(Build.BuildId)
4+
- name: buildNumber
5+
value: $(Build.BuildNumber)
6+
- name: sourceVersion
7+
value: $(Build.SourceVersion)
8+
- name: storeBenchmarkResultsInDatabase
9+
value: ${{ eq(variables['Build.Reason'], 'Schedule') }}

‎eng/perf/http.benchmarks.yml

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
imports:
2+
- https://raw.githubusercontent.com/dotnet/crank/main/src/Microsoft.Crank.Jobs.Bombardier/bombardier.yml
3+
4+
jobs:
5+
6+
server:
7+
sources:
8+
functionshost:
9+
localFolder: '{{HostLocation}}'
10+
project: functionshost/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj
11+
readyStateText: 'Application started.'
12+
environmentVariables:
13+
FUNCTIONS_WORKER_RUNTIME: '{{FunctionsWorkerRuntime}}'
14+
AzureWebJobsScriptRoot: '{{FunctionAppPath}}'
15+
16+
scenarios:
17+
hellohttp:
18+
hostruntime:
19+
job: server
20+
21+
load:
22+
job: bombardier
23+
variables:
24+
serverPort: 5000
25+
path: /api/hello
26+
27+
profiles:
28+
win2022:
29+
variables:
30+
serverAddress: localhost
31+
jobs:
32+
hostruntime:
33+
endpoints:
34+
- http://localhost:5010
35+
load:
36+
endpoints:
37+
- http://localhost:5010

‎test/Performance/Apps/HelloHttpNet9/Hello.cs

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
using Microsoft.AspNetCore.Http;
2+
using Microsoft.AspNetCore.Mvc;
3+
using Microsoft.Azure.Functions.Worker;
4+
using Microsoft.Extensions.Logging;
5+
6+
namespace HelloHttpNet9
7+
{
8+
public sealed class Hello(ILogger<Hello> logger)
9+
{
10+
[Function("Hello")]
11+
public IActionResult Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequest req)
12+
{
13+
logger.LogInformation("C# HTTP trigger function processed a request.");
14+
return new OkObjectResult("Welcome to Azure Functions!");
15+
}
16+
}
17+
}

‎test/Performance/Apps/HelloHttpNet9/HelloHttp.csproj

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
<PropertyGroup>
3+
<TargetFramework>net9.0</TargetFramework>
4+
<AzureFunctionsVersion>v4</AzureFunctionsVersion>
5+
<OutputType>Exe</OutputType>
6+
<ImplicitUsings>enable</ImplicitUsings>
7+
<Nullable>enable</Nullable>
8+
<PublishReadyToRun>true</PublishReadyToRun>
9+
</PropertyGroup>
10+
<ItemGroup>
11+
<FrameworkReference Include="Microsoft.AspNetCore.App" />
12+
<PackageReference Include="Microsoft.Azure.Functions.Worker" Version="2.0.0" />
13+
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http.AspNetCore" Version="2.0.0" />
14+
<PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" Version="2.0.0" />
15+
</ItemGroup>
16+
</Project>

‎test/Performance/Apps/HelloHttpNet9/Program.cs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
using Microsoft.Azure.Functions.Worker.Builder;
2+
using Microsoft.Extensions.Hosting;
3+
4+
var builder = FunctionsApplication.CreateBuilder(args);
5+
6+
builder.ConfigureFunctionsWebApplication();
7+
8+
builder.Build().Run();

‎test/Performance/Apps/HelloHttpNet9/host.json

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"version": "2.0",
3+
"logging": {
4+
"applicationInsights": {
5+
"samplingSettings": {
6+
"isEnabled": true,
7+
"excludedTypes": "Request"
8+
},
9+
"enableLiveMetricsFilters": true
10+
}
11+
}
12+
}

‎test/Performance/Apps/HelloHttpNet9NoProxy/Hello.cs

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
using Microsoft.Azure.Functions.Worker;
2+
using Microsoft.Azure.Functions.Worker.Http;
3+
using Microsoft.Extensions.Logging;
4+
using System.Net;
5+
6+
namespace HelloHttpNet9
7+
{
8+
public sealed class Hello(ILogger<Hello> logger)
9+
{
10+
[Function("Hello")]
11+
public HttpResponseData Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req)
12+
{
13+
logger.LogInformation("C# HTTP trigger function processed a request.");
14+
15+
var response = req.CreateResponse(HttpStatusCode.OK);
16+
response.Headers.Add("Content-Type", "text/plain; charset=utf-8");
17+
response.WriteString("Welcome to Azure Functions!");
18+
19+
return response;
20+
}
21+
}
22+
}

‎test/Performance/Apps/HelloHttpNet9NoProxy/HelloHttp.csproj

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
<PropertyGroup>
3+
<TargetFramework>net9.0</TargetFramework>
4+
<AzureFunctionsVersion>v4</AzureFunctionsVersion>
5+
<OutputType>Exe</OutputType>
6+
<ImplicitUsings>enable</ImplicitUsings>
7+
<Nullable>enable</Nullable>
8+
<PublishReadyToRun>true</PublishReadyToRun>
9+
</PropertyGroup>
10+
<ItemGroup>
11+
<FrameworkReference Include="Microsoft.AspNetCore.App" />
12+
<PackageReference Include="Microsoft.Azure.Functions.Worker" Version="2.0.0" />
13+
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http" Version="3.1.0" />
14+
<PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" Version="2.0.0" />
15+
</ItemGroup>
16+
</Project>

0 commit comments

Comments
 (0)