Skip to content

Commit f5823c1

Browse files
authored
Add Stress Tests to CI (#3558)
* User Story 38131: Stress Tests CI - Added config file path configuration via STRESS_CONFIG_FILE environment variable. - Added support for JSON config file. - Fixed process exit code to reflect success vs failure. - Added emission to console. - Added emission of data source. - Added EntraId password-based auth. - Added CI stage. - Added NuGet package version to MDS assembly. - Added dotnet build configuration pararmeter to CI entry points. - Added MDS package version configurability to stress tests. - Added all target frameworks that MDS supports. - Each test run now creates and drops its own database to avoid collisions. - Moved to 1ES images where possible. - Using a local SQL Server that we setup as part of CI. * User Story 38131: Stress Tests CI - Added top-level pipeline flag to enable stress tests, deafult is false until tests are reliably passing. * User Story 38131: Stress Tests CI - Addressed Copilot suggestions.
1 parent 9041a52 commit f5823c1

29 files changed

+1023
-151
lines changed

.editorconfig

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ indent_size = 4
1414
[*.{json,jsonc}]
1515
indent_size = 2
1616

17+
[*.{yml,yaml}]
18+
indent_size = 2
19+
1720
# C# files
1821
[*.cs]
1922
# New line preferences

eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,11 @@
33
# The .NET Foundation licenses this file to you under the MIT license. #
44
# See the LICENSE file in the project root for more information. #
55
#################################################################################
6+
7+
# This step configures an existing SQL Server running on the local Linux host.
8+
# For example, our 1ES Hosted Pool has images like ADO-UB20-SQL22 that come with
9+
# SQL Server 2022 pre-installed and running.
10+
611
parameters:
712
- name: password
813
type: string

eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@
33
# The .NET Foundation licenses this file to you under the MIT license. #
44
# See the LICENSE file in the project root for more information. #
55
#################################################################################
6+
7+
# This step installs the latest SQL Server 2022 onto the macOS host and
8+
# configures it for use.
9+
610
parameters:
711
- name: password
812
type: string
@@ -13,7 +17,7 @@ parameters:
1317
default: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'))
1418

1519
steps:
16-
# Linux only steps
20+
# macOS only steps
1721
- bash: |
1822
# The "user" pipeline variable conflicts with homebrew, causing errors during install. Set it back to the pipeline user.
1923
USER=`whoami`

eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,11 @@
33
# The .NET Foundation licenses this file to you under the MIT license. #
44
# See the LICENSE file in the project root for more information. #
55
#################################################################################
6+
7+
# This step configures an existing SQL Server running on the local Windows host.
8+
# For example, our 1ES Hosted Pool has images like ADO-MMS22-SQL22 that come
9+
# with SQL Server 2022 pre-installed and running.
10+
611
parameters:
712
# Windows only parameters
813
- name: instanceName
@@ -63,7 +68,7 @@ parameters:
6368
default: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
6469

6570
steps:
66-
# windows only steps
71+
# Windows only steps
6772
- powershell: |
6873
try
6974
{

eng/pipelines/dotnet-sqlclient-ci-core.yml

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,10 +65,22 @@ parameters:
6565
- Project
6666
- Package
6767

68+
- name: buildConfiguration
69+
displayName: 'Build Configuration'
70+
default: Release
71+
values:
72+
- Release
73+
- Debug
74+
6875
- name: defaultPoolName
6976
type: string
7077
default: $(ci_var_defaultPoolName)
7178

79+
- name: enableStressTests
80+
displayName: Enable Stress Tests
81+
type: boolean
82+
default: false
83+
7284
variables:
7385
- template: libraries/ci-build-variables.yml@self
7486

@@ -84,6 +96,7 @@ stages:
8496
jobs:
8597
- template: common/templates/jobs/ci-build-nugets-job.yml@self
8698
parameters:
99+
configuration: ${{ parameters.buildConfiguration }}
87100
artifactName: $(artifactName)
88101
${{if ne(parameters.SNIVersion, '')}}:
89102
prebuildSteps:
@@ -92,6 +105,16 @@ stages:
92105
SNIVersion: ${{parameters.SNIVersion}}
93106
SNIValidationFeed: ${{parameters.SNIValidationFeed}}
94107

108+
- ${{ if eq(parameters.enableStressTests, true) }}:
109+
- template: stages/stress-tests-ci-stage.yml@self
110+
parameters:
111+
buildConfiguration: ${{ parameters.buildConfiguration }}
112+
dependsOn: [build_nugets]
113+
pipelineArtifactName: $(artifactName)
114+
mdsPackageVersion: $(NugetPackageVersion)
115+
${{ if eq(parameters.debug, 'true') }}:
116+
verbosity: 'detailed'
117+
95118
- template: common/templates/stages/ci-run-tests-stage.yml@self
96119
parameters:
97120
debug: ${{ parameters.debug }}
@@ -139,7 +162,6 @@ stages:
139162
testConfigurations:
140163
windows_sql_19_x64: # configuration name
141164
pool: ${{parameters.defaultPoolName }} # pool name
142-
hostedPool: false # whether the pool is hosted or not
143165
images: # list of images to run tests on
144166
Win22_Sql19: ADO-MMS22-SQL19 # stage display name: image name from the pool
145167
TargetFrameworks: ${{parameters.targetFrameworks }} #[net462, net8.0] # list of target frameworks to run
@@ -181,7 +203,6 @@ stages:
181203

182204
windows_sql_19_x86: # configuration name
183205
pool: ${{parameters.defaultPoolName }} # pool name
184-
hostedPool: false # whether the pool is hosted or not
185206
images: # list of images to run tests on
186207
Win22_Sql19_x86: ADO-MMS22-SQL19 # stage display name: image name from the pool
187208
TargetFrameworks: [net8.0] #[net462, net8.0] # list of target frameworks to run

eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,18 @@ parameters: # parameters are shown up in ADO UI in a build queue time
8181
- Project
8282
- Package
8383

84+
- name: buildConfiguration
85+
displayName: 'Build Configuration'
86+
default: Release
87+
values:
88+
- Release
89+
- Debug
90+
91+
- name: enableStressTests
92+
displayName: Enable Stress Tests
93+
type: boolean
94+
default: false
95+
8496
extends:
8597
template: dotnet-sqlclient-ci-core.yml@self
8698
parameters:
@@ -92,3 +104,5 @@ extends:
92104
useManagedSNI: ${{ parameters.useManagedSNI }}
93105
codeCovTargetFrameworks: ${{ parameters.codeCovTargetFrameworks }}
94106
buildType: ${{ parameters.buildType }}
107+
buildConfiguration: ${{ parameters.buildConfiguration }}
108+
enableStressTests: ${{ parameters.enableStressTests }}

eng/pipelines/dotnet-sqlclient-ci-project-reference-pipeline.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,18 @@ parameters: # parameters are shown up in ADO UI in a build queue time
7373
- Project
7474
- Package
7575

76+
- name: buildConfiguration
77+
displayName: 'Build Configuration'
78+
default: Release
79+
values:
80+
- Release
81+
- Debug
82+
83+
- name: enableStressTests
84+
displayName: Enable Stress Tests
85+
type: boolean
86+
default: false
87+
7688
extends:
7789
template: dotnet-sqlclient-ci-core.yml@self
7890
parameters:
@@ -84,3 +96,5 @@ extends:
8496
useManagedSNI: ${{ parameters.useManagedSNI }}
8597
codeCovTargetFrameworks: ${{ parameters.codeCovTargetFrameworks }}
8698
buildType: ${{ parameters.buildType }}
99+
buildConfiguration: ${{ parameters.buildConfiguration }}
100+
enableStressTests: ${{ parameters.enableStressTests }}
Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
################################################################################
2+
# Licensed to the .NET Foundation under one or more agreements. The .NET
3+
# Foundation licenses this file to you under the MIT license. See the LICENSE
4+
# file in the project root for more information.
5+
################################################################################
6+
7+
# This stage builds and runs stress tests against an MDS NuGet package available
8+
# as a pipeline artifact.
9+
#
10+
# The stress tests are located here:
11+
#
12+
# src/Microsoft.Data.SqlClient/tests/StressTests
13+
#
14+
# This template defines a job named 'run_stress_tests_job_<suffix>' that can be
15+
# depended on by downstream jobs.
16+
17+
parameters:
18+
# The suffix to append to the job name.
19+
- name: jobNameSuffix
20+
type: string
21+
default: ''
22+
23+
# The prefix to prepend to the job's display name:
24+
#
25+
# [<prefix>] Run Stress Tests
26+
#
27+
- name: displayNamePrefix
28+
type: string
29+
default: ''
30+
31+
# The name of the Azure Pipelines pool to use.
32+
- name: poolName
33+
type: string
34+
default: ''
35+
36+
# The pool VM image to use.
37+
- name: vmImage
38+
type: string
39+
default: ''
40+
41+
# The pipeline step to run to configure SQL Server.
42+
#
43+
# This step is expected to require no parameters. It must configure a SQL
44+
# Server instance listening on localhost for SQL auth via the 'sa' user with
45+
# the pipeline variable $(Password) as the password.
46+
- name: sqlSetupStep
47+
type: string
48+
default: ''
49+
50+
# The name of the pipeline artifact to download that contains the MDS package
51+
# to stress test.
52+
- name: pipelineArtifactName
53+
type: string
54+
default: ''
55+
56+
# The solution file to restore/build.
57+
- name: solution
58+
type: string
59+
default: ''
60+
61+
# The test project to run.
62+
- name: testProject
63+
type: string
64+
default: ''
65+
66+
# dotnet CLI arguments for the restore step.
67+
- name: restoreArguments
68+
type: string
69+
default: ''
70+
71+
# dotnet CLI arguments for the build and run steps.
72+
- name: buildArguments
73+
type: string
74+
default: ''
75+
76+
# The list of .NET runtimes to test against.
77+
- name: netTestRuntimes
78+
type: object
79+
default: []
80+
81+
# The list of .NET Framework runtimes to test against.
82+
- name: netFrameworkTestRuntimes
83+
type: object
84+
default: []
85+
86+
# The stress test config file contents to write to the config file.
87+
#
88+
# This should point to the SQL Server configured via the sqlSetupStep
89+
# parameter, with user 'sa' and password of $(Password).
90+
- name: configContent
91+
type: string
92+
default: ''
93+
94+
jobs:
95+
- job: run_stress_tests_job_${{ parameters.jobNameSuffix }}
96+
displayName: '[${{ parameters.displayNamePrefix }}] Run Stress Tests'
97+
pool:
98+
name: ${{ parameters.poolName }}
99+
${{ if eq(parameters.poolName, 'Azure Pipelines') }}:
100+
vmImage: ${{ parameters.vmImage }}
101+
${{ else }}:
102+
demands:
103+
- imageOverride -equals ${{ parameters.vmImage }}
104+
105+
variables:
106+
# Stress test command-line arguments.
107+
- name: testArguments
108+
value: -a SqlClient.Stress.Tests -console
109+
110+
# Explicitly unset the $PLATFORM environment variable that is set by the
111+
# 'ADO Build properties' Library in the ADO SqlClientDrivers public project.
112+
# This is defined with a non-standard Platform of 'AnyCPU', and will fail
113+
# the builds if left defined. The stress tests solution does not require
114+
# any specific Platform, and so its solution file doesn't support any
115+
# non-standard platforms.
116+
#
117+
# Note that Azure Pipelines will inject this variable as PLATFORM into the
118+
# environment of all tasks in this job.
119+
#
120+
# See:
121+
# https://learn.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch
122+
#
123+
- name: Platform
124+
value: ''
125+
126+
# Do the same for $CONFIGURATION since we explicitly set it using our
127+
# 'buildConfiguration' parameter, and we don't want the environment to
128+
# override us.
129+
- name: Configuration
130+
value: ''
131+
132+
steps:
133+
134+
# Install the .NET 9.0 SDK.
135+
- task: UseDotNet@2
136+
displayName: Install .NET 9.0 SDK
137+
inputs:
138+
packageType: sdk
139+
version: 9.x
140+
141+
# Install the .NET 8.0 runtime.
142+
- task: UseDotNet@2
143+
displayName: Install .NET 8.0 Runtime
144+
inputs:
145+
packageType: runtime
146+
version: 8.x
147+
148+
# Download the pipeline artifact that contains the MDS package to test.
149+
- task: DownloadPipelineArtifact@2
150+
displayName: Download Pipeline Artifact
151+
inputs:
152+
artifactName: ${{ parameters.pipelineArtifactName }}
153+
# The stress tests solution has a NuGet.config file that configures
154+
# sources to look in this packages/ directory.
155+
targetPath: $(Build.SourcesDirectory)/packages
156+
157+
# Setup the local SQL Server.
158+
- template: ${{ parameters.sqlSetupStep }}@self
159+
160+
# We use the 'custom' command because the DotNetCoreCLI@2 task doesn't support
161+
# all of our argument combinations for the different build steps.
162+
163+
# Restore the solution.
164+
- task: DotNetCoreCLI@2
165+
displayName: Restore Solution
166+
inputs:
167+
command: custom
168+
custom: restore
169+
projects: ${{ parameters.solution }}
170+
arguments: ${{ parameters.restoreArguments }}
171+
172+
# Build the solution.
173+
- task: DotNetCoreCLI@2
174+
displayName: Build Solution
175+
inputs:
176+
command: custom
177+
custom: build
178+
projects: ${{ parameters.solution }}
179+
arguments: ${{ parameters.buildArguments }} --no-restore
180+
181+
# Write the config file.
182+
- task: PowerShell@2
183+
displayName: Write Config File
184+
inputs:
185+
pwsh: true
186+
targetType: inline
187+
script: |
188+
# Capture the multi-line JSON content into a variable.
189+
$content = @"
190+
${{ parameters.configContent }}
191+
"@
192+
193+
# Write the JSON content to the config file.
194+
$content | Out-File -FilePath "config.json"
195+
196+
# Run the stress tests for each .NET runtime.
197+
- ${{ each runtime in parameters.netTestRuntimes }}:
198+
- task: DotNetCoreCLI@2
199+
displayName: Test [${{runtime}}]
200+
inputs:
201+
command: custom
202+
custom: run
203+
projects: ${{ parameters.testProject }}
204+
arguments: ${{ parameters.buildArguments }} --no-build -f ${{runtime}} -e STRESS_CONFIG_FILE=config.json -- $(testArguments)
205+
206+
# Run the stress tests for each .NET Framework runtime.
207+
- ${{ each runtime in parameters.netFrameworkTestRuntimes }}:
208+
- task: DotNetCoreCLI@2
209+
displayName: Test [${{runtime}}]
210+
inputs:
211+
command: custom
212+
custom: run
213+
projects: ${{ parameters.testProject }}
214+
arguments: ${{ parameters.buildArguments }} --no-build -f ${{runtime}} -e STRESS_CONFIG_FILE=config.json -- $(testArguments)

0 commit comments

Comments
 (0)