Skip to content

Commit b01def0

Browse files
andr317candr317c
authored
V15 QA added a fix for flaky integration tests run on SQL Server Linux (#18965)
1 parent b106305 commit b01def0

File tree

2 files changed

+241
-4
lines changed

2 files changed

+241
-4
lines changed

‎build/azure-pipelines.yml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -432,6 +432,33 @@ stages:
432432
displayName: Start SQL Server Docker image (Linux)
433433
condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
434434

435+
- powershell: |
436+
$maxAttempts = 12
437+
$attempt = 0
438+
$status = ""
439+
440+
while (($status -ne 'running') -and ($attempt -lt $maxAttempts)) {
441+
Start-Sleep -Seconds 5
442+
# We use the docker inspect command to check the status of the container. If the container is not running, we wait 5 seconds and try again. And if reaches 12 attempts, we fail the build.
443+
$status = docker inspect -f '{{.State.Status}}' mssql
444+
445+
if ($status -ne 'running') {
446+
Write-Host "Waiting for SQL Server to be ready... Attempt $($attempt + 1)"
447+
$attempt++
448+
}
449+
}
450+
451+
if ($status -eq 'running') {
452+
Write-Host "SQL Server container is running"
453+
docker ps -a
454+
} else {
455+
Write-Host "SQL Server did not become ready in time. Last known status: $status"
456+
docker logs mssql
457+
exit 1
458+
}
459+
displayName: Wait for SQL Server to be ready (Linux)
460+
condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
461+
435462
- pwsh: SqlLocalDB start MSSQLLocalDB
436463
displayName: Start SQL Server LocalDB (Windows)
437464
condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))

‎build/nightly-E2E-test-pipelines.yml

Lines changed: 214 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,189 @@ stages:
9393
targetPath: $(Build.ArtifactStagingDirectory)/npm
9494
artifactName: npm
9595

96+
- stage: Integration
97+
displayName: Integration Tests
98+
dependsOn: Build
99+
jobs:
100+
# Integration Tests (SQLite)
101+
- job:
102+
timeoutInMinutes: 180
103+
displayName: Integration Tests (SQLite)
104+
strategy:
105+
matrix:
106+
# Windows:
107+
# vmImage: 'windows-latest'
108+
# We split the tests into 3 parts for each OS to reduce the time it takes to run them on the pipeline
109+
LinuxPart1Of3:
110+
vmImage: "ubuntu-latest"
111+
# Filter tests that are part of the Umbraco.Infrastructure namespace but not part of the Umbraco.Infrastructure.Service namespace
112+
testFilter: "(FullyQualifiedName~Umbraco.Infrastructure) & (FullyQualifiedName!~Umbraco.Infrastructure.Service)"
113+
LinuxPart2Of3:
114+
vmImage: "ubuntu-latest"
115+
# Filter tests that are part of the Umbraco.Infrastructure.Service namespace
116+
testFilter: "(FullyQualifiedName~Umbraco.Infrastructure.Service)"
117+
LinuxPart3Of3:
118+
vmImage: "ubuntu-latest"
119+
# Filter tests that are not part of the Umbraco.Infrastructure namespace. So this will run all tests that are not part of the Umbraco.Infrastructure namespace
120+
testFilter: "(FullyQualifiedName!~Umbraco.Infrastructure)"
121+
macOSPart1Of3:
122+
vmImage: "macOS-latest"
123+
# Filter tests that are part of the Umbraco.Infrastructure namespace but not part of the Umbraco.Infrastructure.Service namespace
124+
testFilter: "(FullyQualifiedName~Umbraco.Infrastructure) & (FullyQualifiedName!~Umbraco.Infrastructure.Service)"
125+
macOSPart2Of3:
126+
vmImage: "macOS-latest"
127+
# Filter tests that are part of the Umbraco.Infrastructure.Service namespace
128+
testFilter: "(FullyQualifiedName~Umbraco.Infrastructure.Service)"
129+
macOSPart3Of3:
130+
vmImage: "macOS-latest"
131+
# Filter tests that are not part of the Umbraco.Infrastructure namespace.
132+
testFilter: "(FullyQualifiedName!~Umbraco.Infrastructure)"
133+
pool:
134+
vmImage: $(vmImage)
135+
variables:
136+
Tests__Database__DatabaseType: "Sqlite"
137+
steps:
138+
- checkout: self
139+
submodules: false
140+
lfs: false,
141+
fetchDepth: 1
142+
fetchFilter: tree:0
143+
# Setup test environment
144+
- task: DownloadPipelineArtifact@2
145+
displayName: Download build artifacts
146+
inputs:
147+
artifact: build_output
148+
path: $(Build.SourcesDirectory)
149+
150+
- task: UseDotNet@2
151+
displayName: Use .NET SDK from global.json
152+
inputs:
153+
useGlobalJson: true
154+
155+
# Test
156+
- task: DotNetCoreCLI@2
157+
displayName: Run dotnet test
158+
inputs:
159+
command: test
160+
projects: "tests/Umbraco.Tests.Integration/Umbraco.Tests.Integration.csproj"
161+
testRunTitle: Integration Tests SQLite - $(Agent.OS)
162+
arguments: '--filter "$(testFilter)" --configuration $(buildConfiguration) --no-build'
163+
164+
# Integration Tests (SQL Server)
165+
- job:
166+
timeoutInMinutes: 180
167+
displayName: Integration Tests (SQL Server)
168+
strategy:
169+
matrix:
170+
# We split the tests into 3 parts for each OS to reduce the time it takes to run them on the pipeline
171+
WindowsPart1Of3:
172+
vmImage: "windows-latest"
173+
Tests__Database__DatabaseType: LocalDb
174+
Tests__Database__SQLServerMasterConnectionString: N/A
175+
# Filter tests that are part of the Umbraco.Infrastructure namespace but not part of the Umbraco.Infrastructure.Service namespace
176+
testFilter: "(FullyQualifiedName~Umbraco.Infrastructure) & (FullyQualifiedName!~Umbraco.Infrastructure.Service)"
177+
WindowsPart2Of3:
178+
vmImage: "windows-latest"
179+
Tests__Database__DatabaseType: LocalDb
180+
Tests__Database__SQLServerMasterConnectionString: N/A
181+
# Filter tests that are part of the Umbraco.Infrastructure.Service namespace
182+
testFilter: "(FullyQualifiedName~Umbraco.Infrastructure.Service)"
183+
WindowsPart3Of3:
184+
vmImage: "windows-latest"
185+
Tests__Database__DatabaseType: LocalDb
186+
Tests__Database__SQLServerMasterConnectionString: N/A
187+
# Filter tests that are not part of the Umbraco.Infrastructure namespace. So this will run all tests that are not part of the Umbraco.Infrastructure namespace
188+
testFilter: "(FullyQualifiedName!~Umbraco.Infrastructure)"
189+
LinuxPart1Of3:
190+
vmImage: "ubuntu-latest"
191+
SA_PASSWORD: UmbracoIntegration123!
192+
Tests__Database__DatabaseType: SqlServer
193+
Tests__Database__SQLServerMasterConnectionString: "Server=(local);User Id=sa;Password=$(SA_PASSWORD);Encrypt=True;TrustServerCertificate=True"
194+
# Filter tests that are part of the Umbraco.Infrastructure namespace but not part of the Umbraco.Infrastructure.Service namespace
195+
testFilter: "(FullyQualifiedName~Umbraco.Infrastructure) & (FullyQualifiedName!~Umbraco.Infrastructure.Service)"
196+
LinuxPart2Of3:
197+
vmImage: "ubuntu-latest"
198+
SA_PASSWORD: UmbracoIntegration123!
199+
Tests__Database__DatabaseType: SqlServer
200+
Tests__Database__SQLServerMasterConnectionString: "Server=(local);User Id=sa;Password=$(SA_PASSWORD);Encrypt=True;TrustServerCertificate=True"
201+
# Filter tests that are part of the Umbraco.Infrastructure.Service namespace
202+
testFilter: "(FullyQualifiedName~Umbraco.Infrastructure.Service)"
203+
LinuxPart3Of3:
204+
vmImage: "ubuntu-latest"
205+
SA_PASSWORD: UmbracoIntegration123!
206+
Tests__Database__DatabaseType: SqlServer
207+
Tests__Database__SQLServerMasterConnectionString: "Server=(local);User Id=sa;Password=$(SA_PASSWORD);Encrypt=True;TrustServerCertificate=True"
208+
# Filter tests that are not part of the Umbraco.Infrastructure namespace. So this will run all tests that are not part of the Umbraco.Infrastructure namespace
209+
testFilter: "(FullyQualifiedName!~Umbraco.Infrastructure)"
210+
pool:
211+
vmImage: $(vmImage)
212+
steps:
213+
# Setup test environment
214+
- task: DownloadPipelineArtifact@2
215+
displayName: Download build artifacts
216+
inputs:
217+
artifact: build_output
218+
path: $(Build.SourcesDirectory)
219+
220+
- task: UseDotNet@2
221+
displayName: Use .NET SDK from global.json
222+
inputs:
223+
useGlobalJson: true
224+
225+
# Start SQL Server
226+
- powershell: docker run --name mssql -d -p 1433:1433 -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=$(SA_PASSWORD)" mcr.microsoft.com/mssql/server:2022-latest
227+
displayName: Start SQL Server Docker image (Linux)
228+
condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
229+
230+
- powershell: |
231+
$maxAttempts = 12
232+
$attempt = 0
233+
$status = ""
234+
235+
while (($status -ne 'running') -and ($attempt -lt $maxAttempts)) {
236+
Start-Sleep -Seconds 5
237+
# We use the docker inspect command to check the status of the container. If the container is not running, we wait 5 seconds and try again. And if reaches 12 attempts, we fail the build.
238+
$status = docker inspect -f '{{.State.Status}}' mssql
239+
240+
if ($status -ne 'running') {
241+
Write-Host "Waiting for SQL Server to be ready... Attempt $($attempt + 1)"
242+
$attempt++
243+
}
244+
}
245+
246+
if ($status -eq 'running') {
247+
Write-Host "SQL Server container is running"
248+
docker ps -a
249+
} else {
250+
Write-Host "SQL Server did not become ready in time. Last known status: $status"
251+
docker logs mssql
252+
exit 1
253+
}
254+
displayName: Wait for SQL Server to be ready (Linux)
255+
condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
256+
257+
- pwsh: SqlLocalDB start MSSQLLocalDB
258+
displayName: Start SQL Server LocalDB (Windows)
259+
condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
260+
261+
# Test
262+
- task: DotNetCoreCLI@2
263+
displayName: Run dotnet test
264+
inputs:
265+
command: test
266+
projects: "tests/Umbraco.Tests.Integration/Umbraco.Tests.Integration.csproj"
267+
testRunTitle: Integration Tests SQL Server - $(Agent.OS)
268+
arguments: '--filter "$(testFilter)" --configuration $(buildConfiguration) --no-build'
269+
270+
# Stop SQL Server
271+
- pwsh: docker stop mssql
272+
displayName: Stop SQL Server Docker image (Linux)
273+
condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
274+
275+
- pwsh: SqlLocalDB stop MSSQLLocalDB
276+
displayName: Stop SQL Server LocalDB (Windows)
277+
condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
278+
96279
- stage: E2E
97280
displayName: E2E Tests
98281
dependsOn: Build
@@ -290,17 +473,17 @@ stages:
290473
testCommand: "npm run testSqlite -- --shard=1/3"
291474
vmImage: "ubuntu-latest"
292475
SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
293-
CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
476+
CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);Encrypt=True;TrustServerCertificate=True"
294477
LinuxPart2Of3:
295478
testCommand: "npm run testSqlite -- --shard=2/3"
296479
vmImage: "ubuntu-latest"
297480
SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
298-
CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
481+
CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);Encrypt=True;TrustServerCertificate=True"
299482
LinuxPart3Of3:
300483
testCommand: "npm run testSqlite -- --shard=3/3"
301484
vmImage: "ubuntu-latest"
302485
SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
303-
CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
486+
CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);Encrypt=True;TrustServerCertificate=True"
304487
WindowsPart1Of3:
305488
vmImage: "windows-latest"
306489
testCommand: "npm run testSqlite -- --shard=1/3"
@@ -371,6 +554,33 @@ stages:
371554
displayName: Start SQL Server Docker image (Linux)
372555
condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
373556

557+
- powershell: |
558+
$maxAttempts = 12
559+
$attempt = 0
560+
$status = ""
561+
562+
while (($status -ne 'running') -and ($attempt -lt $maxAttempts)) {
563+
Start-Sleep -Seconds 5
564+
# We use the docker inspect command to check the status of the container. If the container is not running, we wait 5 seconds and try again. And if reaches 12 attempts, we fail the build.
565+
$status = docker inspect -f '{{.State.Status}}' mssql
566+
567+
if ($status -ne 'running') {
568+
Write-Host "Waiting for SQL Server to be ready... Attempt $($attempt + 1)"
569+
$attempt++
570+
}
571+
}
572+
573+
if ($status -eq 'running') {
574+
Write-Host "SQL Server container is running"
575+
docker ps -a
576+
} else {
577+
Write-Host "SQL Server did not become ready in time. Last known status: $status"
578+
docker logs mssql
579+
exit 1
580+
}
581+
displayName: Wait for SQL Server to be ready (Linux)
582+
condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
583+
374584
- pwsh: SqlLocalDB start MSSQLLocalDB
375585
displayName: Start SQL Server LocalDB (Windows)
376586
condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
@@ -447,7 +657,7 @@ stages:
447657
inputs:
448658
targetPath: $(Build.ArtifactStagingDirectory)
449659
artifact: "Acceptance Test Results - $(Agent.JobName) - Attempt #$(System.JobAttempt)"
450-
660+
451661
# Publish test results
452662
- task: PublishTestResults@2
453663
displayName: "Publish test results"

0 commit comments

Comments
 (0)