Skip to content

Commit 3cf90ac

Browse files
andr317candr317c
committed
V15 QA add parallelization of e2e tests (#17691)
* Tries parallelizing the E2E tests * Updated variable * Updated SqlServer * Updated playwright config * Bumped to use 3 shards * Implemented shards on nightly pipeline * Fixed indentation * Added a smokeTest * Updated test to run * Added test command * Update Templates.spec.ts (cherry picked from commit 303758d)
1 parent e87cd0a commit 3cf90ac

File tree

3 files changed

+83
-30
lines changed

3 files changed

+83
-30
lines changed

‎build/azure-pipelines.yml‎

Lines changed: 39 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -440,10 +440,24 @@ stages:
440440
CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.Sqlite
441441
strategy:
442442
matrix:
443-
Linux:
443+
LinuxPart1Of3:
444444
vmImage: "ubuntu-latest"
445-
Windows:
445+
testCommand: "npx playwright test DefaultConfig --grep \"@smoke\"--shard=1/3"
446+
LinuxPart2Of3:
447+
vmImage: "ubuntu-latest"
448+
testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=2/3"
449+
LinuxPart3Of3:
450+
vmImage: "ubuntu-latest"
451+
testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=3/3"
452+
WindowsPart1Of3:
446453
vmImage: "windows-latest"
454+
testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=1/3"
455+
WindowsPart2Of3:
456+
vmImage: "windows-latest"
457+
testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=2/3"
458+
WindowsPart3Of3:
459+
vmImage: "windows-latest"
460+
testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=3/3"
447461
pool:
448462
vmImage: $(vmImage)
449463
steps:
@@ -527,7 +541,7 @@ stages:
527541
workingDirectory: tests/Umbraco.Tests.AcceptanceTest
528542

529543
# Test
530-
- pwsh: npm run smokeTestSqlite --ignore-certificate-errors
544+
- pwsh: $(testCommand)
531545
displayName: Run Playwright tests
532546
continueOnError: true
533547
workingDirectory: tests/Umbraco.Tests.AcceptanceTest
@@ -569,13 +583,31 @@ stages:
569583
CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.SqlClient
570584
strategy:
571585
matrix:
572-
${{ if eq(parameters.sqlServerLinuxAcceptanceTests, True) }} :
573-
Linux:
586+
${{ if eq(parameters.sqlServerLinuxAcceptanceTests, True) }}:
587+
LinuxPart1Of3:
588+
testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=1/3"
574589
vmImage: "ubuntu-latest"
575590
SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
576591
CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
577-
Windows:
592+
LinuxPart2Of3:
593+
testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=2/3"
594+
vmImage: "ubuntu-latest"
595+
SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
596+
CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
597+
LinuxPart3Of3:
598+
testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=3/3"
599+
vmImage: "ubuntu-latest"
600+
SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
601+
CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
602+
WindowsPart1Of3:
603+
vmImage: "windows-latest"
604+
testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=1/3"
605+
WindowsPart2Of3:
606+
vmImage: "windows-latest"
607+
testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=2/3"
608+
WindowsPart3Of3:
578609
vmImage: "windows-latest"
610+
testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=3/3"
579611
pool:
580612
vmImage: $(vmImage)
581613
steps:
@@ -667,7 +699,7 @@ stages:
667699
workingDirectory: tests/Umbraco.Tests.AcceptanceTest
668700

669701
# Test
670-
- pwsh: npm run smokeTest --ignore-certificate-errors
702+
- pwsh: $(testCommand)
671703
displayName: Run Playwright tests
672704
continueOnError: true
673705
workingDirectory: tests/Umbraco.Tests.AcceptanceTest

‎build/nightly-E2E-test-pipelines.yml‎

Lines changed: 43 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,6 @@ variables:
2323
npm_config_cache: $(Pipeline.Workspace)/.npm_client
2424
NODE_OPTIONS: --max_old_space_size=16384
2525

26-
parameters:
27-
- name: runSmokeTests
28-
displayName: Run the smoke tests
29-
type: boolean
30-
default: false
31-
3226
stages:
3327
###############################################
3428
## Build
@@ -115,10 +109,24 @@ stages:
115109
CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.Sqlite
116110
strategy:
117111
matrix:
118-
Linux:
119-
vmImage: 'ubuntu-latest'
120-
Windows:
121-
vmImage: 'windows-latest'
112+
LinuxPart1Of3:
113+
vmImage: "ubuntu-latest"
114+
testCommand: "npx playwright test DefaultConfig --shard=1/3"
115+
LinuxPart2Of3:
116+
vmImage: "ubuntu-latest"
117+
testCommand: "npx playwright test DefaultConfig --shard=2/3"
118+
LinuxPart3Of3:
119+
vmImage: "ubuntu-latest"
120+
testCommand: "npx playwright test DefaultConfig --shard=3/3"
121+
WindowsPart1Of3:
122+
vmImage: "windows-latest"
123+
testCommand: "npx playwright test DefaultConfig --shard=1/3"
124+
WindowsPart2Of3:
125+
vmImage: "windows-latest"
126+
testCommand: "npx playwright test DefaultConfig --shard=2/3"
127+
WindowsPart3Of3:
128+
vmImage: "windows-latest"
129+
testCommand: "npx playwright test DefaultConfig --shard=3/3"
122130
pool:
123131
vmImage: $(vmImage)
124132
steps:
@@ -206,10 +214,7 @@ stages:
206214
workingDirectory: tests/Umbraco.Tests.AcceptanceTest
207215

208216
# Test
209-
- ${{ if eq(parameters.runSmokeTests, true) }}:
210-
pwsh: npm run smokeTestSqlite --ignore-certificate-errors
211-
${{ else }}:
212-
pwsh: npm run testSqlite --ignore-certificate-errors
217+
- pwsh: $(testCommand)
213218
displayName: Run Playwright tests
214219
continueOnError: true
215220
workingDirectory: tests/Umbraco.Tests.AcceptanceTest
@@ -252,12 +257,30 @@ stages:
252257
CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.SqlClient
253258
strategy:
254259
matrix:
255-
Linux:
256-
vmImage: 'ubuntu-latest'
260+
LinuxPart1Of3:
261+
testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=1/3"
262+
vmImage: "ubuntu-latest"
263+
SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
264+
CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
265+
LinuxPart2Of3:
266+
testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=2/3"
267+
vmImage: "ubuntu-latest"
268+
SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
269+
CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
270+
LinuxPart3Of3:
271+
testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=3/3"
272+
vmImage: "ubuntu-latest"
257273
SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
258-
CONNECTIONSTRINGS__UMBRACODBDSN: 'Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True'
259-
Windows:
260-
vmImage: 'windows-latest'
274+
CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
275+
WindowsPart1Of3:
276+
vmImage: "windows-latest"
277+
testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=1/3"
278+
WindowsPart2Of3:
279+
vmImage: "windows-latest"
280+
testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=2/3"
281+
WindowsPart3Of3:
282+
vmImage: "windows-latest"
283+
testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=3/3"
261284
pool:
262285
vmImage: $(vmImage)
263286
steps:
@@ -353,10 +376,7 @@ stages:
353376
workingDirectory: tests/Umbraco.Tests.AcceptanceTest
354377

355378
# Test
356-
- ${{ if eq(parameters.runSmokeTests, true) }}:
357-
pwsh: npm run smokeTest --ignore-certificate-errors
358-
${{ else }}:
359-
pwsh: npm run test --ignore-certificate-errors
379+
- pwsh: $(testCommand)
360380
displayName: Run Playwright tests
361381
continueOnError: true
362382
workingDirectory: tests/Umbraco.Tests.AcceptanceTest

‎tests/Umbraco.Tests.AcceptanceTest/playwright.config.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ export default defineConfig({
4747
use: {
4848
...devices['Desktop Chrome'],
4949
// Use prepared auth state.
50+
ignoreHTTPSErrors: true,
5051
storageState: STORAGE_STATE,
5152
},
5253
},

0 commit comments

Comments
 (0)