Skip to content

Commit 303758d

Browse files
andr317candr317c
authored
V15 QA add parallelization of e2e tests (#17691)
* Tries parallelizing the E2E tests * Updated variable * Updated SqlServer * Updated playwright config * Bumped to use 3 shards * Implemented shards on nightly pipeline * Fixed indentation * Added a smokeTest * Updated test to run * Added test command * Update Templates.spec.ts
1 parent 01a19b8 commit 303758d

File tree

3 files changed

+83
-30
lines changed

3 files changed

+83
-30
lines changed

‎build/azure-pipelines.yml‎

Lines changed: 39 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -439,10 +439,24 @@ stages:
439439
CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.Sqlite
440440
strategy:
441441
matrix:
442-
Linux:
442+
LinuxPart1Of3:
443443
vmImage: "ubuntu-latest"
444-
Windows:
444+
testCommand: "npx playwright test DefaultConfig --grep \"@smoke\"--shard=1/3"
445+
LinuxPart2Of3:
446+
vmImage: "ubuntu-latest"
447+
testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=2/3"
448+
LinuxPart3Of3:
449+
vmImage: "ubuntu-latest"
450+
testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=3/3"
451+
WindowsPart1Of3:
445452
vmImage: "windows-latest"
453+
testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=1/3"
454+
WindowsPart2Of3:
455+
vmImage: "windows-latest"
456+
testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=2/3"
457+
WindowsPart3Of3:
458+
vmImage: "windows-latest"
459+
testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --shard=3/3"
446460
pool:
447461
vmImage: $(vmImage)
448462
steps:
@@ -526,7 +540,7 @@ stages:
526540
workingDirectory: tests/Umbraco.Tests.AcceptanceTest
527541

528542
# Test
529-
- pwsh: npm run smokeTestSqlite --ignore-certificate-errors
543+
- pwsh: $(testCommand)
530544
displayName: Run Playwright tests
531545
continueOnError: true
532546
workingDirectory: tests/Umbraco.Tests.AcceptanceTest
@@ -568,13 +582,31 @@ stages:
568582
CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.SqlClient
569583
strategy:
570584
matrix:
571-
${{ if eq(parameters.sqlServerLinuxAcceptanceTests, True) }} :
572-
Linux:
585+
${{ if eq(parameters.sqlServerLinuxAcceptanceTests, True) }}:
586+
LinuxPart1Of3:
587+
testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=1/3"
573588
vmImage: "ubuntu-latest"
574589
SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
575590
CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
576-
Windows:
591+
LinuxPart2Of3:
592+
testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=2/3"
593+
vmImage: "ubuntu-latest"
594+
SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
595+
CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
596+
LinuxPart3Of3:
597+
testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=3/3"
598+
vmImage: "ubuntu-latest"
599+
SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
600+
CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
601+
WindowsPart1Of3:
602+
vmImage: "windows-latest"
603+
testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=1/3"
604+
WindowsPart2Of3:
605+
vmImage: "windows-latest"
606+
testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=2/3"
607+
WindowsPart3Of3:
577608
vmImage: "windows-latest"
609+
testCommand: "npx playwright test DefaultConfig --grep \"@smoke\" --grep-invert \"Users\" --shard=3/3"
578610
pool:
579611
vmImage: $(vmImage)
580612
steps:
@@ -666,7 +698,7 @@ stages:
666698
workingDirectory: tests/Umbraco.Tests.AcceptanceTest
667699

668700
# Test
669-
- pwsh: npm run smokeTest --ignore-certificate-errors
701+
- pwsh: $(testCommand)
670702
displayName: Run Playwright tests
671703
continueOnError: true
672704
workingDirectory: tests/Umbraco.Tests.AcceptanceTest

‎build/nightly-E2E-test-pipelines.yml‎

Lines changed: 43 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,6 @@ variables:
2323
npm_config_cache: $(Pipeline.Workspace)/.npm_client
2424
NODE_OPTIONS: --max_old_space_size=16384
2525

26-
parameters:
27-
- name: runSmokeTests
28-
displayName: Run the smoke tests
29-
type: boolean
30-
default: false
31-
3226
stages:
3327
###############################################
3428
## Build
@@ -115,10 +109,24 @@ stages:
115109
CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.Sqlite
116110
strategy:
117111
matrix:
118-
Linux:
119-
vmImage: 'ubuntu-latest'
120-
Windows:
121-
vmImage: 'windows-latest'
112+
LinuxPart1Of3:
113+
vmImage: "ubuntu-latest"
114+
testCommand: "npx playwright test DefaultConfig --shard=1/3"
115+
LinuxPart2Of3:
116+
vmImage: "ubuntu-latest"
117+
testCommand: "npx playwright test DefaultConfig --shard=2/3"
118+
LinuxPart3Of3:
119+
vmImage: "ubuntu-latest"
120+
testCommand: "npx playwright test DefaultConfig --shard=3/3"
121+
WindowsPart1Of3:
122+
vmImage: "windows-latest"
123+
testCommand: "npx playwright test DefaultConfig --shard=1/3"
124+
WindowsPart2Of3:
125+
vmImage: "windows-latest"
126+
testCommand: "npx playwright test DefaultConfig --shard=2/3"
127+
WindowsPart3Of3:
128+
vmImage: "windows-latest"
129+
testCommand: "npx playwright test DefaultConfig --shard=3/3"
122130
pool:
123131
vmImage: $(vmImage)
124132
steps:
@@ -206,10 +214,7 @@ stages:
206214
workingDirectory: tests/Umbraco.Tests.AcceptanceTest
207215

208216
# Test
209-
- ${{ if eq(parameters.runSmokeTests, true) }}:
210-
pwsh: npm run smokeTestSqlite --ignore-certificate-errors
211-
${{ else }}:
212-
pwsh: npm run testSqlite --ignore-certificate-errors
217+
- pwsh: $(testCommand)
213218
displayName: Run Playwright tests
214219
continueOnError: true
215220
workingDirectory: tests/Umbraco.Tests.AcceptanceTest
@@ -252,12 +257,30 @@ stages:
252257
CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.SqlClient
253258
strategy:
254259
matrix:
255-
Linux:
256-
vmImage: 'ubuntu-latest'
260+
LinuxPart1Of3:
261+
testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=1/3"
262+
vmImage: "ubuntu-latest"
263+
SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
264+
CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
265+
LinuxPart2Of3:
266+
testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=2/3"
267+
vmImage: "ubuntu-latest"
268+
SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
269+
CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
270+
LinuxPart3Of3:
271+
testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=3/3"
272+
vmImage: "ubuntu-latest"
257273
SA_PASSWORD: $(UMBRACO__CMS__UNATTENDED__UNATTENDEDUSERPASSWORD)
258-
CONNECTIONSTRINGS__UMBRACODBDSN: 'Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True'
259-
Windows:
260-
vmImage: 'windows-latest'
274+
CONNECTIONSTRINGS__UMBRACODBDSN: "Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
275+
WindowsPart1Of3:
276+
vmImage: "windows-latest"
277+
testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=1/3"
278+
WindowsPart2Of3:
279+
vmImage: "windows-latest"
280+
testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=2/3"
281+
WindowsPart3Of3:
282+
vmImage: "windows-latest"
283+
testCommand: "npx playwright test DefaultConfig --grep-invert \"Users\" --shard=3/3"
261284
pool:
262285
vmImage: $(vmImage)
263286
steps:
@@ -353,10 +376,7 @@ stages:
353376
workingDirectory: tests/Umbraco.Tests.AcceptanceTest
354377

355378
# Test
356-
- ${{ if eq(parameters.runSmokeTests, true) }}:
357-
pwsh: npm run smokeTest --ignore-certificate-errors
358-
${{ else }}:
359-
pwsh: npm run test --ignore-certificate-errors
379+
- pwsh: $(testCommand)
360380
displayName: Run Playwright tests
361381
continueOnError: true
362382
workingDirectory: tests/Umbraco.Tests.AcceptanceTest

‎tests/Umbraco.Tests.AcceptanceTest/playwright.config.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ export default defineConfig({
4747
use: {
4848
...devices['Desktop Chrome'],
4949
// Use prepared auth state.
50+
ignoreHTTPSErrors: true,
5051
storageState: STORAGE_STATE,
5152
},
5253
},

0 commit comments

Comments
 (0)