diff --git a/.evergreen/buildvariants-and-tasks.in.yml b/.evergreen/buildvariants-and-tasks.in.yml index 1ac7d184f98..85f82594785 100644 --- a/.evergreen/buildvariants-and-tasks.in.yml +++ b/.evergreen/buildvariants-and-tasks.in.yml @@ -62,10 +62,11 @@ const PACKAGE_BUILD_VARIANTS = [ const TEST_PACKAGED_APP_BUILD_VARIANTS = [ { - name: 'test-packaged-app-ubuntu', - display_name: 'Ubuntu 20.04', + name: 'test-packaged-app-ubuntu-ttfq', + display_name: 'Ubuntu 20.04 (time-to-first-query)', run_on: 'ubuntu2004-large', depends_on: 'package-ubuntu', + time_to_first_query: true, }, { name: 'test-packaged-app-windows', @@ -74,23 +75,26 @@ const TEST_PACKAGED_APP_BUILD_VARIANTS = [ depends_on: 'package-windows', }, { - name: 'test-packaged-app-rhel', - display_name: 'RHEL 8.0', + name: 'test-packaged-app-rhel-ttfq', + display_name: 'RHEL 8.0 (time-to-first-query)', run_on: 'rhel80-large', depends_on: 'package-rhel', + time_to_first_query: true, }, { - name: 'test-packaged-app-macos-11-arm', - display_name: 'MacOS arm64 11', + name: 'test-packaged-app-macos-11-arm-ttfq', + display_name: 'MacOS arm64 11 (time-to-first-query)', run_on: 'macos-11-arm64-gui', - depends_on: 'package-macos-arm' + depends_on: 'package-macos-arm', + time_to_first_query: true, }, { - name: 'test-packaged-app-macos-11-x64', - display_name: 'MacOS x64 11', + name: 'test-packaged-app-macos-11-x64-ttfq', + display_name: 'MacOS x64 11 (time-to-first-query)', run_on: 'macos-11-gui', patchable: false, - depends_on: 'package-macos-x64' + depends_on: 'package-macos-x64', + time_to_first_query: true, }, { name: 'test-packaged-app-macos-14-arm', @@ -243,10 +247,10 @@ buildvariants: - name: package-compass variant: <%= buildVariant.depends_on %> tasks: - <% for (const group of E2E_TEST_GROUPS) { %> - <% if (['test-packaged-app-macos-11-arm', 'test-packaged-app-macos-11-x64'].includes(buildVariant.name)) { %> - - name: test-packaged-app-macos13-<%= group.number %> + <% if (buildVariant.time_to_first_query) { %> + - name: test-packaged-app-time-to-first-query <% } else { %> + <% for (const group of E2E_TEST_GROUPS) { %> - name: test-packaged-app-<%= group.number %> <% } %> <% } %> @@ -507,8 +511,9 @@ tasks: e2e_test_groups: <%= E2E_TEST_GROUPS.length %> e2e_test_group: <%= group.number %> debug: 'compass-e2e-tests*,electron*,hadron*,mongo*' +<% } %> - - name: test-packaged-app-macos13-<%= group.number %> + - name: test-packaged-app-time-to-first-query tags: - required-for-publish - run-on-pr @@ -530,10 +535,8 @@ tasks: vars: mongodb_version: 8.0.5 compass_distribution: compass - e2e_test_groups: <%= E2E_TEST_GROUPS.length %> - e2e_test_group: <%= group.number %> + e2e_test_filter: time-to-first-query debug: 'compass-e2e-tests*,electron*,hadron*,mongo*' -<% } %> <% for (const browser of BROWSERS) { %> <% for(const group of E2E_TEST_GROUPS) { %> diff --git a/.evergreen/buildvariants-and-tasks.yml b/.evergreen/buildvariants-and-tasks.yml index 10be3e67ae8..efbb1245bbc 100644 --- a/.evergreen/buildvariants-and-tasks.yml +++ b/.evergreen/buildvariants-and-tasks.yml @@ -169,17 +169,15 @@ buildvariants: - name: test-server-latest-alpha-1 - name: test-server-latest-alpha-2 - name: test-server-latest-alpha-3 - - name: test-packaged-app-ubuntu - display_name: Test Packaged App Ubuntu 20.04 + - name: test-packaged-app-ubuntu-ttfq + display_name: Test Packaged App Ubuntu 20.04 (time-to-first-query) run_on: ubuntu2004-large patchable: true depends_on: - name: package-compass variant: package-ubuntu tasks: - - name: test-packaged-app-1 - - name: test-packaged-app-2 - - name: test-packaged-app-3 + - name: test-packaged-app-time-to-first-query - name: test-packaged-app-windows display_name: Test Packaged App Windows 10 run_on: windows-vsCurrent-large @@ -191,39 +189,33 @@ buildvariants: - name: test-packaged-app-1 - name: test-packaged-app-2 - name: test-packaged-app-3 - - name: test-packaged-app-rhel - display_name: Test Packaged App RHEL 8.0 + - name: test-packaged-app-rhel-ttfq + display_name: Test Packaged App RHEL 8.0 (time-to-first-query) run_on: rhel80-large patchable: true depends_on: - name: package-compass variant: package-rhel tasks: - - name: test-packaged-app-1 - - name: test-packaged-app-2 - - name: test-packaged-app-3 - - name: test-packaged-app-macos-11-arm - display_name: Test Packaged App MacOS arm64 11 + - name: test-packaged-app-time-to-first-query + - name: test-packaged-app-macos-11-arm-ttfq + display_name: Test Packaged App MacOS arm64 11 (time-to-first-query) run_on: macos-11-arm64-gui patchable: true depends_on: - name: package-compass variant: package-macos-arm tasks: - - name: test-packaged-app-macos13-1 - - name: test-packaged-app-macos13-2 - - name: test-packaged-app-macos13-3 - - name: test-packaged-app-macos-11-x64 - display_name: Test Packaged App MacOS x64 11 + - name: test-packaged-app-time-to-first-query + - name: test-packaged-app-macos-11-x64-ttfq + display_name: Test Packaged App MacOS x64 11 (time-to-first-query) run_on: macos-11-gui patchable: false depends_on: - name: package-compass variant: package-macos-x64 tasks: - - name: test-packaged-app-macos13-1 - - name: test-packaged-app-macos13-2 - - name: test-packaged-app-macos13-3 + - name: test-packaged-app-time-to-first-query - name: test-packaged-app-macos-14-arm display_name: Test Packaged App MacOS arm64 14 run_on: macos-14-arm64-gui @@ -1294,31 +1286,6 @@ tasks: e2e_test_groups: 3 e2e_test_group: 1 debug: compass-e2e-tests*,electron*,hadron*,mongo* - - name: test-packaged-app-macos13-1 - tags: - - required-for-publish - - run-on-pr - - assigned_to_jira_team_compass_compass - - foliage_check_task_only - commands: - - func: prepare - - func: install - - func: bootstrap - vars: - scope: compass-e2e-tests - - func: apply-compass-target-expansion - vars: - compass_distribution: compass - - func: get-packaged-app - vars: - compass_distribution: compass - - func: test-packaged-app - vars: - mongodb_version: 8.0.5 - compass_distribution: compass - e2e_test_groups: 3 - e2e_test_group: 1 - debug: compass-e2e-tests*,electron*,hadron*,mongo* - name: test-packaged-app-2 tags: - required-for-publish @@ -1344,31 +1311,6 @@ tasks: e2e_test_groups: 3 e2e_test_group: 2 debug: compass-e2e-tests*,electron*,hadron*,mongo* - - name: test-packaged-app-macos13-2 - tags: - - required-for-publish - - run-on-pr - - assigned_to_jira_team_compass_compass - - foliage_check_task_only - commands: - - func: prepare - - func: install - - func: bootstrap - vars: - scope: compass-e2e-tests - - func: apply-compass-target-expansion - vars: - compass_distribution: compass - - func: get-packaged-app - vars: - compass_distribution: compass - - func: test-packaged-app - vars: - mongodb_version: 8.0.5 - compass_distribution: compass - e2e_test_groups: 3 - e2e_test_group: 2 - debug: compass-e2e-tests*,electron*,hadron*,mongo* - name: test-packaged-app-3 tags: - required-for-publish @@ -1394,7 +1336,7 @@ tasks: e2e_test_groups: 3 e2e_test_group: 3 debug: compass-e2e-tests*,electron*,hadron*,mongo* - - name: test-packaged-app-macos13-3 + - name: test-packaged-app-time-to-first-query tags: - required-for-publish - run-on-pr @@ -1416,8 +1358,7 @@ tasks: vars: mongodb_version: 8.0.5 compass_distribution: compass - e2e_test_groups: 3 - e2e_test_group: 3 + e2e_test_filter: time-to-first-query debug: compass-e2e-tests*,electron*,hadron*,mongo* - name: test-web-sandbox-chrome-1 tags: diff --git a/.evergreen/functions.yml b/.evergreen/functions.yml index 8d6fc8143d8..bb40be96bf9 100644 --- a/.evergreen/functions.yml +++ b/.evergreen/functions.yml @@ -606,8 +606,8 @@ functions: DEBUG: ${debug|} MONGODB_VERSION: ${mongodb_version|} MONGODB_RUNNER_VERSION: ${mongodb_version|} - COMPASS_E2E_TEST_GROUPS: ${e2e_test_groups} - COMPASS_E2E_TEST_GROUP: ${e2e_test_group} + COMPASS_E2E_TEST_GROUPS: ${e2e_test_groups|1} + COMPASS_E2E_TEST_GROUP: ${e2e_test_group|1} ATLAS_LOCAL_VERSION: latest script: | set -e @@ -638,8 +638,9 @@ functions: DEBUG: ${debug|} MONGODB_VERSION: ${mongodb_version|} MONGODB_RUNNER_VERSION: ${mongodb_version|} - COMPASS_E2E_TEST_GROUPS: ${e2e_test_groups} - COMPASS_E2E_TEST_GROUP: ${e2e_test_group} + COMPASS_E2E_TEST_GROUPS: ${e2e_test_groups|1} + COMPASS_E2E_TEST_GROUP: ${e2e_test_group|1} + COMPASS_E2E_TEST_FILTER: ${e2e_test_filter|} script: | set -e # Load environment variables @@ -720,8 +721,8 @@ functions: MONGODB_VERSION: ${mongodb_version|} MONGODB_RUNNER_VERSION: ${mongodb_version|} COMPASS_E2E_BROWSER_NAME: ${browser_name} - COMPASS_E2E_TEST_GROUPS: ${e2e_test_groups} - COMPASS_E2E_TEST_GROUP: ${e2e_test_group} + COMPASS_E2E_TEST_GROUPS: ${e2e_test_groups|1} + COMPASS_E2E_TEST_GROUP: ${e2e_test_group|1} script: | set -e # Load environment variables diff --git a/packages/compass-e2e-tests/helpers/test-runner-context.ts b/packages/compass-e2e-tests/helpers/test-runner-context.ts index b05e059ba32..e22a156109b 100644 --- a/packages/compass-e2e-tests/helpers/test-runner-context.ts +++ b/packages/compass-e2e-tests/helpers/test-runner-context.ts @@ -35,6 +35,7 @@ function buildCommonArgs(yargs: Argv) { type: 'string', description: 'Filter the spec files picked up for testing', default: '*', + coerce: (arg) => (arg === '' ? '*' : arg), }) .option('webdriver-waitfor-timeout', { type: 'number',