ci: e2e tests on windows (#750)

Author: Dane Pilcher

.codebuild/e2e_workflow.yml

@@ -63,7 +63,7 @@ batch:
       depend-on:
         - publish_to_local_registry
     - identifier: >-
-        add_codegen_ios_configure_codegen_android_configure_codegen_js_graphql_codegen_android
+        l_add_codegen_ios_configure_codegen_android_configure_codegen_js_graphql_codegen_android
       buildspec: .codebuild/run_e2e_tests.yml
       env:
         compute-type: BUILD_GENERAL1_LARGE
@@ -74,7 +74,7 @@ batch:
       depend-on:
         - publish_to_local_registry
     - identifier: >-
-        graphql_codegen_js_remove_codegen_android_remove_codegen_ios_add_codegen_android
+        l_graphql_codegen_js_remove_codegen_android_remove_codegen_ios_add_codegen_android
       buildspec: .codebuild/run_e2e_tests.yml
       env:
         compute-type: BUILD_GENERAL1_LARGE
@@ -85,7 +85,7 @@ batch:
       depend-on:
         - publish_to_local_registry
     - identifier: >-
-        configure_codegen_ios_datastore_modelgen_android_datastore_modelgen_js_feature_flags
+        l_configure_codegen_ios_datastore_modelgen_android_datastore_modelgen_js_feature_flags
       buildspec: .codebuild/run_e2e_tests.yml
       env:
         compute-type: BUILD_GENERAL1_LARGE
@@ -96,7 +96,7 @@ batch:
       depend-on:
         - publish_to_local_registry
     - identifier: >-
-        graphql_codegen_ios_add_codegen_js_datastore_modelgen_ios_remove_codegen_js
+        l_graphql_codegen_ios_add_codegen_js_datastore_modelgen_ios_remove_codegen_js
       buildspec: .codebuild/run_e2e_tests.yml
       env:
         compute-type: BUILD_GENERAL1_LARGE
@@ -107,7 +107,7 @@ batch:
       depend-on:
         - publish_to_local_registry
     - identifier: >-
-        datastore_modelgen_flutter_env_codegen_model_introspection_codegen_pull_codegen
+        l_datastore_modelgen_flutter_env_codegen_model_introspection_codegen_pull_codegen
       buildspec: .codebuild/run_e2e_tests.yml
       env:
         compute-type: BUILD_GENERAL1_LARGE
@@ -118,7 +118,7 @@ batch:
       depend-on:
         - publish_to_local_registry
     - identifier: >-
-        push_codegen_ios_push_codegen_android_graphql_documents_generator_push_codegen_js
+        l_push_codegen_ios_push_codegen_android_graphql_documents_generator_push_codegen_js
       buildspec: .codebuild/run_e2e_tests.yml
       env:
         compute-type: BUILD_GENERAL1_LARGE
@@ -129,7 +129,7 @@ batch:
       depend-on:
         - publish_to_local_registry
     - identifier: >-
-        build_app_ts_uninitialized_project_codegen_js_uninitialized_project_modelgen_android_uninitialized_project_modelgen_flutter
+        l_build_app_ts_uninitialized_project_codegen_js_uninitialized_project_modelgen_android_uninitialized_project_modelgen_flutter
       buildspec: .codebuild/run_e2e_tests.yml
       env:
         compute-type: BUILD_GENERAL1_LARGE
@@ -139,7 +139,7 @@ batch:
           CLI_REGION: ap-southeast-1
       depend-on:
         - publish_to_local_registry
-    - identifier: uninitialized_project_modelgen_ios_uninitialized_project_modelgen_js
+    - identifier: l_uninitialized_project_modelgen_ios_uninitialized_project_modelgen_js
       buildspec: .codebuild/run_e2e_tests.yml
       env:
         compute-type: BUILD_GENERAL1_LARGE
@@ -149,10 +149,121 @@ batch:
           CLI_REGION: ap-southeast-2
       depend-on:
         - publish_to_local_registry
+    - identifier: >-
+        w_add_codegen_ios_configure_codegen_android_configure_codegen_js_graphql_codegen_android
+      buildspec: .codebuild/run_e2e_tests_windows.yml
+      env:
+        compute-type: BUILD_GENERAL1_LARGE
+        image: $WINDOWS_IMAGE_2019
+        type: WINDOWS_SERVER_2019_CONTAINER
+        variables:
+          TEST_SUITE: >-
+            src/__tests__/add-codegen-ios.test.ts|src/__tests__/configure-codegen-android.test.ts|src/__tests__/configure-codegen-js.test.ts|src/__tests__/graphql-codegen-android.test.ts
+          CLI_REGION: us-east-1
+      depend-on:
+        - publish_to_local_registry
+        - build_windows
+    - identifier: >-
+        w_graphql_codegen_js_remove_codegen_android_remove_codegen_ios_add_codegen_android
+      buildspec: .codebuild/run_e2e_tests_windows.yml
+      env:
+        compute-type: BUILD_GENERAL1_LARGE
+        image: $WINDOWS_IMAGE_2019
+        type: WINDOWS_SERVER_2019_CONTAINER
+        variables:
+          TEST_SUITE: >-
+            src/__tests__/graphql-codegen-js.test.ts|src/__tests__/remove-codegen-android.test.ts|src/__tests__/remove-codegen-ios.test.ts|src/__tests__/add-codegen-android.test.ts
+          CLI_REGION: us-east-1
+      depend-on:
+        - publish_to_local_registry
+        - build_windows
+    - identifier: >-
+        w_configure_codegen_ios_datastore_modelgen_android_datastore_modelgen_js_feature_flags
+      buildspec: .codebuild/run_e2e_tests_windows.yml
+      env:
+        compute-type: BUILD_GENERAL1_LARGE
+        image: $WINDOWS_IMAGE_2019
+        type: WINDOWS_SERVER_2019_CONTAINER
+        variables:
+          TEST_SUITE: >-
+            src/__tests__/configure-codegen-ios.test.ts|src/__tests__/datastore-modelgen-android.test.ts|src/__tests__/datastore-modelgen-js.test.ts|src/__tests__/feature-flags.test.ts
+          CLI_REGION: us-east-1
+      depend-on:
+        - publish_to_local_registry
+        - build_windows
+    - identifier: >-
+        w_graphql_codegen_ios_add_codegen_js_datastore_modelgen_ios_remove_codegen_js
+      buildspec: .codebuild/run_e2e_tests_windows.yml
+      env:
+        compute-type: BUILD_GENERAL1_LARGE
+        image: $WINDOWS_IMAGE_2019
+        type: WINDOWS_SERVER_2019_CONTAINER
+        variables:
+          TEST_SUITE: >-
+            src/__tests__/graphql-codegen-ios.test.ts|src/__tests__/add-codegen-js.test.ts|src/__tests__/datastore-modelgen-ios.test.ts|src/__tests__/remove-codegen-js.test.ts
+          CLI_REGION: us-east-1
+      depend-on:
+        - publish_to_local_registry
+        - build_windows
+    - identifier: >-
+        w_datastore_modelgen_flutter_env_codegen_model_introspection_codegen_pull_codegen
+      buildspec: .codebuild/run_e2e_tests_windows.yml
+      env:
+        compute-type: BUILD_GENERAL1_LARGE
+        image: $WINDOWS_IMAGE_2019
+        type: WINDOWS_SERVER_2019_CONTAINER
+        variables:
+          TEST_SUITE: >-
+            src/__tests__/datastore-modelgen-flutter.test.ts|src/__tests__/env-codegen.test.ts|src/__tests__/model-introspection-codegen.test.ts|src/__tests__/pull-codegen.test.ts
+          CLI_REGION: us-east-1
+      depend-on:
+        - publish_to_local_registry
+        - build_windows
+    - identifier: >-
+        w_push_codegen_ios_push_codegen_android_graphql_documents_generator_push_codegen_js
+      buildspec: .codebuild/run_e2e_tests_windows.yml
+      env:
+        compute-type: BUILD_GENERAL1_LARGE
+        image: $WINDOWS_IMAGE_2019
+        type: WINDOWS_SERVER_2019_CONTAINER
+        variables:
+          TEST_SUITE: >-
+            src/__tests__/push-codegen-ios.test.ts|src/__tests__/push-codegen-android.test.ts|src/__tests__/graphql-documents-generator.test.ts|src/__tests__/push-codegen-js.test.ts
+          CLI_REGION: us-east-1
+      depend-on:
+        - publish_to_local_registry
+        - build_windows
+    - identifier: >-
+        w_build_app_ts_uninitialized_project_codegen_js_uninitialized_project_modelgen_android_uninitialized_project_modelgen_flutter
+      buildspec: .codebuild/run_e2e_tests_windows.yml
+      env:
+        compute-type: BUILD_GENERAL1_LARGE
+        image: $WINDOWS_IMAGE_2019
+        type: WINDOWS_SERVER_2019_CONTAINER
+        variables:
+          TEST_SUITE: >-
+            src/__tests__/build-app-ts.test.ts|src/__tests__/uninitialized-project-codegen-js.test.ts|src/__tests__/uninitialized-project-modelgen-android.test.ts|src/__tests__/uninitialized-project-modelgen-flutter.test.ts
+          CLI_REGION: us-east-1
+      depend-on:
+        - publish_to_local_registry
+        - build_windows
+    - identifier: w_uninitialized_project_modelgen_ios_uninitialized_project_modelgen_js
+      buildspec: .codebuild/run_e2e_tests_windows.yml
+      env:
+        compute-type: BUILD_GENERAL1_LARGE
+        image: $WINDOWS_IMAGE_2019
+        type: WINDOWS_SERVER_2019_CONTAINER
+        variables:
+          TEST_SUITE: >-
+            src/__tests__/uninitialized-project-modelgen-ios.test.ts|src/__tests__/uninitialized-project-modelgen-js.test.ts
+          CLI_REGION: us-east-1
+      depend-on:
+        - publish_to_local_registry
+        - build_windows
     - identifier: cleanup_e2e_resources
       buildspec: .codebuild/cleanup_e2e_resources.yml
       env:
         compute-type: BUILD_GENERAL1_MEDIUM
       depend-on:
         - >-
-          add_codegen_ios_configure_codegen_android_configure_codegen_js_graphql_codegen_android
+          l_add_codegen_ios_configure_codegen_android_configure_codegen_js_graphql_codegen_android

.codebuild/run_e2e_tests_windows.yml

@@ -0,0 +1,24 @@
+version: 0.2
+env:
+  shell: powershell.exe
+  variables:
+    AMPLIFY_DIR: /root/.npm-global/lib/node_modules/@aws-amplify/cli-internal/bin
+    AMPLIFY_PATH: /root/.npm-global/lib/node_modules/@aws-amplify/cli-internal/bin/amplify
+    CI: true
+    CODEBUILD: true
+    NODE_OPTIONS: --max-old-space-size=8096
+    SKIP_SET_NPM_PREFIX: true
+phases:
+  build:
+    commands:
+      # commands need to be run in stand-alone bash scripts so that bash can be used on windows
+      - bash ./.codebuild/scripts/e2e_test_windows.sh
+  post_build:
+    commands:
+      # commands need to be run in stand-alone bash scripts so that bash can be used on windows
+      - bash ./.codebuild/scripts/post_e2e_test.sh
+
+artifacts:
+  files:
+    - '**/*'
+  base-directory: $Env:CODEBUILD_SRC_DIR\packages\amplify-codegen-e2e-tests\amplify-e2e-reports

.codebuild/scripts/e2e_test_windows.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# set exit on error to true
+set -e
+
+source ./shared-scripts.sh && _setupE2ETestsWindows
+codebuild-breakpoint
+source ./shared-scripts.sh && _runE2ETestsWindows

.codebuild/scripts/local_publish_helpers.sh

@@ -7,7 +7,7 @@ function startLocalRegistry {
     # Start local registry
     tmp_registry_log="$(mktemp)"
     echo "Registry output file: $tmp_registry_log"
-    (cd && nohup npx ${VERDACCIO_PACKAGE:-$default_verdaccio_package} -c $1 &>$tmp_registry_log &)
+    nohup npx ${VERDACCIO_PACKAGE:-$default_verdaccio_package} -c $1 &>$tmp_registry_log &
     # Wait for Verdaccio to boot
     grep -q 'http address' <(tail -f $tmp_registry_log)
 }

.codebuild/scripts/post_e2e_test.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# set exit on error to true
+set -e
+
+source ./shared-scripts.sh && _unassumeTestAccountCredentials
+aws sts get-caller-identity
+source ./shared-scripts.sh && _scanArtifacts

packages/amplify-codegen-e2e-core/package.json

@@ -32,7 +32,7 @@
     "jest-circus": "^27.5.1",
     "jest-environment-node": "^26.6.2",
     "lodash": "^4.17.19",
-    "node-pty": "beta",
+    "node-pty": "1.0.0",
     "retimer": "2.0.0",
     "rimraf": "^3.0.0",
     "strip-ansi": "^6.0.0",

packages/amplify-codegen-e2e-core/src/categories/api.ts

@@ -49,9 +49,7 @@ export function addApiWithoutSchema(cwd: string, opts: Partial<AddApiOptions & {
       .sendCarriageReturn()
       .wait('Do you want to edit the schema now?')
       .sendConfirmNo()
-      .wait(
-        '"amplify publish" will build all your local backend and frontend resources (if you have hosting category added) and provision it in the cloud',
-      )
+      .wait('"amplify publish" will build all your local backend and frontend resources')
       .run((err: Error) => {
         if (!err) {
           resolve();
@@ -79,10 +77,8 @@ export function addApiWithBlankSchema(cwd: string, opts: Partial<AddApiOptions &
       .sendKeyDown(2)
       .sendCarriageReturn()
       .wait('Do you want to edit the schema now?')
-      .sendLine('n')
-      .wait(
-        '"amplify publish" will build all your local backend and frontend resources (if you have hosting category added) and provision it in the cloud',
-      )
+      .sendConfirmNo()
+      .wait('"amplify publish" will build all your local backend and frontend resources')
       .sendEof()
       .run((err: Error) => {
         if (!err) {
@@ -112,10 +108,8 @@ export function addApiWithBlankSchemaAndConflictDetection(cwd: string) {
       .sendKeyDown(2)
       .sendCarriageReturn()
       .wait('Do you want to edit the schema now?')
-      .sendLine('n')
-      .wait(
-        '"amplify publish" will build all your local backend and frontend resources (if you have hosting category added) and provision it in the cloud',
-      )
+      .sendConfirmNo()
+      .wait('"amplify publish" will build all your local backend and frontend resources')
       .run((err: Error) => {
         if (!err) {
           resolve();

packages/amplify-codegen-e2e-core/src/index.ts

@@ -5,6 +5,7 @@ import { spawnSync, execSync } from 'child_process';
 import { v4 as uuid } from 'uuid';
 import * as ini from 'ini';
 import { pathManager } from '@aws-amplify/amplify-cli-core';
+import { getCommandPath } from './utils';
 
 export * from './configure/';
 export * from './init/';
@@ -24,14 +25,16 @@ declare global {
 const amplifyTestsDir = 'amplify-codegen-e2e-tests';
 
 export function getCLIPath(testingWithLatestCodebase = false) {
+  let commandName = 'amplify-dev';
   if (isCI() && !testingWithLatestCodebase) {
-    return 'amplify';
+    commandName = 'amplify';
   }
-  return 'amplify-dev';
+
+  return getCommandPath(commandName);
 }
 
 export function isCI(): boolean {
-  return process.env.CI && (process.env.CODEBUILD) ? true : false;
+  return process.env.CI && process.env.CODEBUILD ? true : false;
 }
 
 export function injectSessionToken(profileName: string) {
@@ -57,7 +60,9 @@ export async function createNewProjectDir(
   projectName: string,
   prefix = path.join(fs.realpathSync(os.tmpdir()), amplifyTestsDir),
 ): Promise<string> {
-  const currentHash = execSync('git rev-parse --short HEAD', { cwd: __dirname }).toString().trim();
+  const currentHash = execSync('git rev-parse --short HEAD', { cwd: __dirname })
+    .toString()
+    .trim();
   let projectDir;
   do {
     const randomId = await global.getRandomId();

packages/amplify-codegen-e2e-core/src/init/cra.ts

@@ -1,5 +1,5 @@
 /* commands for a Create React App */
-import { nspawn as spawn } from '..';
+import { nspawn as spawn, getCommandPath } from '..';
 
 const defaultSettings = {
   disableCIDetection: false,
@@ -8,7 +8,7 @@ const defaultSettings = {
 export function craInstall(cwd: string, settings: Object = {}): Promise<void> {
   return new Promise((resolve, reject) => {
     const s = { ...defaultSettings, ...settings };
-    const chain = spawn('npm', ['install'], { cwd, stripColors: true, disableCIDetection: s.disableCIDetection });
+    const chain = spawn(getCommandPath('npm'), ['install'], { cwd, stripColors: true, disableCIDetection: s.disableCIDetection });
 
     chain.run((err: Error) => {
       if (err) {
@@ -23,7 +23,7 @@ export function craInstall(cwd: string, settings: Object = {}): Promise<void> {
 export function craBuild(cwd: string, settings: Object = {}): Promise<void> {
   return new Promise((resolve, reject) => {
     const s = { ...defaultSettings, ...settings };
-    const chain = spawn('npm', ['run', 'build'], { cwd, stripColors: true, disableCIDetection: s.disableCIDetection });
+    const chain = spawn(getCommandPath('npm'), ['run', 'build'], { cwd, stripColors: true, disableCIDetection: s.disableCIDetection });
 
     chain.run((err: Error) => {
       if (err) {
@@ -41,8 +41,8 @@ export function cypressRun(cwd: string, settings: any = {}): Promise<void> {
     const args = ['cypress', 'run'];
     if (s?.componentsTesting) {
       args.push('--component');
-    };
-    const chain = spawn('npx', args, { cwd, stripColors: true, disableCIDetection: s.disableCIDetection });
+    }
+    const chain = spawn(getCommandPath('npx'), args, { cwd, stripColors: true, disableCIDetection: s.disableCIDetection });
 
     chain.run((err: Error) => {
       if (err) {

packages/amplify-codegen-e2e-core/src/utils/getCommandPath.ts

@@ -0,0 +1,10 @@
+import { execSync } from 'child_process';
+/* Get the full absolute path of a command with `which`
+ *
+ * Windows executors on CB fail to spawn a command with custom nexpect unless the full path is used.
+ */
+export function getCommandPath(command: string): string {
+  return execSync(`which ${command}`)
+    .toString()
+    .trim();
+}