test: include amplify app in smoke tests (#13259)

* test: include amplify app in smoke tests suite

* chore: add some prints

* test: debug

* test: less tests

* test: generate

* chore: try this

* test: try this

* test: undo that

* chore: why

* chore: try that

* test: try this

* test: try this

* test: try this

* test: try this

* test: try this

* test: try this

* test: try this

* chore: this works

* test: try this

* test: try this

* Revert "test: try this"

This reverts commit 4762355.

* test: try this

Author: sobolk

.github/actions/run-smoke-tests/action.yml

@@ -70,5 +70,5 @@ runs:
       uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce #v3.1.2
       if: always()
       with:
-        name: test report
+        name: test report ${{ runner.os }}
         path: packages/amplify-e2e-tests/amplify-e2e-reports

.github/workflows/smoke-tests.yml

@@ -19,6 +19,7 @@ on:
 
 env:
   NODE_OPTIONS: --max-old-space-size=8096
+  IS_AMPLIFY_CLI_SMOKE_TEST_RUN: true
 
 jobs:
   run-smoke-tests:

codebuild_specs/e2e_workflow_generated.yml

@@ -138,6 +138,7 @@
   "l_schema_model_b_schema_model_d_auth_5f",
   "l_schema_predictions_model_migration_auth_3c",
   "l_searchable_migration",
+  "l_smoketest_amplify_app",
   "l_storage_1b_function_11_function_2b",
   "l_storage_5",
   "l_storage_migration_v10",

codebuild_specs/wait_for_ids.json

@@ -36,6 +36,11 @@ export * from './credentials-rotator';
  */
 export const isCI = (): boolean => JSON.parse(process.env.CI || 'false') && JSON.parse(process.env.CIRCLECI || 'false');
 
+/**
+ * Whether the current run is smoke test run.
+ */
+export const isSmokeTestRun = (): boolean => JSON.parse(process.env.IS_AMPLIFY_CLI_SMOKE_TEST_RUN || 'false');
+
 // eslint-disable-next-line spellcheck/spell-checker
 export const TEST_PROFILE_NAME = isCI() ? 'amplify-integ-test-user' : 'default';
 

packages/amplify-e2e-core/src/utils/index.ts

@@ -1,21 +1,11 @@
-import {
-  amplifyAppAndroid,
-  amplifyAppIos,
-  amplifyAppAngular,
-  amplifyAppReact,
-  amplifyModelgen,
-  amplifyPush,
-  addIntegAccountInConfig,
-} from '../amplify-app-helpers/amplify-app-setup';
-import { createNewProjectDir, deleteProject, deleteProjectDir, isCI } from '@aws-amplify/amplify-e2e-core';
+import { amplifyAppAndroid, amplifyAppIos, amplifyAppAngular } from '../amplify-app-helpers/amplify-app-setup';
+import { createNewProjectDir, deleteProjectDir, isCI } from '@aws-amplify/amplify-e2e-core';
 import { AmplifyFrontend } from '@aws-amplify/amplify-cli-core';
 import {
   validateProject,
   validateProjectConfig,
   validateApi,
   validateBackendConfig,
-  validateModelgen,
-  validateAmplifyPush,
   validateFeatureFlags,
 } from '../amplify-app-helpers/amplify-app-validation';
 
@@ -30,7 +20,7 @@ describe('amplify-app platform tests', () => {
     deleteProjectDir(projRoot);
   });
 
-  jest.setTimeout(1000 * 60 * 30); // 30 minutes is suffice as push operations are taking time
+  jest.setTimeout(1000 * 60 * 30); // 30 minutes is enough as push operations are taking time
 
   it('should set up an android project', async () => {
     await amplifyAppAndroid(projRoot);
@@ -64,19 +54,4 @@ describe('amplify-app platform tests', () => {
     validateBackendConfig(projRoot);
     validateFeatureFlags(projRoot);
   });
-
-  it('should set up a react project and run scripts', async () => {
-    await amplifyAppReact(projRoot);
-    validateProject(projRoot, AmplifyFrontend.javascript);
-    validateProjectConfig(projRoot, AmplifyFrontend.javascript, 'react');
-    validateApi(projRoot);
-    validateBackendConfig(projRoot);
-    validateFeatureFlags(projRoot);
-    addIntegAccountInConfig(projRoot);
-    await amplifyModelgen(projRoot);
-    validateModelgen(projRoot);
-    await amplifyPush(projRoot);
-    validateAmplifyPush(projRoot);
-    await deleteProject(projRoot);
-  });
 });

packages/amplify-e2e-tests/src/__tests__/amplify-app.test.ts

@@ -0,0 +1,50 @@
+import { amplifyAppReact, amplifyModelgen, amplifyPush, addIntegAccountInConfig } from '../../amplify-app-helpers/amplify-app-setup';
+import { createNewProjectDir, deleteProject, deleteProjectDir, getCLIPath } from '@aws-amplify/amplify-e2e-core';
+import { AmplifyFrontend } from '@aws-amplify/amplify-cli-core';
+import {
+  validateProject,
+  validateProjectConfig,
+  validateApi,
+  validateBackendConfig,
+  validateModelgen,
+  validateAmplifyPush,
+  validateFeatureFlags,
+} from '../../amplify-app-helpers/amplify-app-validation';
+import execa from 'execa';
+
+describe('amplify-app platform tests', () => {
+  let projRoot: string;
+
+  beforeAll(async () => {
+    const cliPath = getCLIPath();
+    const cliVersion = (await execa(cliPath, ['version'])).stdout;
+    // Print CLI path, CLI version and process.env.AMPLIFY_PATH for reference
+    // Code generated by amplify-app honors AMPLIFY_PATH env variable
+    console.log(`amplify-app tests: CLI path ${cliPath}, AMPLIFY_PATH=${process.env.AMPLIFY_PATH}, CLI version ${cliVersion}`);
+  });
+
+  beforeEach(async () => {
+    projRoot = await createNewProjectDir('amplify-app');
+  });
+
+  afterEach(() => {
+    deleteProjectDir(projRoot);
+  });
+
+  jest.setTimeout(1000 * 60 * 30); // 30 minutes is enough as push operations are taking time
+
+  it('should set up a react project and run scripts', async () => {
+    await amplifyAppReact(projRoot);
+    validateProject(projRoot, AmplifyFrontend.javascript);
+    validateProjectConfig(projRoot, AmplifyFrontend.javascript, 'react');
+    validateApi(projRoot);
+    validateBackendConfig(projRoot);
+    validateFeatureFlags(projRoot);
+    addIntegAccountInConfig(projRoot);
+    await amplifyModelgen(projRoot);
+    validateModelgen(projRoot);
+    await amplifyPush(projRoot);
+    validateAmplifyPush(projRoot);
+    await deleteProject(projRoot);
+  });
+});

packages/amplify-e2e-tests/src/__tests__/smoke-tests/smoketest-amplify-app.test.ts

@@ -1,14 +1,24 @@
-import { nspawn as spawn, KEY_DOWN_ARROW, isCI } from '@aws-amplify/amplify-e2e-core';
+import { nspawn as spawn, KEY_DOWN_ARROW, isCI, isSmokeTestRun } from '@aws-amplify/amplify-e2e-core';
 import * as fs from 'fs-extra';
 import * as path from 'path';
 
-const npm = /^win/.test(process.platform) ? 'npm.cmd' : 'npm';
+const isRunningOnWindows = /^win/.test(process.platform);
+const npm = isRunningOnWindows ? 'npm.cmd' : 'npm';
+const npx = isRunningOnWindows ? 'npx.cmd' : 'npx';
 const amplifyAppBinPath = path.join(__dirname, '..', '..', '..', 'amplify-app', 'bin', 'amplify-app');
-const spawnCommand = isCI() ? 'amplify-app' : amplifyAppBinPath;
+const getSpawnCommand = () => {
+  if (isSmokeTestRun()) {
+    return [npx, 'amplify-app', '--yes'];
+  } else if (isCI() && !isRunningOnWindows) {
+    return 'amplify-app';
+  } else {
+    return amplifyAppBinPath;
+  }
+};
 
 function amplifyAppAndroid(projRoot: string): Promise<void> {
   return new Promise((resolve, reject) => {
-    spawn(spawnCommand, ['--platform', 'android'], { cwd: projRoot, stripColors: true })
+    spawn(getSpawnCommand(), ['--platform', 'android'], { cwd: projRoot, stripColors: true })
       .wait('Successfully created base Amplify Project')
       .wait('Amplify setup completed successfully')
       .run(function (err) {
@@ -23,7 +33,7 @@ function amplifyAppAndroid(projRoot: string): Promise<void> {
 
 function amplifyAppIos(projRoot: string): Promise<void> {
   return new Promise((resolve, reject) => {
-    spawn(spawnCommand, ['--platform', 'ios'], { cwd: projRoot, stripColors: true })
+    spawn(getSpawnCommand(), ['--platform', 'ios'], { cwd: projRoot, stripColors: true })
       .wait('Successfully created base Amplify Project')
       .wait('Amplify setup completed successfully')
       .run(function (err) {
@@ -38,7 +48,7 @@ function amplifyAppIos(projRoot: string): Promise<void> {
 
 function amplifyAppAngular(projRoot: string): Promise<void> {
   return new Promise((resolve, reject) => {
-    spawn(spawnCommand, [], { cwd: projRoot, stripColors: true })
+    spawn(getSpawnCommand(), [], { cwd: projRoot, stripColors: true })
       .wait('What type of app are you building')
       .sendCarriageReturn()
       .wait('What javascript framework are you using')
@@ -54,8 +64,24 @@ function amplifyAppAngular(projRoot: string): Promise<void> {
 }
 
 function amplifyAppReact(projRoot: string): Promise<void> {
+  const env: Record<string, string> = {};
+  if (isSmokeTestRun()) {
+    // If we're smoke testing we have to prepend a directory component of AMPLIFY_PATH to PATH
+    // Internally amplify-app spawns 'amplify' which makes OS look into PATH
+    // However, yarn injects local binaries into PATH as well which makes OS find packages/amplify-cli/bin content
+    // and packages are not fully built in smoke tests.
+    // OS traverses PATH from left to right, so prepending forces it to use AMPLIFY_PATH location.
+    if (!process.env.AMPLIFY_PATH) {
+      throw new Error('AMPLIFY_PATH must be set in smoke tests');
+    }
+    const amplifyPathDir = path.parse(process.env.AMPLIFY_PATH).dir;
+    let pathEnvVar = process.env.PATH;
+    const separator = isRunningOnWindows ? ';' : ':';
+    pathEnvVar = amplifyPathDir + separator + pathEnvVar;
+    env['PATH'] = pathEnvVar;
+  }
   return new Promise((resolve, reject) => {
-    spawn(spawnCommand, [], { cwd: projRoot, stripColors: true })
+    spawn(getSpawnCommand(), [], { cwd: projRoot, stripColors: true, env })
       .wait('What type of app are you building')
       .sendCarriageReturn()
       .wait('What javascript framework are you using')

packages/amplify-e2e-tests/src/amplify-app-helpers/amplify-app-setup.ts

@@ -2,6 +2,8 @@ import * as fs from 'fs-extra';
 import * as path from 'path';
 import { AmplifyFrontend, pathManager } from '@aws-amplify/amplify-cli-core';
 
+const npm = /^win/.test(process.platform) ? 'npm.cmd' : 'npm';
+
 function validateProject(projRoot: string, platform: AmplifyFrontend) {
   expect(fs.existsSync(path.join(projRoot, 'amplify'))).toBe(true);
   expect(fs.existsSync(path.join(projRoot, '.gitignore'))).toBe(true);
@@ -46,7 +48,7 @@ function validateProjectConfig(projRoot: string, platform: AmplifyFrontend, fram
           break;
         case 'react':
           expect(config['javascript']['framework']).toBe('react');
-          expect(config['javascript']['config']['StartCommand']).toBe('npm run-script start');
+          expect(config['javascript']['config']['StartCommand']).toBe(`${npm} run-script start`);
           break;
         default:
           expect(false).toBe(true);

packages/amplify-e2e-tests/src/amplify-app-helpers/amplify-app-validation.ts

@@ -43,7 +43,11 @@ const RUN_SOLO = [
   'src/__tests__/transformer-migrations/searchable-migration.test.ts',
   'src/__tests__/uibuilder.test.ts',
 ];
-const DISABLE_COVERAGE = ['src/__tests__/datastore-modelgen.test.ts', 'src/__tests__/amplify-app.test.ts'];
+const DISABLE_COVERAGE = [
+  'src/__tests__/datastore-modelgen.test.ts',
+  'src/__tests__/amplify-app.test.ts',
+  'src/__tests__/smoke-tests/smoketest-amplify-app.test.ts',
+];
 const TEST_EXCLUSIONS: { l: string[]; w: string[] } = {
   l: [],
   w: [
@@ -55,6 +59,7 @@ const TEST_EXCLUSIONS: { l: string[]; w: string[] } = {
     'src/__tests__/opensearch-simulator/opensearch-simulator.test.ts',
     'src/__tests__/storage-simulator/S3server.test.ts',
     'src/__tests__/amplify-app.test.ts',
+    'src/__tests__/smoke-tests/smoketest-amplify-app.test.ts',
     // failing in parsing JSON strings on powershell
     'src/__tests__/auth_2g.test.ts',
     'src/__tests__/auth_12.test.ts',