test: run e2e tests on pre-compiled packages

jbedard · alan-agius4 · commit 202313844c0c · 2022-09-26T10:44:49.000Z
The NPM packages being tested must be pre-compiled and the tar packages specified via --package. This way the real packages such as snapshots, release artifacts or cached packages can be tested. Previously the e2e tests compiled and packaged during test execution. (cherry picked from commit 2624d89)
diff --git a/.circleci/dynamic_config.yml b/.circleci/dynamic_config.yml
@@ -264,7 +264,7 @@ jobs:
         # Waits for the Saucelabs tunnel to be ready. This ensures that we don't run tests
         # too early without Saucelabs not being ready.
       - run: ./scripts/saucelabs/wait-for-tunnel.sh
-      - run: node ./tests/legacy-cli/run_e2e ./tests/legacy-cli/e2e/tests/misc/browsers.ts
+      - run: node ./tests/legacy-cli/run_e2e --glob="tests/misc/browsers.ts"
       - run: ./scripts/saucelabs/stop-tunnel.sh
       - fail_fast
 
@@ -273,6 +273,10 @@ jobs:
     steps:
       - custom_attach_workspace
       - run: yarn build
+      - persist_to_workspace:
+          root: *workspace_location
+          paths:
+            - dist/_*.tgz
 
   build-bazel-e2e:
     executor: action-executor
@@ -376,6 +380,8 @@ jobs:
             # Path where Arsenal Image Mounter files are downloaded.
             # Must match path in .circleci/win-ram-disk.ps1
             - ./aim
+      # Build the npm packages for the e2e tests
+      - run: yarn build
       # Run partial e2e suite on PRs only. Release branches will run the full e2e suite.
       - run:
           name: Execute E2E Tests
diff --git a/docs/DEVELOPER.md b/docs/DEVELOPER.md
@@ -82,8 +82,10 @@ You can find more info about debugging [tests with Bazel in the docs.](https://g
 
 ### End to end tests
 
-- Run: `node tests/legacy-cli/run_e2e.js`
+- Compile the packages being tested: `yarn build`
+- Run all tests: `node tests/legacy-cli/run_e2e.js`
 - Run a subset of the tests: `node tests/legacy-cli/run_e2e.js tests/legacy-cli/e2e/tests/i18n/ivy-localize-*`
+- Run on a custom set of npm packages (tar files): `node tests/legacy-cli/run_e2e.js --package _angular_cli.tgz _angular_create.tgz dist/*.tgz ...`
 
 When running the debug commands, Node will stop and wait for a debugger to attach.
 You can attach your IDE to the debugger to stop on breakpoints and step through the code. Also, see [IDE Specific Usage](#ide-specific-usage) for a
diff --git a/lib/BUILD.bazel b/lib/BUILD.bazel
diff --git a/package.json b/package.json
@@ -119,6 +119,7 @@
     "@types/progress": "^2.0.3",
     "@types/resolve": "^1.17.1",
     "@types/semver": "^7.3.12",
+    "@types/tar": "^6.1.2",
     "@types/text-table": "^0.2.1",
     "@types/uuid": "^8.0.0",
     "@types/yargs": "^17.0.8",
diff --git a/tests/legacy-cli/e2e/setup/010-local-publish.ts b/tests/legacy-cli/e2e/setup/010-local-publish.ts
@@ -1,26 +1,30 @@
 import { getGlobalVariable } from '../utils/env';
+import { PkgInfo } from '../utils/packages';
 import { globalNpm, extractNpmEnv } from '../utils/process';
 import { isPrereleaseCli } from '../utils/project';
 
 export default async function () {
   const testRegistry: string = getGlobalVariable('package-registry');
-  await globalNpm(
-    [
-      'run',
-      'admin',
-      '--',
-      'publish',
-      '--no-versionCheck',
-      '--no-branchCheck',
-      `--registry=${testRegistry}`,
-      '--tag',
-      isPrereleaseCli() ? 'next' : 'latest',
-    ],
-    {
-      ...extractNpmEnv(),
-      // Also set an auth token value for the local test registry which is required by npm 7+
-      // even though it is never actually used.
-      'NPM_CONFIG__AUTH': 'e2e-testing',
-    },
+  const packageTars: PkgInfo[] = Object.values(getGlobalVariable('package-tars'));
+
+  // Publish packages specified with --package
+  await Promise.all(
+    packageTars.map(({ path: p }) =>
+      globalNpm(
+        [
+          'publish',
+          `--registry=${testRegistry}`,
+          '--tag',
+          isPrereleaseCli() ? 'next' : 'latest',
+          p,
+        ],
+        {
+          ...extractNpmEnv(),
+          // Also set an auth token value for the local test registry which is required by npm 7+
+          // even though it is never actually used.
+          'NPM_CONFIG__AUTH': 'e2e-testing',
+        },
+      ),
+    ),
   );
 }
diff --git a/tests/legacy-cli/e2e/tests/update/update.ts b/tests/legacy-cli/e2e/tests/update/update.ts
@@ -4,7 +4,7 @@ import { createProjectFromAsset } from '../../utils/assets';
 import { expectFileMatchToExist, readFile } from '../../utils/fs';
 import { getActivePackageManager } from '../../utils/packages';
 import { ng, noSilentNg } from '../../utils/process';
-import { isPrereleaseCli, useCIChrome, useCIDefaults, NgCLIVersion } from '../../utils/project';
+import { isPrereleaseCli, useCIChrome, useCIDefaults, getNgCLIVersion } from '../../utils/project';
 
 export default async function () {
   let restoreRegistry: (() => Promise<void>) | undefined;
@@ -32,7 +32,7 @@ export default async function () {
     const cliMajorProjectVersion = new SemVer(cliVersion).major;
 
     // CLI current version.
-    const cliMajorVersion = NgCLIVersion.major;
+    const cliMajorVersion = getNgCLIVersion().major;
 
     for (let version = cliMajorProjectVersion + 1; version < cliMajorVersion; version++) {
       // Run all the migrations until the current build major version - 1.
diff --git a/tests/legacy-cli/e2e/utils/BUILD.bazel b/tests/legacy-cli/e2e/utils/BUILD.bazel
@@ -6,11 +6,11 @@ ts_library(
     srcs = glob(["**/*.ts"]),
     visibility = ["//visibility:public"],
     deps = [
-        "//lib",
         "//tests/legacy-cli/e2e/ng-snapshot",
         "@npm//@types/glob",
         "@npm//@types/node-fetch",
         "@npm//@types/semver",
+        "@npm//@types/tar",
         "@npm//@types/yargs-parser",
         "@npm//ansi-colors",
         "@npm//glob",
@@ -19,6 +19,7 @@ ts_library(
         "@npm//puppeteer",
         "@npm//rxjs",
         "@npm//semver",
+        "@npm//tar",
         "@npm//tree-kill",
         "@npm//verdaccio",
         "@npm//verdaccio-auth-memory",
diff --git a/tests/legacy-cli/e2e/utils/packages.ts b/tests/legacy-cli/e2e/utils/packages.ts
@@ -1,6 +1,12 @@
 import { getGlobalVariable } from './env';
 import { ProcessOutput, silentNpm, silentYarn } from './process';
 
+export interface PkgInfo {
+  readonly name: string;
+  readonly version: string;
+  readonly path: string;
+}
+
 export function getActivePackageManager(): 'npm' | 'yarn' {
   const value = getGlobalVariable('package-manager');
   if (value && value !== 'npm' && value !== 'yarn') {
diff --git a/tests/legacy-cli/e2e/utils/project.ts b/tests/legacy-cli/e2e/utils/project.ts
@@ -2,12 +2,11 @@ import * as fs from 'fs';
 import * as path from 'path';
 import { prerelease, SemVer } from 'semver';
 import yargsParser from 'yargs-parser';
-import { packages } from '../../../../lib/packages';
 import { getGlobalVariable } from './env';
 import { prependToFile, readFile, replaceInFile, writeFile } from './fs';
 import { gitCommit } from './git';
 import { findFreePort } from './network';
-import { installWorkspacePackages } from './packages';
+import { installWorkspacePackages, PkgInfo } from './packages';
 import { exec, execAndWaitForOutputToMatch, git, ng } from './process';
 
 export function updateJsonFile(filePath: string, fn: (json: any) => any | void) {
@@ -96,6 +95,8 @@ export async function prepareProjectForE2e(name: string) {
 }
 
 export function useBuiltPackagesVersions(): Promise<void> {
+  const packages: { [name: string]: PkgInfo } = getGlobalVariable('package-tars');
+
   return updateJsonFile('package.json', (json) => {
     json['dependencies'] ??= {};
     json['devDependencies'] ??= {};
@@ -221,8 +222,12 @@ export async function useCIChrome(projectDir: string = ''): Promise<void> {
   }
 }
 
-export const NgCLIVersion = new SemVer(packages['@angular/cli'].version);
+export function getNgCLIVersion(): SemVer {
+  const packages: { [name: string]: PkgInfo } = getGlobalVariable('package-tars');
+
+  return new SemVer(packages['@angular/cli'].version);
+}
 
 export function isPrereleaseCli(): boolean {
-  return (prerelease(NgCLIVersion)?.length ?? 0) > 0;
+  return (prerelease(getNgCLIVersion())?.length ?? 0) > 0;
 }
diff --git a/tests/legacy-cli/e2e/utils/tar.ts b/tests/legacy-cli/e2e/utils/tar.ts
@@ -0,0 +1,39 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import fs from 'fs';
+import { normalize } from 'path';
+import { Parse } from 'tar';
+
+/**
+ * Extract and return the contents of a single file out of a tar file.
+ *
+ * @param tarball the tar file to extract from
+ * @param filePath the path of the file to extract
+ * @returns the Buffer of file or an error on fs/tar error or file not found
+ */
+export async function extractFile(tarball: string, filePath: string): Promise<Buffer> {
+  return new Promise((resolve, reject) => {
+    fs.createReadStream(tarball)
+      .pipe(
+        new Parse({
+          strict: true,
+          filter: (p) => normalize(p) === normalize(filePath),
+          // TODO: @types/tar 'entry' does not have ReadEntry.on
+          onentry: (entry: any) => {
+            const chunks: Buffer[] = [];
+
+            entry.on('data', (chunk: any) => chunks!.push(chunk));
+            entry.on('error', reject);
+            entry.on('finish', () => resolve(Buffer.concat(chunks!)));
+          },
+        }),
+      )
+      .on('close', () => reject(`${tarball} does not contain ${filePath}`));
+  });
+}
diff --git a/tests/legacy-cli/e2e_runner.ts b/tests/legacy-cli/e2e_runner.ts
@@ -10,6 +10,9 @@ import { createNpmRegistry } from './e2e/utils/registry';
 import { launchTestProcess } from './e2e/utils/process';
 import { join } from 'path';
 import { findFreePort } from './e2e/utils/network';
+import { extractFile } from './e2e/utils/tar';
+import { realpathSync } from 'fs';
+import { PkgInfo } from './e2e/utils/packages';
 
 Error.stackTraceLimit = Infinity;
 
@@ -34,6 +37,8 @@ Error.stackTraceLimit = Infinity;
  *   --shard          Index of this processes' shard.
  *   --tmpdir=path    Override temporary directory to use for new projects.
  *   --yarn           Use yarn as package manager.
+ *   --package=path   An npm package to be published before running tests
+ *
  * If unnamed flags are passed in, the list of tests will be filtered to include only those passed.
  */
 const argv = yargsParser(process.argv.slice(2), {
@@ -49,10 +54,14 @@ const argv = yargsParser(process.argv.slice(2), {
   ],
   string: ['devkit', 'glob', 'ignore', 'reuse', 'ng-tag', 'tmpdir', 'ng-version'],
   number: ['nb-shards', 'shard'],
+  array: ['package'],
   configuration: {
     'dot-notation': false,
     'camel-case-expansion': false,
   },
+  default: {
+    'package': ['./dist/_*.tgz'],
+  },
 });
 
 /**
@@ -162,10 +171,11 @@ console.log(['Tests:', ...testsToRun].join('\n '));
 setGlobalVariable('argv', argv);
 setGlobalVariable('package-manager', argv.yarn ? 'yarn' : 'npm');
 
-Promise.all([findFreePort(), findFreePort()])
-  .then(async ([httpPort, httpsPort]) => {
+Promise.all([findFreePort(), findFreePort(), findPackageTars()])
+  .then(async ([httpPort, httpsPort, packageTars]) => {
     setGlobalVariable('package-registry', 'http://localhost:' + httpPort);
     setGlobalVariable('package-secure-registry', 'http://localhost:' + httpsPort);
+    setGlobalVariable('package-tars', packageTars);
 
     // NPM registries for the lifetime of the test execution
     const registryProcess = await createNpmRegistry(httpPort, httpPort);
@@ -308,3 +318,23 @@ function printFooter(testName: string, type: 'setup' | 'initializer' | 'test', s
   );
   console.log('');
 }
+
+// Collect the packages passed as arguments and return as {package-name => pkg-path}
+async function findPackageTars(): Promise<{ [pkg: string]: PkgInfo }> {
+  const pkgs: string[] = (getGlobalVariable('argv').package as string[]).flatMap((p) =>
+    glob.sync(p, { realpath: true }),
+  );
+
+  const pkgJsons = await Promise.all(pkgs.map((pkg) => extractFile(pkg, './package/package.json')));
+
+  return pkgs.reduce((all, pkg, i) => {
+    const json = pkgJsons[i].toString('utf8');
+    const { name, version } = JSON.parse(json);
+    if (!name) {
+      throw new Error(`Package ${pkg} - package.json name/version not found`);
+    }
+
+    all[name] = { path: realpathSync(pkg), name, version };
+    return all;
+  }, {} as { [pkg: string]: PkgInfo });
+}
diff --git a/yarn.lock b/yarn.lock
@@ -2446,6 +2446,14 @@
   resolved "https://registry.yarnpkg.com/@types/tapable/-/tapable-1.0.8.tgz#b94a4391c85666c7b73299fd3ad79d4faa435310"
   integrity sha512-ipixuVrh2OdNmauvtT51o3d8z12p6LtFW9in7U79der/kwejjdNchQC5UMn5u/KxNoM7VHHOs/l8KS8uHxhODQ==
 
+"@types/tar@^6.1.2":
+  version "6.1.2"
+  resolved "https://registry.yarnpkg.com/@types/tar/-/tar-6.1.2.tgz#e60108a7d1b08cc91bf2faf1286cc08fdad48bbe"
+  integrity sha512-bnX3RRm70/n1WMwmevdOAeDU4YP7f5JSubgnuU+yrO+xQQjwDboJj3u2NTJI5ngCQhXihqVVAH5h5J8YpdpEvg==
+  dependencies:
+    "@types/node" "*"
+    minipass "^3.3.5"
+
 "@types/text-table@^0.2.1":
   version "0.2.2"
   resolved "https://registry.yarnpkg.com/@types/text-table/-/text-table-0.2.2.tgz#774c90cfcfbc8b4b0ebb00fecbe861dc8b1e8e26"
@@ -7741,6 +7749,13 @@ minipass@^3.0.0, minipass@^3.1.1, minipass@^3.1.6:
   dependencies:
     yallist "^4.0.0"
 
+minipass@^3.3.5:
+  version "3.3.5"
+  resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.3.5.tgz#6da7e53a48db8a856eeb9153d85b230a2119e819"
+  integrity sha512-rQ/p+KfKBkeNwo04U15i+hOwoVBVmekmm/HcfTkTN2t9pbQKCMm4eN5gFeqgrrSp/kH/7BYYhTIHOxGqzbBPaA==
+  dependencies:
+    yallist "^4.0.0"
+
 minizlib@^2.1.1, minizlib@^2.1.2:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-2.1.2.tgz#e90d3466ba209b932451508a11ce3d3632145931"
