chore: add retry for e2e tests to improve dx (#520)

piotr-oles · web-flow · commit 544bd5603bcb · 2020-10-18T21:44:02.000+02:00
diff --git a/package.json b/package.json
@@ -34,8 +34,8 @@
     "build": "cross-env rimraf lib && cross-env tsc --version && cross-env tsc",
     "lint": "cross-env eslint ./src ./test --ext .ts",
     "test": "yarn build && yarn test:unit && yarn test:e2e",
-    "test:unit": "cross-env jest unit",
-    "test:e2e": "npm pack && cross-env jest e2e --ci --runInBand --bail --verbose",
+    "test:unit": "cross-env jest --config=test/unit/jest.config.js",
+    "test:e2e": "npm pack && cross-env jest --config=test/e2e/jest.config.js --ci -i -b",
     "precommit": "cross-env lint-staged && yarn build && yarn test:unit",
     "commit": "cross-env git-cz",
     "semantic-release": "semantic-release"
@@ -73,43 +73,45 @@
     "tapable": "^1.0.0"
   },
   "devDependencies": {
-    "@commitlint/config-conventional": "^8.3.4",
+    "@commitlint/config-conventional": "^11.0.0",
     "@semantic-release/commit-analyzer": "^8.0.1",
     "@semantic-release/exec": "^5.0.0",
-    "@semantic-release/github": "^7.0.7",
-    "@semantic-release/npm": "^7.0.5",
+    "@semantic-release/github": "^7.1.1",
+    "@semantic-release/npm": "^7.0.6",
     "@semantic-release/release-notes-generator": "^9.0.1",
-    "@types/babel__code-frame": "^7.0.1",
+    "@types/babel__code-frame": "^7.0.2",
     "@types/cross-spawn": "^6.0.2",
-    "@types/eslint": "^6.8.0",
-    "@types/fs-extra": "^8.1.0",
-    "@types/jest": "^25.2.1",
+    "@types/eslint": "^7.2.4",
+    "@types/fs-extra": "^9.0.2",
+    "@types/jest": "^26.0.14",
     "@types/minimatch": "^3.0.1",
-    "@types/mock-fs": "^4.10.0",
-    "@types/node": "^13.11.1",
+    "@types/mock-fs": "^4.13.0",
+    "@types/node": "^14.11.10",
     "@types/rimraf": "^3.0.0",
-    "@types/semver": "^7.1.0",
-    "@types/webpack": "^4.41.11",
+    "@types/semver": "^7.3.4",
+    "@types/webpack": "^4.41.22",
     "@typescript-eslint/eslint-plugin": "^2.27.0",
     "@typescript-eslint/parser": "^2.27.0",
-    "commitlint": "^8.3.5",
+    "commitlint": "^11.0.0",
     "cross-env": "^7.0.2",
-    "cross-spawn": "^7.0.2",
+    "cross-spawn": "^7.0.3",
     "eslint": "^6.8.0",
-    "eslint-config-prettier": "^6.10.1",
+    "eslint-config-prettier": "^6.13.0",
     "eslint-plugin-node": "^11.1.0",
-    "eslint-plugin-prettier": "^3.1.3",
-    "git-cz": "^4.3.1",
-    "husky": "^4.2.5",
-    "jest": "^25.3.0",
-    "lint-staged": "^10.1.3",
-    "mock-fs": "^4.11.0",
-    "prettier": "^2.0.4",
+    "eslint-plugin-prettier": "^3.1.4",
+    "git-cz": "^4.7.1",
+    "husky": "^4.3.0",
+    "jest": "^26.5.3",
+    "jest-circus": "^26.5.3",
+    "jest-environment-node": "^26.5.2",
+    "lint-staged": "^10.4.2",
+    "mock-fs": "^4.13.0",
+    "prettier": "^2.1.2",
     "rimraf": "^3.0.2",
-    "semantic-release": "^17.0.0",
+    "semantic-release": "^17.2.1",
     "strip-ansi": "^6.0.0",
     "tree-kill": "^1.2.2",
-    "ts-jest": "^25.3.1",
+    "ts-jest": "^26.4.1",
     "typescript": "^3.8.3",
     "webpack": "^4.42.1"
   },
diff --git a/src/typescript-reporter/TypeScriptReporterConfiguration.ts b/src/typescript-reporter/TypeScriptReporterConfiguration.ts
@@ -1,6 +1,5 @@
 import webpack from 'webpack';
 import path from 'path';
-import * as ts from 'typescript';
 import semver from 'semver';
 import { TypeScriptDiagnosticsOptions } from './TypeScriptDiagnosticsOptions';
 import { TypeScriptReporterOptions } from './TypeScriptReporterOptions';
diff --git a/src/typescript-reporter/file-system/RealFileSystem.ts b/src/typescript-reporter/file-system/RealFileSystem.ts
@@ -51,7 +51,10 @@ function createRealFileSystem(caseSensitive = false): FileSystem {
       const stats = readStats(normalizedPath);
 
       if (stats && stats.isFile()) {
-        readFileCache.set(normalizedPath, fs.readFileSync(normalizedPath, { encoding }).toString());
+        readFileCache.set(
+          normalizedPath,
+          fs.readFileSync(normalizedPath, { encoding: encoding as BufferEncoding }).toString()
+        );
       } else {
         readFileCache.set(normalizedPath, undefined);
       }
diff --git a/test/e2e/EsLint.spec.ts b/test/e2e/EsLint.spec.ts
@@ -25,8 +25,8 @@ describe('EsLint', () => {
   it.each([
     { async: false, webpack: '4.0.0', absolute: false },
     { async: true, webpack: '^4.0.0', absolute: true },
-    { async: false, webpack: '^5.0.0-beta.16', absolute: true },
-    { async: true, webpack: '^5.0.0-beta.16', absolute: false },
+    { async: false, webpack: '^5.0.0', absolute: true },
+    { async: true, webpack: '^5.0.0', absolute: false },
   ])('reports lint error for %p', async ({ async, webpack, absolute }) => {
     await sandbox.load([
       await readFixture(join(__dirname, 'fixtures/environment/eslint-basic.fixture'), {
diff --git a/test/e2e/OutOfMemoryAndCosmiconfig.spec.ts b/test/e2e/OutOfMemoryAndCosmiconfig.spec.ts
@@ -21,8 +21,8 @@ describe('ForkTsCheckerWebpackPlugin Out Of Memory and Cosmiconfig', () => {
   it.each([
     { async: false, webpack: '4.0.0' },
     { async: true, webpack: '^4.0.0' },
-    { async: false, webpack: '^5.0.0-beta.16' },
-    { async: true, webpack: '^5.0.0-beta.16' },
+    { async: false, webpack: '^5.0.0' },
+    { async: true, webpack: '^5.0.0' },
   ])('handles out of memory for %p', async ({ async, webpack }) => {
     await sandbox.load([
       await readFixture(join(__dirname, 'fixtures/environment/typescript-basic.fixture'), {
diff --git a/test/e2e/TypeScriptConfigurationChange.spec.ts b/test/e2e/TypeScriptConfigurationChange.spec.ts
@@ -25,8 +25,8 @@ describe('TypeScript Configuration Change', () => {
   it.each([
     { async: true, webpack: '~4.0.0', typescript: '2.7.1', tsloader: '^5.0.0' },
     { async: false, webpack: '^4.0.0', typescript: '~3.0.0', tsloader: '^6.0.0' },
-    { async: true, webpack: '^5.0.0-beta.16', typescript: '~3.7.0', tsloader: '^7.0.0' },
-    { async: false, webpack: '^5.0.0-beta.16', typescript: '~3.8.0', tsloader: '^6.0.0' },
+    { async: true, webpack: '^5.0.0', typescript: '~3.7.0', tsloader: '^7.0.0' },
+    { async: false, webpack: '^5.0.0', typescript: '~3.8.0', tsloader: '^6.0.0' },
   ])(
     'change in the tsconfig.json affects compilation for %p',
     async ({ async, webpack, typescript, tsloader }) => {
diff --git a/test/e2e/TypeScriptWatchApi.spec.ts b/test/e2e/TypeScriptWatchApi.spec.ts
@@ -25,8 +25,8 @@ describe('TypeScript Watch API', () => {
   it.each([
     { async: false, webpack: '4.0.0' },
     { async: true, webpack: '^4.0.0' },
-    { async: false, webpack: '^5.0.0-beta.16' },
-    { async: true, webpack: '^5.0.0-beta.16' },
+    { async: false, webpack: '^5.0.0' },
+    { async: true, webpack: '^5.0.0' },
   ])(
     'reports semantic error for %p with importsNotUsedAsValues configuration with ts-loader',
     async ({ async, webpack }) => {
@@ -152,8 +152,8 @@ describe('TypeScript Watch API', () => {
   it.each([
     { async: false, webpack: '4.0.0' },
     { async: true, webpack: '^4.0.0' },
-    { async: false, webpack: '^5.0.0-beta.16' },
-    { async: true, webpack: '^5.0.0-beta.16' },
+    { async: false, webpack: '^5.0.0' },
+    { async: true, webpack: '^5.0.0' },
   ])(
     'reports semantic error for %p with onlyRemoveTypeImports configuration with babel-loader',
     async ({ async, webpack }) => {
diff --git a/test/e2e/WebpackIssueScope.spec.ts b/test/e2e/WebpackIssueScope.spec.ts
@@ -27,8 +27,8 @@ describe('Webpack Issue Scope', () => {
     { webpack: '4.0.0', async: false, scope: 'all' },
     { webpack: '^4.0.0', async: false, scope: 'webpack' },
     { webpack: '^4.0.0', async: true, scope: 'all' },
-    { webpack: '^5.0.0-beta.16', async: true, scope: 'webpack' },
-    { webpack: '^5.0.0-beta.16', async: false, scope: 'all' },
+    { webpack: '^5.0.0', async: true, scope: 'webpack' },
+    { webpack: '^5.0.0', async: false, scope: 'all' },
   ])(
     'reports errors only related to the given scope with %p',
     async ({ webpack, async, scope }) => {
diff --git a/test/e2e/WebpackNodeApi.spec.ts b/test/e2e/WebpackNodeApi.spec.ts
@@ -18,7 +18,7 @@ describe('Webpack Node Api', () => {
     await sandbox.cleanup();
   });
 
-  it.each([{ webpack: '4.0.0' }, { webpack: '^4.0.0' }, { webpack: '^5.0.0-beta.16' }])(
+  it.each([{ webpack: '4.0.0' }, { webpack: '^4.0.0' }, { webpack: '^5.0.0' }])(
     'compiles the project successfully with %p',
     async ({ webpack }) => {
       await sandbox.load([
diff --git a/test/e2e/WebpackProductionBuild.spec.ts b/test/e2e/WebpackProductionBuild.spec.ts
@@ -20,7 +20,7 @@ describe('Webpack Production Build', () => {
     await sandbox.cleanup();
   });
 
-  it.each([{ webpack: '4.0.0' }, { webpack: '^4.0.0' }, { webpack: '^5.0.0-beta.16' }])(
+  it.each([{ webpack: '4.0.0' }, { webpack: '^4.0.0' }, { webpack: '^5.0.0' }])(
     'compiles the project successfully with %p',
     async ({ webpack }) => {
       await sandbox.load([
@@ -65,7 +65,7 @@ describe('Webpack Production Build', () => {
     }
   );
 
-  it.each([{ webpack: '4.0.0' }, { webpack: '^4.0.0' }, { webpack: '^5.0.0-beta.16' }])(
+  it.each([{ webpack: '4.0.0' }, { webpack: '^4.0.0' }, { webpack: '^5.0.0' }])(
     'exits with error on the project error with %p',
     async ({ webpack }) => {
       await sandbox.load([
diff --git a/test/e2e/jest.config.js b/test/e2e/jest.config.js
@@ -0,0 +1,17 @@
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: './jest.environment.js',
+  testRunner: 'jest-circus/runner',
+  testTimeout: 300000,
+  verbose: true,
+  setupFilesAfterEnv: ['./jest.setup.js'],
+  rootDir: '.',
+  moduleNameMapper: {
+    '^lib/(.*)$': '<rootDir>/../../lib/$1',
+  },
+  globals: {
+    'ts-jest': {
+      tsConfig: '<rootDir>/../tsconfig.json',
+    },
+  },
+};
diff --git a/test/e2e/jest.environment.js b/test/e2e/jest.environment.js
@@ -0,0 +1,31 @@
+const NodeEnvironment = require('jest-environment-node');
+
+function printBlock(message) {
+  process.stdout.write('┏━' + '━'.repeat(message.length) + '━┓\n');
+  process.stdout.write('┃ ' + message + ' ┃\n');
+  process.stdout.write('┗━' + '━'.repeat(message.length) + '━┛\n');
+}
+
+class E2EEnvironment extends NodeEnvironment {
+  constructor(config) {
+    super(config);
+  }
+
+  async handleTestEvent(event, state) {
+    switch (event.name) {
+      case 'test_start':
+        printBlock(`Test Start: ${event.test.name}`);
+        break;
+
+      case 'test_retry':
+        printBlock(`Test Retry: ${event.test.name}`);
+        break;
+
+      case 'test_done':
+        printBlock(`Test Done: ${event.test.name} (${Math.round(event.test.duration / 1000)}s)`);
+        break;
+    }
+  }
+}
+
+module.exports = E2EEnvironment;
diff --git a/test/e2e/jest.setup.js b/test/e2e/jest.setup.js
@@ -0,0 +1 @@
+jest.retryTimes(5);
diff --git a/test/e2e/sandbox/Logger.ts b/test/e2e/sandbox/Logger.ts
@@ -0,0 +1,19 @@
+import chalk from 'chalk';
+
+interface Logger {
+  log: (message: string) => void;
+}
+
+const logger: Logger = {
+  log(message) {
+    process.stdout.write(
+      message
+        .split('\n')
+        .map((line) => chalk.grey(`$ ${line}`))
+        .join('\n')
+    );
+    process.stdout.write('\n');
+  },
+};
+
+export { Logger, logger };
diff --git a/test/e2e/sandbox/Sandbox.ts b/test/e2e/sandbox/Sandbox.ts
diff --git a/test/setup.ts b/test/setup.ts
diff --git a/test/unit/jest.config.js b/test/unit/jest.config.js
diff --git a/yarn.lock b/yarn.lock
