test: use karton for e2e tests (#627)

Use external package to handle e2e tests to simplify code base

Author: piotr-oles

.eslintignore

@@ -1 +1,2 @@
 /lib/**
+/test/e2e/fixtures/**

.github/workflows/main.yml

@@ -5,7 +5,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v1
-      
+
       - name: Setup node
         uses: actions/setup-node@v1
         with:
@@ -44,7 +44,7 @@ jobs:
         os: [ubuntu-latest, macos-latest, windows-latest]
     steps:
     - uses: actions/checkout@v1
-      
+
     - name: Setup node
       uses: actions/setup-node@v1
       with:
@@ -61,7 +61,13 @@ jobs:
         key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
         restore-keys: |
           ${{ runner.os }}-yarn-
-      
+
+    - name: Locks cache
+      uses: actions/cache@v2
+      with:
+        path: test/e2e/__locks__
+        key: ${{ runner.os }}-locks
+
     - name: Install dependencies
       run: yarn install --frozen-lockfile
 
@@ -76,7 +82,7 @@ jobs:
 
     - name: Run e2e tests
       run: yarn test:e2e
-      
+
   release:
     runs-on: ubuntu-latest
     env:
@@ -86,7 +92,7 @@ jobs:
     if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/alpha' || github.ref == 'refs/heads/beta')
     steps:
       - uses: actions/checkout@v1
-      
+
       - name: Setup node
         uses: actions/setup-node@v1
         with:

.gitignore

@@ -7,6 +7,9 @@ lib
 # Package archive used by e2e tests
 fork-ts-checker-webpack-plugin-0.0.0-semantic-release.tgz
 
+# E2E tests lock file cache dir
+__locks__
+
 # Coverage directory used by tools like istanbul
 coverage
 

package.json

@@ -105,6 +105,7 @@
     "jest": "^26.5.3",
     "jest-circus": "^26.5.3",
     "jest-environment-node": "^26.5.2",
+    "karton": "^0.4.1",
     "lint-staged": "^10.4.2",
     "mock-fs": "^4.13.0",
     "prettier": "^2.1.2",

test/e2e/OutOfMemoryAndCosmiconfig.spec.ts

@@ -1,51 +1,23 @@
-import { join } from 'path';
-import { createSandbox, Sandbox } from './sandbox/Sandbox';
-import { readFixture } from './sandbox/Fixture';
-import { createGenericProcessDriver } from './sandbox/GenericProcessDriver';
-import { FORK_TS_CHECKER_WEBPACK_PLUGIN_VERSION } from './sandbox/Plugin';
+import path from 'path';
+import { createProcessDriver } from 'karton';
 
 describe('ForkTsCheckerWebpackPlugin Out Of Memory and Cosmiconfig', () => {
-  let sandbox: Sandbox;
-
-  beforeAll(async () => {
-    sandbox = await createSandbox();
-  });
-
-  beforeEach(async () => {
-    await sandbox.reset();
-  });
-
-  afterAll(async () => {
-    await sandbox.cleanup();
-  });
-
   it.each([
     { async: false, webpack: '4.0.0' },
     { async: true, webpack: '^4.0.0' },
     { async: false, webpack: '^5.0.0' },
     { async: true, webpack: '^5.0.0' },
   ])('handles out of memory for %p', async ({ async, webpack }) => {
-    await sandbox.load([
-      await readFixture(join(__dirname, 'fixtures/environment/typescript-basic.fixture'), {
-        FORK_TS_CHECKER_WEBPACK_PLUGIN_VERSION: JSON.stringify(
-          FORK_TS_CHECKER_WEBPACK_PLUGIN_VERSION
-        ),
-        TS_LOADER_VERSION: JSON.stringify('^5.0.0'),
-        TYPESCRIPT_VERSION: JSON.stringify('~3.8.0'),
-        WEBPACK_VERSION: JSON.stringify(webpack),
-        WEBPACK_CLI_VERSION: JSON.stringify('^3.3.11'),
-        WEBPACK_DEV_SERVER_VERSION: JSON.stringify('^3.10.3'),
-        ASYNC: JSON.stringify(async),
-      }),
-      await readFixture(join(__dirname, 'fixtures/implementation/typescript-basic.fixture')),
-    ]);
+    await sandbox.load(path.join(__dirname, 'fixtures/typescript-basic'));
+    await sandbox.install('yarn', { webpack });
+    await sandbox.patch('webpack.config.js', 'async: false,', `async: ${JSON.stringify(async)},`);
 
     await sandbox.write(
       'fork-ts-checker.config.js',
       `module.exports = { typescript: { memoryLimit: 10 } };`
     );
 
-    const driver = createGenericProcessDriver(sandbox.spawn('npm run webpack-dev-server'));
+    const driver = createProcessDriver(sandbox.spawn('npm run webpack-dev-server'));
 
     // we should see an error message about out of memory
     await driver.waitForStderrIncludes(

test/e2e/TypeDefinitions.spec.ts

@@ -1,31 +1,9 @@
-import { join } from 'path';
-import { createSandbox, Sandbox } from './sandbox/Sandbox';
-import { readFixture } from './sandbox/Fixture';
-import { FORK_TS_CHECKER_WEBPACK_PLUGIN_VERSION } from './sandbox/Plugin';
+import path from 'path';
 
 describe('Type Definitions', () => {
-  let sandbox: Sandbox;
-
-  beforeAll(async () => {
-    sandbox = await createSandbox();
-  });
-
-  beforeEach(async () => {
-    await sandbox.reset();
-  });
-
-  afterAll(async () => {
-    await sandbox.cleanup();
-  });
-
   it('provides valid type definitions', async () => {
-    await sandbox.load(
-      await readFixture(join(__dirname, 'fixtures/type-definitions.fixture'), {
-        FORK_TS_CHECKER_WEBPACK_PLUGIN_VERSION: JSON.stringify(
-          FORK_TS_CHECKER_WEBPACK_PLUGIN_VERSION
-        ),
-      })
-    );
+    await sandbox.load(path.join(__dirname, 'fixtures/type-definitions'));
+    await sandbox.install('yarn', {});
 
     expect(await sandbox.exec('npm run tsc').catch((error) => error)).toContain(
       "webpack.config.ts(7,7): error TS2322: Type 'string' is not assignable to type 'boolean | undefined'."

test/e2e/TypeScriptConfiguration.spec.ts

@@ -0,0 +1,73 @@
+import path from 'path';
+import {
+  createWebpackDevServerDriver,
+  WebpackDevServerDriver,
+} from './driver/WebpackDevServerDriver';
+
+describe('TypeScript Configuration', () => {
+  it.each([
+    { async: true, webpack: '~4.0.0', typescript: '2.7.1', 'ts-loader': '^5.0.0' },
+    { async: false, webpack: '^4.0.0', typescript: '~3.0.0', 'ts-loader': '^6.0.0' },
+    { async: true, webpack: '^5.0.0', typescript: '~3.7.0', 'ts-loader': '^7.0.0' },
+    { async: false, webpack: '^5.0.0', typescript: '~3.8.0', 'ts-loader': '^6.0.0' },
+  ])(
+    'change in the tsconfig.json affects compilation for %p',
+    async ({ async, ...dependencies }) => {
+      await sandbox.load(path.join(__dirname, 'fixtures/typescript-basic'));
+      await sandbox.install('yarn', { ...dependencies });
+      await sandbox.patch('webpack.config.js', 'async: false,', `async: ${JSON.stringify(async)},`);
+
+      const driver = createWebpackDevServerDriver(
+        sandbox.spawn('npm run webpack-dev-server'),
+        async
+      );
+
+      // first compilation is successful
+      await driver.waitForNoErrors();
+
+      // change available libraries
+      await sandbox.patch('tsconfig.json', '"lib": ["ES6", "DOM"]', '"lib": ["ES6"],');
+
+      const errors = await driver.waitForErrors();
+      expect(errors.length).toBeGreaterThan(0);
+
+      // revert the change
+      await sandbox.patch('tsconfig.json', '"lib": ["ES6"],', '"lib": ["DOM", "ES6"]');
+
+      await driver.waitForNoErrors();
+    }
+  );
+
+  it.each([
+    { typescript: '2.7.1' },
+    { typescript: '~3.0.0' },
+    { typescript: '~3.6.0' },
+    { typescript: '^3.8.0' },
+  ])('reports errors because of the misconfiguration', async (dependencies) => {
+    await sandbox.load(path.join(__dirname, 'fixtures/typescript-basic'));
+    await sandbox.install('yarn', { ...dependencies });
+
+    let driver: WebpackDevServerDriver;
+    let errors: string[];
+
+    await sandbox.write(
+      'fork-ts-checker.config.js',
+      'module.exports = { typescript: { configOverwrite: { compilerOptions: { target: "ES3", lib: ["ES3"] } } } };'
+    );
+
+    driver = createWebpackDevServerDriver(sandbox.spawn('npm run webpack-dev-server'), false);
+    errors = await driver.waitForErrors();
+    expect(errors.length).toBeGreaterThan(0);
+    await sandbox.kill(driver.process);
+
+    await sandbox.write(
+      'fork-ts-checker.config.js',
+      'module.exports = { typescript: { configOverwrite: { include: [] } } };'
+    );
+
+    driver = createWebpackDevServerDriver(sandbox.spawn('npm run webpack-dev-server'), false);
+    errors = await driver.waitForErrors();
+    expect(errors.length).toBeGreaterThan(0);
+    await sandbox.kill(driver.process);
+  });
+});