test(linter/plugins): automate testing all fixtures (#15911)

Remove the need to list every fixture in `e2e.test.ts` file by automatically running tests on every fixture in `test/fixtures` directory.

Running Oxlint with `--fix` and running fixtures with ESLint is now controlled by adding an `options.json` file to the fixture directory contining e.g. `{ "fix": true }` or `{ "eslint": true }`.

Now we can add a new fixture just by creating a new directory in `fixtures`.

Author: overlookmotel

apps/oxlint/test/e2e.test.ts

@@ -1,265 +1,59 @@
-// oxlint-disable jest/expect-expect
-
-import fs from 'node:fs/promises';
 import { join as pathJoin } from 'node:path';
 import { describe, it } from 'vitest';
-import { FIXTURES_DIR_PATH, PACKAGE_ROOT_PATH, testFixtureWithCommand } from './utils.js';
+import { PACKAGE_ROOT_PATH, getFixtures, testFixtureWithCommand } from './utils.js';
+
+import type { Fixture } from './utils.ts';
 
 const CLI_PATH = pathJoin(PACKAGE_ROOT_PATH, 'dist/cli.js');
 
-// Options to pass to `testFixture`.
-interface TestOptions {
-  // Arguments to pass to the CLI.
-  args?: string[];
-  // Name of the snapshot file.
-  // Defaults to `output`.
-  // Supply a different name when there are multiple tests for a single fixture.
-  snapshotName?: string;
-  // Function to get extra data to include in the snapshot
-  getExtraSnapshotData?: (dirPath: string) => Promise<Record<string, string>>;
-}
+// Use current NodeJS executable, rather than `node`, to avoid problems with a Node version manager
+// installed on system resulting in using wrong NodeJS version
+const NODE_BIN_PATH = process.execPath;
 
 /**
- * Run a test fixture.
- * @param fixtureName - Name of the fixture directory within `test/fixtures`
- * @param options - Options to customize the test (optional)
+ * Run Oxlint tests for all fixtures in `test/fixtures`.
+ *
+ * Oxlint is run with:
+ * - CWD set to the fixture directory.
+ * - `files` as the only argument (so only lints the files in the fixture's `files` directory).
+ *
+ * Fixtures with an `options.json` file containing `"fix": true` are also run with `--fix` CLI option.
+ * The files' contents after fixes are recorded in the snapshot.
+ *
+ * Fixtures with an `options.json` file containing `"oxlint": false` are skipped.
  */
-async function testFixture(fixtureName: string, options?: TestOptions): Promise<void> {
-  const args = options?.args ?? [];
-
-  await testFixtureWithCommand({
-    // Use current NodeJS executable, rather than `node`, to avoid problems with a Node version manager
-    // installed on system resulting in using wrong NodeJS version
-    command: process.execPath,
-    args: [CLI_PATH, ...args, 'files'],
-    fixtureName,
-    snapshotName: options?.snapshotName ?? 'output',
-    getExtraSnapshotData: options?.getExtraSnapshotData,
-    isESLint: false,
-  });
-}
-
 describe('oxlint CLI', () => {
-  it('should lint a directory without errors', async () => {
-    await testFixture('built_in_no_errors');
-  });
-
-  it('should lint a directory with errors', async () => {
-    await testFixture('built_in_errors');
-  });
-
-  it('should load a custom plugin', async () => {
-    await testFixture('basic_custom_plugin');
-  });
-
-  it('should support message placeholder interpolation', async () => {
-    await testFixture('message_interpolation');
-  });
-
-  it('should support messageId', async () => {
-    await testFixture('message_id_plugin');
-  });
-
-  it('should support messageId placeholder interpolation', async () => {
-    await testFixture('message_id_interpolation');
-  });
-
-  it('should report an error for unknown messageId', async () => {
-    await testFixture('message_id_error');
-  });
-
-  it('should load a custom plugin with various import styles', async () => {
-    await testFixture('load_paths');
-  });
-
-  it('should load a custom plugin with multiple files', async () => {
-    await testFixture('basic_custom_plugin_many_files');
-  });
-
-  it('should load a custom plugin correctly when extending in a nested config', async () => {
-    await testFixture('custom_plugin_nested_config');
-  });
-  it('should do something', async () => {
-    await testFixture('custom_plugin_nested_config_duplicate');
-  });
-
-  it('should load a custom plugin when configured in overrides', async () => {
-    await testFixture('custom_plugin_via_overrides');
-  });
-
-  it('should report an error if a custom plugin is missing', async () => {
-    await testFixture('missing_custom_plugin');
-  });
-
-  it('should report an error if a custom plugin has a reserved name', async () => {
-    await testFixture('reserved_name');
-  });
-
-  it('should report an error if a custom plugin throws an error during import', async () => {
-    await testFixture('custom_plugin_import_error');
-  });
-
-  it('should report an error if a rule is not found within a custom plugin', async () => {
-    await testFixture('custom_plugin_missing_rule');
-  });
-
-  it('should report an error if a a rule is not found within a custom plugin (via overrides)', async () => {
-    await testFixture('custom_plugin_via_overrides_missing_rule');
-  });
-
-  describe('should report an error if a custom plugin throws an error during linting', () => {
-    it('in `create` method', async () => {
-      await testFixture('custom_plugin_lint_create_error');
-    });
-
-    it('in `createOnce` method', async () => {
-      await testFixture('custom_plugin_lint_createOnce_error');
-    });
-
-    it('in visit function', async () => {
-      await testFixture('custom_plugin_lint_visit_error');
-    });
-
-    it('in `before` hook', async () => {
-      await testFixture('custom_plugin_lint_before_hook_error');
-    });
-
-    it('in `after` hook', async () => {
-      await testFixture('custom_plugin_lint_after_hook_error');
-    });
-
-    it('in `fix` function', async () => {
-      await testFixture('custom_plugin_lint_fix_error');
-    });
-  });
-
-  it('should report the correct severity when using a custom plugin', async () => {
-    await testFixture('basic_custom_plugin_warn_severity');
-  });
+  const fixtures = getFixtures();
+  for (const fixture of fixtures) {
+    if (!fixture.options.oxlint) continue;
 
-  it('should work with multiple rules', async () => {
-    await testFixture('basic_custom_plugin_multiple_rules');
-  });
-
-  it('should support reporting diagnostic with `loc`', async () => {
-    await testFixture('diagnostic_loc');
-  });
-
-  it('should receive ESTree-compatible AST', async () => {
-    await testFixture('estree');
-  });
-
-  it('should receive AST with all nodes having `parent` property', async () => {
-    await testFixture('parent');
-  });
-
-  it('should receive data via `context`', async () => {
-    await testFixture('context_properties');
-  });
-
-  it('should give access to source code via `context.sourceCode`', async () => {
-    await testFixture('sourceCode');
-  });
-
-  it('should give access to settings via `context.settings`', async () => {
-    await testFixture('settings');
-  });
-
-  it('should get source text and AST from `context.sourceCode` when accessed late', async () => {
-    await testFixture('sourceCode_late_access');
-  });
-
-  it('should get source text and AST from `context.sourceCode` when accessed in `after` hook only', async () => {
-    await testFixture('sourceCode_late_access_after_only');
-  });
-
-  it('should support scopeManager', async () => {
-    await testFixture('scope_manager');
-  });
-
-  it('should support scope helper methods in `context.sourceCode`', async () => {
-    await testFixture('sourceCode_scope_methods');
-  });
-
-  it('should support languageOptions', async () => {
-    await testFixture('languageOptions');
-  });
-
-  it('should support selectors', async () => {
-    await testFixture('selector');
-  });
-
-  it('should support `createOnce`', async () => {
-    await testFixture('createOnce');
-  });
-
-  it('should support `definePlugin`', async () => {
-    await testFixture('definePlugin');
-  });
-
-  it('should support `defineRule`', async () => {
-    await testFixture('defineRule');
-  });
-
-  it('should support `definePlugin` and `defineRule` together', async () => {
-    await testFixture('definePlugin_and_defineRule');
-  });
-
-  it('should have UTF-16 spans in AST', async () => {
-    await testFixture('utf16_offsets');
-  });
+    // oxlint-disable-next-line jest/expect-expect
+    it(`fixture: ${fixture.name}`, () => runFixture(fixture));
+  }
+});
 
-  it('should respect disable directives for custom plugin rules', async () => {
-    await testFixture('custom_plugin_disable_directives');
+/**
+ * Run Oxlint on a test fixture.
+ * @param fixture - Fixture object
+ */
+async function runFixture(fixture: Fixture): Promise<void> {
+  // Run Oxlint without `--fix` option
+  await testFixtureWithCommand({
+    command: NODE_BIN_PATH,
+    args: [CLI_PATH, 'files'],
+    fixture,
+    snapshotName: 'output',
+    isESLint: false,
   });
 
-  it('should not apply fixes when `--fix` is disabled', async () => {
-    await testFixture('fixes', {
-      snapshotName: 'fixes_disabled',
-      async getExtraSnapshotData(fixtureDirPath) {
-        const fixtureFilePath = pathJoin(fixtureDirPath, 'files/index.js');
-        const codeAfter = await fs.readFile(fixtureFilePath, 'utf8');
-        return { 'Code after': codeAfter };
-      },
+  // Run Oxlint with `--fix` option
+  if (fixture.options.fix) {
+    await testFixtureWithCommand({
+      command: NODE_BIN_PATH,
+      args: [CLI_PATH, '--fix', 'files'],
+      fixture,
+      snapshotName: 'fix',
+      isESLint: false,
     });
-  });
-
-  it('should apply fixes when `--fix` is enabled', async () => {
-    const fixtureFilePath = pathJoin(FIXTURES_DIR_PATH, 'fixes/files/index.js');
-    const codeBefore = await fs.readFile(fixtureFilePath, 'utf8');
-
-    try {
-      await testFixture('fixes', {
-        args: ['--fix'],
-        snapshotName: 'fixes_enabled',
-        async getExtraSnapshotData() {
-          const codeAfter = await fs.readFile(fixtureFilePath, 'utf8');
-          return { 'Code after': codeAfter };
-        },
-      });
-    } finally {
-      // Revert fixture file code changes
-      await fs.writeFile(fixtureFilePath, codeBefore);
-    }
-  });
-
-  it('should support comments-related APIs in `context.sourceCode`', async () => {
-    await testFixture('comments');
-  });
-
-  it('should support UTF16 characters in source code and comments with correct spans', async () => {
-    await testFixture('unicode_comments');
-  });
-
-  it('should return empty object for `parserServices` without throwing', async () => {
-    await testFixture('parser_services');
-  });
-
-  it('wrapping context should work', async () => {
-    await testFixture('context_wrapping');
-  });
-
-  it('should support `isSpaceBetween` in `context.sourceCode`', async () => {
-    await testFixture('isSpaceBetween');
-  });
-});
+  }
+}

apps/oxlint/test/eslint-compat.test.ts

@@ -1,37 +1,39 @@
-// oxlint-disable jest/expect-expect
-
 import { join as pathJoin } from 'node:path';
 import { describe, it } from 'vitest';
-import { testFixtureWithCommand } from './utils.js';
+import { PACKAGE_ROOT_PATH, getFixtures, testFixtureWithCommand } from './utils.js';
+
+import type { Fixture } from './utils.ts';
+
+const ESLINT_PATH = pathJoin(PACKAGE_ROOT_PATH, 'node_modules/.bin/eslint');
+
+/**
+ * Run ESLint tests for all fixtures in `test/fixtures` which contain an `options.json` file
+ * containing `"eslint": true`.
+ *
+ * ESLint is run with CWD set to the fixture directory.
+ */
+// These tests take longer than 5 seconds on CI, so increase timeout to 10 seconds
+// oxlint-disable-next-line jest/valid-describe-callback
+describe('ESLint compatibility', { timeout: 10_000 }, () => {
+  const fixtures = getFixtures();
+  for (const fixture of fixtures) {
+    if (!fixture.options.eslint) continue;
 
-const ESLINT_PATH = pathJoin(import.meta.dirname, '../node_modules/.bin/eslint');
+    // oxlint-disable-next-line jest/expect-expect
+    it(`fixture: ${fixture.name}`, () => runFixture(fixture));
+  }
+});
 
 /**
  * Run ESLint on a test fixture.
- * @param fixtureName - Name of the fixture directory within `test/fixtures`
+ * @param fixture - Fixture object
  */
-async function testFixture(fixtureName: string): Promise<void> {
+async function runFixture(fixture: Fixture): Promise<void> {
   await testFixtureWithCommand({
     command: ESLINT_PATH,
     args: [],
-    fixtureName,
+    fixture,
     snapshotName: 'eslint',
     isESLint: true,
   });
 }
-
-// These tests take longer than 5 seconds on CI, so increase timeout to 10 seconds
-// oxlint-disable-next-line jest/valid-describe-callback
-describe('ESLint compatibility', { timeout: 10_000 }, () => {
-  it('`definePlugin` should work', async () => {
-    await testFixture('definePlugin');
-  });
-
-  it('`defineRule` should work', async () => {
-    await testFixture('defineRule');
-  });
-
-  it('`definePlugin` and `defineRule` together should work', async () => {
-    await testFixture('definePlugin_and_defineRule');
-  });
-});

apps/oxlint/test/fixtures/definePlugin/options.json

@@ -0,0 +1,3 @@
+{
+  "eslint": true
+}

apps/oxlint/test/fixtures/definePlugin_and_defineRule/options.json

@@ -0,0 +1,3 @@
+{
+  "eslint": true
+}

apps/oxlint/test/fixtures/defineRule/options.json

@@ -0,0 +1,3 @@
+{
+  "eslint": true
+}

apps/oxlint/test/fixtures/fixes/fixes_enabled.snap.md renamed to apps/oxlint/test/fixtures/fixes/fix.snap.md

@@ -13,7 +13,7 @@ WARNING: JS plugins are experimental and not subject to semver.
 Breaking changes are possible while JS plugins support is under development.
 ```
 
-# Code after
+# File altered: files/index.js
 ```
 
 

apps/oxlint/test/fixtures/fixes/options.json

@@ -0,0 +1,3 @@
+{
+  "fix": true
+}