ci: add e2e tests to CI

.github/workflows/main.yml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   test:
-    name: Node.js ${{ matrix.node-version }} on ${{ matrix.os }}
+    name: Test / Node.js ${{ matrix.node-version }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -67,3 +67,46 @@ jobs:
           node-version: 'lts/*'
       - run: npm install
       - run: npm run typecheck
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 'lts/*'
+      - run: npm install
+      - run: npm run build
+      - name: Upload build results
+        uses: actions/upload-artifact@v4
+        with:
+          name: build_dist
+          path: dist
+  e2e:
+    name: E2E / Node.js ${{ matrix.node-version }} on ${{ matrix.os }}
+    needs: build
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        node-version:
+          - '24.0.0' # minimum supported v24
+          - 24
+          - '22.13.1' # minimum supported v22
+          - 22
+          - '20.19.0' #minimum supported v20
+          - 20
+        os:
+          - ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: Download build results
+        uses: actions/download-artifact@v4
+        with:
+          name: build_dist
+          path: dist
+      - run: npm install
+      - run: npm run e2e

.npmpackagejsonlintignore

@@ -0,0 +1 @@
+e2e/

e2e/all-typed-config/.npmrc

@@ -0,0 +1 @@
+package-lock = false

e2e/all-typed-config/eslint.config.ts

@@ -0,0 +1,10 @@
+import { defineConfig } from 'eslint/config';
+
+import eslintPlugin from 'eslint-plugin-eslint-plugin';
+
+export default defineConfig([
+  {
+    extends: [eslintPlugin.configs.all],
+    files: ['./index.js', './rule.js'],
+  },
+]);

e2e/all-typed-config/index.js

@@ -0,0 +1,25 @@
+import rule from './rule.js';
+
+/** @type {import('eslint').ESLint.Plugin} */
+const plugin = {
+  meta: {
+    name: '@e2e/all-typed-config',
+    version: '1.0.0',
+  },
+  rules: { 'e2e-test': rule },
+  configs: {
+    recommended: {
+      name: '@e2e/all-typed-config/recommended',
+      plugins: {
+        get ['@e2e/all-typed-config']() {
+          return plugin;
+        },
+      },
+      rules: {
+        '@e2e/all-typed-config/e2e-test': 'error',
+      },
+    },
+  },
+};
+
+export default plugin;

e2e/all-typed-config/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "@e2e/all-typed-config",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "lint": "tsc && eslint"
+  },
+  "main": "./index.js",
+  "devDependencies": {
+    "eslint": "^9.33.0",
+    "eslint-plugin-eslint-plugin": "file:../..",
+    "jiti": "^2.5.1",
+    "typescript": "^5.9.2"
+  }
+}

e2e/all-typed-config/rule.js

@@ -0,0 +1,32 @@
+/** @type {import('eslint').Rule.RuleModule} */
+const rule = {
+  meta: {
+    type: 'suggestion',
+    docs: {
+      description: 'enforce a test',
+      recommended: false,
+      url: 'https://test.org',
+    },
+    fixable: undefined, // or "code" or "whitespace"
+    schema: [],
+    messages: {
+      missingOutput: 'This is a test',
+    },
+  },
+
+  create(context) {
+    return {
+      Program(ast) {
+        const sourceCode = context.sourceCode;
+        if (false) {
+          context.report({
+            node: ast,
+            messageId: 'missingOutput',
+          });
+        }
+      },
+    };
+  },
+};
+
+export default rule;

e2e/all-typed-config/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "rootDir": ".",
+    "module": "nodenext",
+    "moduleResolution": "nodenext",
+    "noEmit": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "target": "ES2024",
+    "verbatimModuleSyntax": true,
+    "erasableSyntaxOnly": true,
+    "types": []
+  },
+  "include": ["eslint.config.ts"]
+}

e2e/all/.npmrc

@@ -0,0 +1 @@
+package-lock = false

e2e/all/eslint.config.js

@@ -0,0 +1,10 @@
+import { defineConfig } from 'eslint/config';
+
+import eslintPlugin from 'eslint-plugin-eslint-plugin';
+
+export default defineConfig([
+  {
+    extends: [eslintPlugin.configs.all],
+    files: ['./index.js', './rule.js'],
+  },
+]);

e2e/all/index.js

@@ -0,0 +1,25 @@
+import rule from './rule.js';
+
+/** @type {import('eslint').ESLint.Plugin} */
+const plugin = {
+  meta: {
+    name: '@e2e/all',
+    version: '1.0.0',
+  },
+  rules: { 'e2e-test': rule },
+  configs: {
+    recommended: {
+      name: '@e2e/all/recommended',
+      plugins: {
+        get ['@e2e/all']() {
+          return plugin;
+        },
+      },
+      rules: {
+        '@e2e/all/e2e-test': 'error',
+      },
+    },
+  },
+};
+
+export default plugin;

e2e/all/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "@e2e/all",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "lint": "eslint"
+  },
+  "main": "./index.js",
+  "devDependencies": {
+    "eslint": "^9.33.0",
+    "eslint-plugin-eslint-plugin": "file:../.."
+  }
+}

e2e/all/rule.js

@@ -0,0 +1,32 @@
+/** @type {import('eslint').Rule.RuleModule} */
+const rule = {
+  meta: {
+    type: 'suggestion',
+    docs: {
+      description: 'enforce a test',
+      recommended: false,
+      url: 'https://test.org',
+    },
+    fixable: undefined, // or "code" or "whitespace"
+    schema: [],
+    messages: {
+      missingOutput: 'This is a test',
+    },
+  },
+
+  create(context) {
+    return {
+      Program(ast) {
+        const sourceCode = context.sourceCode;
+        if (false) {
+          context.report({
+            node: ast,
+            messageId: 'missingOutput',
+          });
+        }
+      },
+    };
+  },
+};
+
+export default rule;

e2e/runAllTests.js

@@ -0,0 +1,55 @@
+import { execSync } from 'node:child_process';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import process from 'node:process';
+
+const TEST_COMMAND = 'npm run lint';
+
+const getRoot = () => {
+  return execSync('git rev-parse --show-toplevel', {
+    encoding: 'utf-8',
+  }).trim();
+};
+
+const executeAllE2eTests = async () => {
+  const e2eDir = path.resolve(getRoot(), './e2e');
+  const failedTests = [];
+
+  // Get all directories in the e2e dir
+  const testDirs = (await fs.readdir(e2eDir, { withFileTypes: true }))
+    .filter((dirEnt) => dirEnt.isDirectory())
+    .map((dirEnt) => path.join(dirEnt.parentPath, dirEnt.name));
+
+  for (const testDir of testDirs) {
+    const dirName = path.basename(testDir);
+    console.log(`🧪 Executing test: ${dirName}`);
+
+    // Run install and the test command
+    execSync('npm install', {
+      cwd: testDir,
+      stdio: ['ignore', 'ignore', 'pipe'],
+    });
+    try {
+      execSync(TEST_COMMAND, {
+        cwd: testDir,
+        stdio: ['ignore', 'ignore', 'pipe'],
+      });
+      console.log(`✅ Test passed\n`);
+    } catch (error) {
+      console.log(`❌ Test failed`);
+      console.error(`${error}\n`);
+      failedTests.push(dirName);
+    }
+  }
+
+  if (failedTests.length) {
+    console.log(
+      `Testing complete. ${failedTests.length} of ${testDirs.length} tests failed!`,
+    );
+    process.exitCode = 1;
+  } else {
+    console.log(`Testing complete.  All ${testDirs.length} tests passed!`);
+  }
+};
+
+executeAllE2eTests();

eslint.config.ts

@@ -18,7 +18,7 @@ const compat = new FlatCompat({
 export default tseslint.config([
   // Global ignores
   {
-    ignores: ['node_modules', 'coverage', 'dist', 'tests/lib/fixtures'],
+    ignores: ['node_modules', 'coverage', 'dist', 'tests/lib/fixtures', 'e2e'],
   },
   // Global settings
   {

package.json

@@ -12,6 +12,7 @@
   "license": "MIT",
   "scripts": {
     "build": "tsdown",
+    "e2e": "node ./e2e/runAllTests.js",
     "format": "prettier --write .",
     "format:check": "prettier --check .",
     "lint": "npm-run-all --continue-on-error --aggregate-output --parallel lint:*",