feat(plugin-eslint): add support for custom groups (#925)

Author: hanna-skryl

packages/core/package.json

@@ -41,8 +41,7 @@
   "dependencies": {
     "@code-pushup/models": "0.57.0",
     "@code-pushup/utils": "0.57.0",
-    "ansis": "^3.3.0",
-    "zod-validation-error": "^3.4.0"
+    "ansis": "^3.3.0"
   },
   "peerDependencies": {
     "@code-pushup/portal-client": "^0.9.0"

packages/core/src/lib/implementation/read-rc-file.integration.test.ts

@@ -1,7 +1,7 @@
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { describe, expect } from 'vitest';
-import { ConfigValidationError, readRcByPath } from './read-rc-file.js';
+import { readRcByPath } from './read-rc-file.js';
 
 describe('readRcByPath', () => {
   const configDirPath = path.join(
@@ -69,7 +69,7 @@ describe('readRcByPath', () => {
   it('should throw if the configuration is empty', async () => {
     await expect(
       readRcByPath(path.join(configDirPath, 'code-pushup.empty.config.js')),
-    ).rejects.toThrow(expect.any(ConfigValidationError));
+    ).rejects.toThrow(/invalid_type/);
   });
 
   it('should throw if the configuration is invalid', async () => {

packages/core/src/lib/implementation/read-rc-file.ts

@@ -1,31 +1,18 @@
-import { bold } from 'ansis';
 import path from 'node:path';
-import { fromError, isZodErrorLike } from 'zod-validation-error';
 import {
   CONFIG_FILE_NAME,
   type CoreConfig,
   SUPPORTED_CONFIG_FILE_FORMATS,
   coreConfigSchema,
 } from '@code-pushup/models';
-import {
-  fileExists,
-  importModule,
-  zodErrorMessageBuilder,
-} from '@code-pushup/utils';
+import { fileExists, importModule, parseSchema } from '@code-pushup/utils';
 
 export class ConfigPathError extends Error {
   constructor(configPath: string) {
     super(`Provided path '${configPath}' is not valid.`);
   }
 }
 
-export class ConfigValidationError extends Error {
-  constructor(configPath: string, message: string) {
-    const relativePath = path.relative(process.cwd(), configPath);
-    super(`Failed parsing core config in ${bold(relativePath)}.\n\n${message}`);
-  }
-}
-
 export async function readRcByPath(
   filepath: string,
   tsconfig?: string,
@@ -38,18 +25,16 @@ export async function readRcByPath(
     throw new ConfigPathError(filepath);
   }
 
-  const cfg = await importModule({ filepath, tsconfig, format: 'esm' });
+  const cfg: CoreConfig = await importModule({
+    filepath,
+    tsconfig,
+    format: 'esm',
+  });
 
-  try {
-    return coreConfigSchema.parse(cfg);
-  } catch (error) {
-    const validationError = fromError(error, {
-      messageBuilder: zodErrorMessageBuilder,
-    });
-    throw isZodErrorLike(error)
-      ? new ConfigValidationError(filepath, validationError.message)
-      : error;
-  }
+  return parseSchema(coreConfigSchema, cfg, {
+    schemaType: 'core config',
+    sourcePath: filepath,
+  });
 }
 
 export async function autoloadRc(tsconfig?: string): Promise<CoreConfig> {

packages/plugin-eslint/README.md

@@ -93,6 +93,42 @@ Detected ESLint rules are mapped to Code PushUp audits. Audit reports are calcul
 
 5. Run the CLI with `npx code-pushup collect` and view or upload report (refer to [CLI docs](../cli/README.md)).
 
+### Custom groups
+
+You can extend the plugin configuration with custom groups to categorize ESLint rules according to your project's specific needs. Custom groups allow you to assign weights to individual rules, influencing their impact on the report. Rules can be defined as an object with explicit weights or as an array where each rule defaults to a weight of 1. Additionally, you can use wildcard patterns (`*`) to include multiple rules with similar prefixes.
+
+```js
+import eslintPlugin from '@code-pushup/eslint-plugin';
+
+export default {
+  // ...
+  plugins: [
+    // ...
+    await eslintPlugin(
+      { eslintrc: '.eslintrc.js', patterns: ['src/**/*.js'] },
+      {
+        groups: [
+          {
+            slug: 'modern-angular',
+            title: 'Modern Angular',
+            rules: {
+              '@angular-eslint/template/prefer-control-flow': 3,
+              '@angular-eslint/template/prefer-ngsrc': 2,
+              '@angular-eslint/component-selector': 1,
+            },
+          },
+          {
+            slug: 'type-safety',
+            title: 'Type safety',
+            rules: ['@typescript-eslint/no-unsafe-*'],
+          },
+        ],
+      },
+    ),
+  ],
+};
+```
+
 ### Optionally set up categories
 
 1. Reference audits (or groups) which you wish to include in custom categories (use `npx code-pushup print-config` to list audits and groups).

packages/plugin-eslint/src/lib/config.ts

@@ -33,3 +33,36 @@ export type ESLintPluginRunnerConfig = {
   targets: ESLintTarget[];
   slugs: string[];
 };
+
+const customGroupRulesSchema = z.union(
+  [
+    z
+      .array(z.string())
+      .min(1, 'Custom group rules must contain at least 1 element'),
+    z.record(z.string(), z.number()).refine(
+      schema => Object.keys(schema).length > 0,
+      () => ({
+        code: 'too_small',
+        message: 'Custom group rules must contain at least 1 element',
+      }),
+    ),
+  ],
+  {
+    description:
+      'Array of rule IDs with equal weights or object mapping rule IDs to specific weights',
+  },
+);
+
+const customGroupSchema = z.object({
+  slug: z.string({ description: 'Unique group identifier' }),
+  title: z.string({ description: 'Group display title' }),
+  description: z.string({ description: 'Group metadata' }).optional(),
+  docsUrl: z.string({ description: 'Group documentation site' }).optional(),
+  rules: customGroupRulesSchema,
+});
+export type CustomGroup = z.infer<typeof customGroupSchema>;
+
+export const eslintPluginOptionsSchema = z.object({
+  groups: z.array(customGroupSchema).optional(),
+});
+export type ESLintPluginOptions = z.infer<typeof eslintPluginOptionsSchema>;

packages/plugin-eslint/src/lib/eslint-plugin.integration.test.ts

@@ -71,11 +71,76 @@ describe('eslintPlugin', () => {
     );
   });
 
+  it('should initialize with plugin options for custom groups', async () => {
+    cwdSpy.mockReturnValue(path.join(fixturesDir, 'nx-monorepo'));
+    const plugin = await eslintPlugin(
+      {
+        eslintrc: './packages/nx-plugin/eslint.config.js',
+        patterns: ['packages/nx-plugin/**/*.ts'],
+      },
+      {
+        groups: [
+          {
+            slug: 'type-safety',
+            title: 'Type safety',
+            rules: [
+              '@typescript-eslint/no-explicit-any',
+              '@typescript-eslint/no-unsafe-*',
+            ],
+          },
+        ],
+      },
+    );
+
+    expect(plugin.groups).toContainEqual({
+      slug: 'type-safety',
+      title: 'Type safety',
+      refs: [
+        { slug: 'typescript-eslint-no-explicit-any', weight: 1 },
+        {
+          slug: 'typescript-eslint-no-unsafe-declaration-merging',
+          weight: 1,
+        },
+        { slug: 'typescript-eslint-no-unsafe-function-type', weight: 1 },
+      ],
+    });
+    expect(plugin.audits).toContainEqual(
+      expect.objectContaining<Partial<Audit>>({
+        slug: 'typescript-eslint-no-explicit-any',
+      }),
+    );
+  });
+
+  it('should throw when custom group rules are empty', async () => {
+    await expect(
+      eslintPlugin(
+        {
+          eslintrc: './packages/nx-plugin/eslint.config.js',
+          patterns: ['packages/nx-plugin/**/*.ts'],
+        },
+        {
+          groups: [{ slug: 'type-safety', title: 'Type safety', rules: [] }],
+        },
+      ),
+    ).rejects.toThrow(/Custom group rules must contain at least 1 element/);
+    await expect(
+      eslintPlugin(
+        {
+          eslintrc: './packages/nx-plugin/eslint.config.js',
+          patterns: ['packages/nx-plugin/**/*.ts'],
+        },
+        {
+          groups: [{ slug: 'type-safety', title: 'Type safety', rules: {} }],
+        },
+      ),
+    ).rejects.toThrow(/Custom group rules must contain at least 1 element/);
+  });
+
   it('should throw when invalid parameters provided', async () => {
     await expect(
       // @ts-expect-error simulating invalid non-TS config
       eslintPlugin({ eslintrc: '.eslintrc.json' }),
-    ).rejects.toThrow('patterns');
+    ).rejects.toThrow(/Invalid input/);
   });
 
   it("should throw if eslintrc file doesn't exist", async () => {

packages/plugin-eslint/src/lib/eslint-plugin.ts

@@ -2,7 +2,13 @@ import { createRequire } from 'node:module';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import type { PluginConfig } from '@code-pushup/models';
-import { type ESLintPluginConfig, eslintPluginConfigSchema } from './config.js';
+import { parseSchema } from '@code-pushup/utils';
+import {
+  type ESLintPluginConfig,
+  type ESLintPluginOptions,
+  eslintPluginConfigSchema,
+  eslintPluginOptionsSchema,
+} from './config.js';
 import { listAuditsAndGroups } from './meta/index.js';
 import { createRunnerConfig } from './runner/index.js';
 
@@ -24,14 +30,24 @@ import { createRunnerConfig } from './runner/index.js';
  * }
  *
  * @param config Configuration options.
+ * @param options Optional settings for customizing the plugin behavior.
  * @returns Plugin configuration as a promise.
  */
 export async function eslintPlugin(
   config: ESLintPluginConfig,
+  options?: ESLintPluginOptions,
 ): Promise<PluginConfig> {
-  const targets = eslintPluginConfigSchema.parse(config);
+  const targets = parseSchema(eslintPluginConfigSchema, config, {
+    schemaType: 'ESLint plugin config',
+  });
 
-  const { audits, groups } = await listAuditsAndGroups(targets);
+  const customGroups = options
+    ? parseSchema(eslintPluginOptionsSchema, options, {
+        schemaType: 'ESLint plugin options',
+      }).groups
+    : undefined;
+
+  const { audits, groups } = await listAuditsAndGroups(targets, customGroups);
 
   const runnerScriptPath = path.join(
     fileURLToPath(path.dirname(import.meta.url)),

packages/plugin-eslint/src/lib/meta/groups.ts

@@ -1,8 +1,10 @@
 import type { Rule } from 'eslint';
 import type { Group, GroupRef } from '@code-pushup/models';
-import { objectToKeys, slugify } from '@code-pushup/utils';
+import { objectToKeys, slugify, ui } from '@code-pushup/utils';
+import type { CustomGroup } from '../config.js';
 import { ruleToSlug } from './hash.js';
 import { type RuleData, parseRuleId } from './parse.js';
+import { expandWildcardRules } from './rules.js';
 
 type RuleType = NonNullable<Rule.RuleMetaData['type']>;
 
@@ -87,3 +89,80 @@ export function groupsFromRuleCategories(rules: RuleData[]): Group[] {
 
   return groups.toSorted((a, b) => a.slug.localeCompare(b.slug));
 }
+
+export function groupsFromCustomConfig(
+  rules: RuleData[],
+  groups: CustomGroup[],
+): Group[] {
+  const rulesMap = createRulesMap(rules);
+
+  return groups.map(group => {
+    const groupRules = Array.isArray(group.rules)
+      ? Object.fromEntries(group.rules.map(rule => [rule, 1]))
+      : group.rules;
+
+    const { refs, invalidRules } = resolveGroupRefs(groupRules, rulesMap);
+
+    if (invalidRules.length > 0 && Object.entries(groupRules).length > 0) {
+      if (refs.length === 0) {
+        throw new Error(
+          `Invalid rule configuration in group ${group.slug}. All rules are invalid.`,
+        );
+      }
+      ui().logger.warning(
+        `Some rules in group ${group.slug} are invalid: ${invalidRules.join(', ')}`,
+      );
+    }
+
+    return {
+      slug: group.slug,
+      title: group.title,
+      refs,
+    };
+  });
+}
+
+export function createRulesMap(rules: RuleData[]): Record<string, RuleData[]> {
+  return rules.reduce<Record<string, RuleData[]>>(
+    (acc, rule) => ({
+      ...acc,
+      [rule.id]: [...(acc[rule.id] || []), rule],
+    }),
+    {},
+  );
+}
+
+export function resolveGroupRefs(
+  groupRules: Record<string, number>,
+  rulesMap: Record<string, RuleData[]>,
+): { refs: Group['refs']; invalidRules: string[] } {
+  return Object.entries(groupRules).reduce<{
+    refs: Group['refs'];
+    invalidRules: string[];
+  }>(
+    (acc, [rule, weight]) => {
+      const matchedRuleIds = rule.endsWith('*')
+        ? expandWildcardRules(rule, Object.keys(rulesMap))
+        : [rule];
+
+      const matchedRefs = matchedRuleIds.flatMap(ruleId => {
+        const matchingRules = rulesMap[ruleId] || [];
+        const weightPerRule = weight / matchingRules.length;
+
+        return matchingRules.map(ruleData => ({
+          slug: ruleToSlug(ruleData),
+          weight: weightPerRule,
+        }));
+      });
+
+      return {
+        refs: [...acc.refs, ...matchedRefs],
+        invalidRules:
+          matchedRefs.length > 0
+            ? acc.invalidRules
+            : [...acc.invalidRules, rule],
+      };
+    },
+    { refs: [], invalidRules: [] },
+  );
+}