feat(core): enhance config validation

packages/core/src/lib/implementation/read-rc-file.ts

@@ -1,40 +1,17 @@
-import { bold, red } from 'ansis';
+import { bold } from 'ansis';
 import path, { join } from 'node:path';
-import {
-  type MessageBuilder,
-  fromError,
-  isZodErrorLike,
-} from 'zod-validation-error';
+import { fromError, isZodErrorLike } from 'zod-validation-error';
 import {
   CONFIG_FILE_NAME,
   type CoreConfig,
   SUPPORTED_CONFIG_FILE_FORMATS,
   coreConfigSchema,
 } from '@code-pushup/models';
-import { fileExists, importModule } from '@code-pushup/utils';
-
-function formatErrorPath(errorPath: (string | number)[]): string {
-  return errorPath
-    .map((key, index) => {
-      if (typeof key === 'number') {
-        return `[${key}]`;
-      }
-      return index > 0 ? `.${key}` : key;
-    })
-    .join('');
-}
-
-const coreConfigMessageBuilder: MessageBuilder = issues =>
-  issues
-    .map(issue => {
-      const formattedMessage = red(`${bold(issue.code)}: ${issue.message}`);
-      const formattedPath = formatErrorPath(issue.path);
-      if (formattedPath) {
-        return `Validation error at ${bold(formattedPath)}\n${formattedMessage}\n`;
-      }
-      return `${formattedMessage}\n`;
-    })
-    .join('\n');
+import {
+  coreConfigMessageBuilder,
+  fileExists,
+  importModule,
+} from '@code-pushup/utils';
 
 export class ConfigPathError extends Error {
   constructor(configPath: string) {
@@ -65,13 +42,13 @@
 
   try {
     return coreConfigSchema.parse(cfg);
   } catch (error) {
     const validationError = fromError(error, {
       messageBuilder: coreConfigMessageBuilder,
     });
     throw isZodErrorLike(error)
       ? new ConfigValidationError(filepath, validationError.message)
       : error;
   }
 }
 

packages/models/src/lib/category-config.ts

@@ -23,22 +23,21 @@ export const categoryRefSchema = weightedRefSchema(
 );
 export type CategoryRef = z.infer<typeof categoryRefSchema>;
 
-export const categoryConfigSchema = z
-  .intersection(
-    scorableSchema(
-      'Category with a score calculated from audits and groups from various plugins',
-      categoryRefSchema,
-      getDuplicateRefsInCategoryMetrics,
-      duplicateRefsInCategoryMetricsErrorMsg,
-    ),
+export const categoryConfigSchema = scorableSchema(
+  'Category with a score calculated from audits and groups from various plugins',
+  categoryRefSchema,
+  getDuplicateRefsInCategoryMetrics,
+  duplicateRefsInCategoryMetricsErrorMsg,
+)
+  .merge(
     metaSchema({
       titleDescription: 'Category Title',
       docsUrlDescription: 'Category docs URL',
       descriptionDescription: 'Category description',
       description: 'Meta info for category',
     }),
   )
-  .and(
+  .merge(
     z.object({
       isBinary: z
         .boolean({

packages/models/src/lib/category-config.unit.test.ts

@@ -129,7 +129,7 @@ describe('categoryConfigSchema', () => {
         title: 'This category is empty for now',
         refs: [],
       } satisfies CategoryConfig),
-    ).toThrow('In category in-progress, there has to be at least one ref');
+    ).toThrow('In a category, there has to be at least one ref');
   });
 
   it('should throw for duplicate category references', () => {
@@ -176,7 +176,7 @@ describe('categoryConfigSchema', () => {
         ],
       } satisfies CategoryConfig),
     ).toThrow(
-      'In category informational, there has to be at least one ref with weight > 0. Affected refs: functional/immutable-data, lighthouse-experimental',
+      /In a category, there has to be at least one ref with weight > 0. Affected refs: \\"functional\/immutable-data\\", \\"lighthouse-experimental\\"/,
     );
   });
 });

packages/models/src/lib/group.ts

@@ -25,16 +25,14 @@ export const groupMetaSchema = metaSchema({
 });
 export type GroupMeta = z.infer<typeof groupMetaSchema>;
 
-export const groupSchema = z.intersection(
-  scorableSchema(
-    'A group aggregates a set of audits into a single score which can be referenced from a category. ' +
-      'E.g. the group slug "performance" groups audits and can be referenced in a category',
-    groupRefSchema,
-    getDuplicateRefsInGroups,
-    duplicateRefsInGroupsErrorMsg,
-  ),
-  groupMetaSchema,
-);
+export const groupSchema = scorableSchema(
+  'A group aggregates a set of audits into a single score which can be referenced from a category. ' +
+    'E.g. the group slug "performance" groups audits and can be referenced in a category',
+  groupRefSchema,
+  getDuplicateRefsInGroups,
+  duplicateRefsInGroupsErrorMsg,
+).merge(groupMetaSchema);
+
 export type Group = z.infer<typeof groupSchema>;
 
 export const groupsSchema = z

packages/models/src/lib/group.unit.test.ts

@@ -69,7 +69,7 @@ describe('groupSchema', () => {
         title: 'Empty group',
         refs: [],
       } satisfies Group),
-    ).toThrow('In category empty-group, there has to be at least one ref');
+    ).toThrow('In a category, there has to be at least one ref');
   });
 
   it('should throw for duplicate group references', () => {

packages/models/src/lib/implementation/schemas.ts

@@ -167,43 +167,28 @@ export function scorableSchema<T extends ReturnType<typeof weightedRefSchema>>(
   duplicateCheckFn: (metrics: z.infer<T>[]) => false | string[],
   duplicateMessageFn: (metrics: z.infer<T>[]) => string,
 ) {
-  return (
-    z
-      .object(
-        {
-          slug: slugSchema.describe(
-            'Human-readable unique ID, e.g. "performance"',
-          ),
-          refs: z
-            .array(refSchema)
-            .min(1)
-            // refs are unique
-            .refine(
-              refs => !duplicateCheckFn(refs),
-              refs => ({
-                message: duplicateMessageFn(refs),
-              }),
-            ),
-        },
-        { description },
-      )
-      // category weights are correct
-      .superRefine(({ slug, refs }, ctx) => {
-        if (refs.length === 0) {
-          ctx.addIssue({
-            code: z.ZodIssueCode.custom,
-            message: `In category ${slug}, there has to be at least one ref`,
-            path: ['refs'],
-          });
-        } else if (!hasNonZeroWeightedRef(refs)) {
-          const affectedRefs = refs.map(ref => ref.slug).join(', ');
-          ctx.addIssue({
-            code: z.ZodIssueCode.custom,
-            message: `In category ${slug}, there has to be at least one ref with weight > 0. Affected refs: ${affectedRefs}`,
-            path: ['refs'],
-          });
-        }
-      })
+  return z.object(
+    {
+      slug: slugSchema.describe('Human-readable unique ID, e.g. "performance"'),
+      refs: z
+        .array(refSchema)
+        .min(1, { message: 'In a category, there has to be at least one ref' })
+        // refs are unique
+        .refine(
+          refs => !duplicateCheckFn(refs),
+          refs => ({
+            message: duplicateMessageFn(refs),
+          }),
+        )
+        // category weights are correct
+        .refine(hasNonZeroWeightedRef, refs => {
+          const affectedRefs = refs.map(ref => `"${ref.slug}"`).join(', ');
+          return {
+            message: `In a category, there has to be at least one ref with weight > 0. Affected refs: ${affectedRefs}`,
+          };
+        }),
+    },
+    { description },
   );
 }
 

packages/plugin-lighthouse/src/lib/lighthouse-plugin.unit.test.ts

@@ -57,7 +57,7 @@ describe('lighthousePlugin-config-object', () => {
     });
 
     expect(() => pluginConfigSchema.parse(pluginConfig)).toThrow(
-      'In category best-practices, there has to be at least one ref with weight > 0. Affected refs: csp-xss',
+      /In a category, there has to be at least one ref with weight > 0. Affected refs: \\"csp-xss\\"/,
     );
   });
 });

packages/utils/package.json

@@ -33,6 +33,7 @@
     "esbuild": "^0.19.2",
     "multi-progress-bars": "^5.0.3",
     "semver": "^7.6.0",
-    "simple-git": "^3.20.0"
+    "simple-git": "^3.20.0",
+    "zod-validation-error": "^3.4.0"
   }
 }

packages/utils/src/index.ts

@@ -122,3 +122,4 @@ export type {
   WithRequired,
 } from './lib/types.js';
 export { verboseUtils } from './lib/verbose-utils.js';
+export { coreConfigMessageBuilder } from './lib/zod-validation.js';

packages/utils/src/lib/zod-validation.ts

@@ -0,0 +1,25 @@
+import { bold, red } from 'ansis';
+import type { MessageBuilder } from 'zod-validation-error';
+
+export function formatErrorPath(errorPath: (string | number)[]): string {
+  return errorPath
+    .map((key, index) => {
+      if (typeof key === 'number') {
+        return `[${key}]`;
+      }
+      return index > 0 ? `.${key}` : key;
+    })
+    .join('');
+}
+
+export const coreConfigMessageBuilder: MessageBuilder = issues =>
+  issues
+    .map(issue => {
+      const formattedMessage = red(`${bold(issue.code)}: ${issue.message}`);
+      const formattedPath = formatErrorPath(issue.path);
+      if (formattedPath) {
+        return `Validation error at ${bold(formattedPath)}\n${formattedMessage}\n`;
+      }
+      return `${formattedMessage}\n`;
+    })
+    .join('\n');

packages/utils/src/lib/zod-validation.unit.test.ts

@@ -0,0 +1,14 @@
+import { formatErrorPath } from './zod-validation';
+
+describe('formatErrorPath', () => {
+  it.each([
+    [['categories', 1, 'slug'], 'categories[1].slug'],
+    [['plugins', 2, 'groups', 0, 'refs'], 'plugins[2].groups[0].refs'],
+    [['refs', 0, 'slug'], 'refs[0].slug'],
+    [['categories'], 'categories'],
+    [[], ''],
+    [['path', 5], 'path[5]'],
+  ])('formats error path correctly for $input', (input, expected) => {
+    expect(formatErrorPath(input)).toBe(expected);
+  });
+});