feat: config file schema and validation

Author: boneskull

package-lock.json

@@ -48,7 +48,9 @@
     "testing"
   ],
   "scripts": {
-    "build": "zshy",
+    "build": "run-s -sl build:compile build:schema",
+    "build:compile": "zshy",
+    "build:schema": "node --import tsx scripts/generate-schema.ts",
     "clean": "rm -rf dist",
     "commitlint": "commitlint",
     "dev": "zshy --watch",
@@ -74,7 +76,8 @@
     "cosmiconfig-typescript-loader": "^6.2.0",
     "glob": "^11.0.3",
     "tinybench": "^5.0.1",
-    "yargs": "^17.7.2"
+    "yargs": "^17.7.2",
+    "zod": "^4.1.12"
   },
   "devDependencies": {
     "@commitlint/cli": "20.1.0",
@@ -126,6 +129,7 @@
     ],
     "node": {
       "entry": [
+        "scripts/**/*.ts",
         "test/**/*.test.ts",
         "src/index.ts",
         "src/cli/index.ts",

package.json

@@ -0,0 +1,54 @@
+#!/usr/bin/env node
+
+/**
+ * Generate JSON Schema from Zod schemas
+ *
+ * This script converts the ModestBench Zod configuration schema to JSON Schema
+ * format, enabling IDE autocomplete and validation in config files.
+ */
+
+import { mkdir, writeFile } from 'node:fs/promises';
+import { dirname, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import * as z from 'zod';
+
+import { partialModestBenchConfigSchema } from '../src/config/schema.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+const generateSchema = async () => {
+  try {
+    // Convert Zod schema to JSON Schema using native Zod v4 functionality
+    const jsonSchema = z.toJSONSchema(partialModestBenchConfigSchema, {
+      target: 'draft-2020-12',
+    });
+
+    // Add top-level schema metadata
+    const schemaWithMetadata = {
+      $id: 'https://github.com/boneskull/modestbench/schema/config.json',
+      $schema: 'https://json-schema.org/draft/2020-12/schema',
+      ...jsonSchema,
+    };
+
+    // Ensure output directory exists
+    const outputPath = resolve(__dirname, '../dist/schema');
+    await mkdir(outputPath, { recursive: true });
+
+    // Write the JSON Schema to file with pretty printing
+    const outputFile = resolve(outputPath, 'modestbench-config.schema.json');
+    await writeFile(
+      outputFile,
+      JSON.stringify(schemaWithMetadata, null, 2) + '\n',
+      'utf-8',
+    );
+
+    console.log(`✓ Generated JSON Schema: ${outputFile}`);
+  } catch (error) {
+    console.error('Failed to generate JSON Schema:', error);
+    process.exit(1);
+  }
+};
+
+// Run the generator
+void generateSchema();

scripts/generate-schema.ts

@@ -20,7 +20,7 @@ interface HistoryOptions {
   maxAge?: number | undefined;
   maxRuns?: number | undefined;
   maxSize?: number | undefined;
-  output?: string | undefined;
+  outputDir?: string | undefined;
   pattern?: string | undefined;
   quiet?: boolean | undefined;
   since?: string | undefined;
@@ -252,12 +252,12 @@ const handleExportCommand = async (
       query,
     );
 
-    if (options.output) {
+    if (options.outputDir) {
       // Write to file
       const fs = await import('node:fs/promises');
-      await fs.writeFile(options.output, exportData, 'utf8');
+      await fs.writeFile(options.outputDir, exportData, 'utf8');
       if (!options.quiet) {
-        console.log(`Exported history to ${options.output}`);
+        console.log(`Exported history to ${options.outputDir}`);
       }
     } else {
       // Write to stdout

src/cli/commands/history.ts

@@ -23,7 +23,7 @@ interface RunOptions {
   iterations?: number | undefined;
   json?: boolean | undefined;
   noColor?: boolean | undefined;
-  output?: string | undefined;
+  outputDir?: string | undefined;
   pattern: string[];
   quiet?: boolean | undefined;
   reporters: string[];
@@ -45,7 +45,7 @@ export const handleRunCommand = async (
   // (i.e., outputting to stdout where we need clean JSON)
   const isUsingJsonReporter = options.reporters?.includes('json') ?? false;
   const shouldBeQuiet =
-    options.quiet || (isUsingJsonReporter && !options.output);
+    options.quiet || (isUsingJsonReporter && !options.outputDir);
 
   try {
     // Step 1: Load and merge configuration
@@ -62,7 +62,7 @@ export const handleRunCommand = async (
       context,
       config,
       shouldBeQuiet,
-      options.output,
+      options.outputDir,
     );
 
     // Step 3: Discovery phase
@@ -208,8 +208,8 @@ const loadConfiguration = async (context: CliContext, options: RunOptions) => {
     if (options.reporters) {
       cliArgs.reporters = options.reporters;
     }
-    if (options.output) {
-      cliArgs.outputDir = resolve(options.cwd, options.output);
+    if (options.outputDir) {
+      cliArgs.outputDir = resolve(options.cwd, options.outputDir);
     }
     if (options.iterations) {
       cliArgs.iterations = options.iterations;

src/cli/commands/run.ts

@@ -255,7 +255,7 @@ export const main = async (
             iterations: argv.iterations,
             json: argv.json,
             noColor: argv.noColor,
-            output: argv.output,
+            outputDir: argv.output,
             pattern: argv.pattern,
             quiet: argv.quiet,
             reporters: argv.reporters,
@@ -360,7 +360,7 @@ export const main = async (
             maxAge: argv.maxAge,
             maxRuns: argv.maxRuns,
             maxSize: argv.maxSize,
-            output: argv.output,
+            outputDir: argv.output,
             pattern: argv.pattern,
             quiet: Boolean(argv.quiet),
             since: argv.since,