diff --git a/package.json b/package.json index 66fe3b3..a9a100d 100644 --- a/package.json +++ b/package.json @@ -181,7 +181,6 @@ "@types/wallabyjs", "@types/mocha", "markdownlint-cli2-formatter-pretty", - "asciinema-player", "@astrojs/mdx", "astro-broken-link-checker", "jest", diff --git a/scripts/generate-schema.ts b/scripts/generate-schema.ts index ffd16ae..938b6b2 100644 --- a/scripts/generate-schema.ts +++ b/scripts/generate-schema.ts @@ -12,7 +12,7 @@ import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import * as z from 'zod'; -import { partialModestBenchConfigSchema } from '../src/config/schema.js'; +import { partialModestBenchConfigInputSchema } from '../src/config/schema.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -20,7 +20,7 @@ const __dirname = dirname(__filename); const generateSchema = async () => { try { // Convert Zod schema to JSON Schema using native Zod v4 functionality - const jsonSchema = z.toJSONSchema(partialModestBenchConfigSchema, { + const jsonSchema = z.toJSONSchema(partialModestBenchConfigInputSchema, { target: 'draft-2020-12', }); diff --git a/src/adapters/types.ts b/src/adapters/types.ts index 61b35cd..1e5d495 100644 --- a/src/adapters/types.ts +++ b/src/adapters/types.ts @@ -8,7 +8,7 @@ import type { BenchmarkDefinition, BenchmarkSuite, -} from '../core/benchmark-schema.js'; +} from '../config/benchmark-schema.js'; /** * A captured test suite (describe block) from a test framework diff --git a/src/core/benchmark-schema.ts b/src/config/benchmark-schema.ts similarity index 98% rename from src/core/benchmark-schema.ts rename to src/config/benchmark-schema.ts index 74558fb..8228588 100644 --- a/src/core/benchmark-schema.ts +++ b/src/config/benchmark-schema.ts @@ -11,7 +11,7 @@ import { z } from 'zod'; import type { ModestBenchConfig } from '../types/core.js'; -import { partialModestBenchConfigSchema } from '../config/schema.js'; +import { partialModestBenchConfigSchema } from './schema.js'; /** * Schema for benchmark functions diff --git a/src/config/schema.ts b/src/config/schema.ts index 73bd582..e5de808 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -8,6 +8,8 @@ import * as z from 'zod'; +import type { Budget } from '../types/budgets.js'; + import { BENCHMARK_FILE_PATTERN } from '../constants.js'; import { parsePercentageString, parseTimeString } from './budget-schema.js'; @@ -83,11 +85,11 @@ const thresholdConfigSchema = z }); /** - * Inline budget schema for configuration (no transforms for JSON Schema - * compatibility - transforms are applied manually in transformBudgets - * function) + * Input schema for budget values (before transformation) + * + * Accepts string values like "10ms" or "10%" for human-readable configuration. */ -const budgetSchema = z +const budgetInputSchema = z .object({ absolute: z .object({ @@ -97,6 +99,7 @@ const budgetSchema = z .describe('Maximum 99th percentile in nanoseconds or time string'), maxTime: z .union([z.number().positive(), z.string()]) + .optional() .describe( 'Maximum mean time in nanoseconds or time string (e.g., "10ms")', ), @@ -123,320 +126,336 @@ const budgetSchema = z .describe('Performance budget with absolute and/or relative thresholds'); /** - * Schema for the main ModestBench configuration + * Transform budget values (parse time/percentage strings to numbers) * - * This is the complete configuration schema used for validating benchmark - * configuration from all sources (files, CLI args, defaults). + * Returns a Budget object with all string values converted to numbers. */ -const modestBenchConfigSchema = z - .object({ - $schema: z - .string() - .optional() - .describe( - 'JSON Schema reference for IDE support (not used by ModestBench)', - ), - bail: z.boolean().describe('Stop benchmark execution on first failure'), - baseline: z - .string() - .optional() - .describe( - 'Name of baseline to use for relative budget comparisons. Must match a saved baseline name.', - ), - budgetMode: z - .enum(['fail', 'warn', 'report']) - .optional() - .describe( - 'How to handle budget violations: "fail" exits with error (default), "warn" shows warnings, "report" includes in output without failing', - ), - budgets: z - .record( - z.string(), - z.record(z.string(), z.record(z.string(), budgetSchema)), - ) - .optional() - .describe( - 'Performance budgets organized by file → suite → task. Budgets define acceptable performance thresholds.', - ), - exclude: z - .array(z.string()) - .describe( - 'Glob patterns to exclude from benchmark file discovery (e.g., "node_modules/**", ".git/**")', - ), - excludeTags: z - .array(z.string()) - .describe( - 'Tags to exclude from benchmark execution. Benchmarks matching any of these tags will be skipped.', - ), - iterations: z - .number() - .int() - .positive() - .describe( - 'Default number of iterations to run for each benchmark task. Higher values provide more accurate statistics but take longer to execute.', - ), - limitBy: z - .enum(['time', 'iterations', 'any', 'all']) - .describe( - 'How to limit benchmark execution: "time" stops after time limit, "iterations" stops after iteration count, "any" stops at whichever comes first, "all" runs until both limits are reached', - ), - metadata: z - .record(z.string(), z.unknown()) - .describe( - 'Custom metadata to attach to benchmark runs. Can include project name, version, environment details, etc.', - ), - outputDir: z - .string() - .min(1) - .optional() - .describe( - 'Directory path where benchmark results and reports will be written. If not specified, data reporters will write to stdout.', - ), - pattern: z - .union([z.string().min(1), z.array(z.string().min(1))]) - .describe( - `Glob pattern(s) for discovering benchmark files. Can be a single pattern string or array of patterns (e.g., "**/*${BENCHMARK_FILE_PATTERN}")`, - ), - profile: z - .object({ - exclude: z - .array(z.string()) - .optional() - .describe('Glob patterns to exclude from profiling results'), - focus: z - .array(z.string()) - .optional() - .describe( - 'Glob patterns to focus on in profiling results. If specified, only matching files will be shown', - ), - minCallCount: z - .number() - .int() - .nonnegative() - .optional() - .describe( - 'Minimum number of times a function must be called to be included in results', - ), - minExecutionPercent: z - .number() - .nonnegative() - .max(100) - .default(1.0) - .describe( - 'Minimum execution percentage threshold for including functions in results', - ), - outputFile: z - .string() - .optional() - .describe('Path to write profile report to file'), - smartDetection: z - .boolean() - .default(true) - .describe( - 'Automatically detect and focus on user code, excluding node_modules and Node.js internals', - ), - topN: z - .number() - .int() - .positive() - .default(25) - .describe('Maximum number of top functions to show in results'), - }) - .optional() - .describe( - 'Configuration for profile command to identify benchmark candidates', - ), - quiet: z - .boolean() - .describe( - 'Run in quiet mode with minimal console output (only errors and final results)', - ), - reporterConfig: reporterConfigSchema, - reporters: z - .array(z.string()) - .min(1) - .describe( - 'List of reporter names to use for output. Available reporters: "human", "json", "csv"', - ), - tags: z - .array(z.string()) - .describe( - 'Tags to filter which benchmarks to run. If empty, all benchmarks are included. Only benchmarks with matching tags will execute.', - ), - thresholds: thresholdConfigSchema, - time: z - .number() - .int() - .positive() - .describe( - 'Maximum time to spend on each benchmark task in milliseconds. Tasks will run at least until this duration or iteration count is reached, depending on limitBy setting.', - ), - timeout: z - .number() - .int() - .positive() - .describe( - 'Timeout for individual benchmark tasks in milliseconds. Tasks exceeding this duration will be terminated and marked as failed.', - ), - verbose: z - .boolean() - .describe( - 'Enable verbose output. Provides more detailed console output including progress, intermediate results, and diagnostic information', - ), - warmup: z - .number() - .int() - .nonnegative() - .describe( - 'Number of warmup iterations to run before measurement begins. Warmup helps stabilize performance by allowing JIT compilation and caching to occur.', - ), - }) - .strict() - .describe( - 'ModestBench configuration for controlling benchmark discovery, execution, and reporting', - ) - .meta({ - title: 'ModestBench Configuration', - }); +const transformBudgetValues = ( + budget: z.infer, +): Budget => { + return { + // Build absolute budget object if present + absolute: budget.absolute + ? { + maxP99: + budget.absolute.maxP99 !== undefined + ? typeof budget.absolute.maxP99 === 'string' + ? parseTimeString(budget.absolute.maxP99) + : budget.absolute.maxP99 + : undefined, + maxTime: + budget.absolute.maxTime !== undefined + ? typeof budget.absolute.maxTime === 'string' + ? parseTimeString(budget.absolute.maxTime) + : budget.absolute.maxTime + : undefined, + minOpsPerSec: budget.absolute.minOpsPerSec, + } + : undefined, + // Build relative budget object if present + relative: budget.relative + ? { + maxRegression: + budget.relative.maxRegression !== undefined + ? typeof budget.relative.maxRegression === 'string' + ? parsePercentageString(budget.relative.maxRegression) + : budget.relative.maxRegression + : undefined, + } + : undefined, + }; +}; /** - * Validate a partial configuration object + * Budget schema with transform for string-to-number conversion * - * This is used for validating configuration from files or CLI args before - * merging with defaults. + * Input: Budget with string values like "10ms" or "10%" Output: Budget with + * numeric values only */ -export const partialModestBenchConfigSchema: z.ZodType< - Partial -> = modestBenchConfigSchema.partial(); +const budgetSchema = budgetInputSchema.transform(transformBudgetValues); /** - * Input budget type (before transformation) + * Input schema for budgets (nested file → suite → task → budget structure) + * without transforms - used for JSON Schema generation. */ -interface BudgetInput { - absolute?: { - maxP99?: number | string; - maxTime?: number | string; - minOpsPerSec?: number; - }; - relative?: { - maxRegression?: number | string; - }; -} +const budgetsRawInputSchema = z.record( + z.string(), + z.record(z.string(), z.record(z.string(), budgetInputSchema)), +); /** - * Output budget type (after transformation) + * Input schema for budgets with individual budget transforms applied. + * + * Used to validate the human-readable nested format from config files. */ -interface BudgetOutput { - absolute?: { - maxP99?: number; - maxTime?: number; - minOpsPerSec?: number; - }; - relative?: { - maxRegression?: number; - }; -} +const budgetsInputSchema = z.record( + z.string(), + z.record(z.string(), z.record(z.string(), budgetSchema)), +); /** - * Transform budget values (parse time/percentage strings) + * Transform nested budget structure to flat TaskId → Budget mapping + * + * @param nested - Nested budgets structure (file → suite → task → budget) + * @returns Flat budgets map (taskId → budget) */ -const transformBudgetValues = (budget: BudgetInput): BudgetOutput => { - const transformed: BudgetOutput = {}; - - if (budget.absolute) { - transformed.absolute = {}; +const flattenBudgets = ( + nested: z.infer, +): Record => { + const flat: Record = {}; - // Copy minOpsPerSec as-is (already a number) - if (budget.absolute.minOpsPerSec !== undefined) { - transformed.absolute.minOpsPerSec = budget.absolute.minOpsPerSec; - } - - // Parse time strings - if (budget.absolute.maxTime !== undefined) { - transformed.absolute.maxTime = - typeof budget.absolute.maxTime === 'string' - ? parseTimeString(budget.absolute.maxTime) - : budget.absolute.maxTime; - } - if (budget.absolute.maxP99 !== undefined) { - transformed.absolute.maxP99 = - typeof budget.absolute.maxP99 === 'string' - ? parseTimeString(budget.absolute.maxP99) - : budget.absolute.maxP99; + for (const [file, suites] of Object.entries(nested)) { + for (const [suite, tasks] of Object.entries(suites)) { + for (const [task, budget] of Object.entries(tasks)) { + const taskId = `${file}/${suite}/${task}`; + flat[taskId] = budget; + } } } - if (budget.relative) { - transformed.relative = {}; + return flat; +}; - // Parse percentage strings - if (budget.relative.maxRegression !== undefined) { - transformed.relative.maxRegression = - typeof budget.relative.maxRegression === 'string' - ? parsePercentageString(budget.relative.maxRegression) - : budget.relative.maxRegression; - } - } +/** + * Budgets schema with transform for nested-to-flat conversion + * + * Input: { [file]: { [suite]: { [task]: Budget } } } Output: { [taskId]: Budget + * } where taskId = "file/suite/task" + */ +const budgetsSchema = budgetsInputSchema.transform(flattenBudgets); - return transformed; +/** + * Shared configuration properties (everything except budgets) + * + * These properties are identical between the runtime schema (with transforms) + * and the JSON Schema generation schema (without transforms). + */ +const baseConfigProperties = { + $schema: z + .string() + .optional() + .describe( + 'JSON Schema reference for IDE support (not used by ModestBench)', + ), + bail: z.boolean().describe('Stop benchmark execution on first failure'), + baseline: z + .string() + .optional() + .describe( + 'Name of baseline to use for relative budget comparisons. Must match a saved baseline name.', + ), + budgetMode: z + .enum(['fail', 'warn', 'report']) + .optional() + .describe( + 'How to handle budget violations: "fail" exits with error (default), "warn" shows warnings, "report" includes in output without failing', + ), + exclude: z + .array(z.string()) + .describe( + 'Glob patterns to exclude from benchmark file discovery (e.g., "node_modules/**", ".git/**")', + ), + excludeTags: z + .array(z.string()) + .describe( + 'Tags to exclude from benchmark execution. Benchmarks matching any of these tags will be skipped.', + ), + iterations: z + .number() + .int() + .positive() + .describe( + 'Default number of iterations to run for each benchmark task. Higher values provide more accurate statistics but take longer to execute.', + ), + limitBy: z + .enum(['time', 'iterations', 'any', 'all']) + .describe( + 'How to limit benchmark execution: "time" stops after time limit, "iterations" stops after iteration count, "any" stops at whichever comes first, "all" runs until both limits are reached', + ), + metadata: z + .record(z.string(), z.unknown()) + .describe( + 'Custom metadata to attach to benchmark runs. Can include project name, version, environment details, etc.', + ), + outputDir: z + .string() + .min(1) + .optional() + .describe( + 'Directory path where benchmark results and reports will be written. If not specified, data reporters will write to stdout.', + ), + pattern: z + .union([z.string().min(1), z.array(z.string().min(1))]) + .describe( + `Glob pattern(s) for discovering benchmark files. Can be a single pattern string or array of patterns (e.g., "**/*${BENCHMARK_FILE_PATTERN}")`, + ), + profile: z + .object({ + exclude: z + .array(z.string()) + .optional() + .describe('Glob patterns to exclude from profiling results'), + focus: z + .array(z.string()) + .optional() + .describe( + 'Glob patterns to focus on in profiling results. If specified, only matching files will be shown', + ), + minCallCount: z + .number() + .int() + .nonnegative() + .optional() + .describe( + 'Minimum number of times a function must be called to be included in results', + ), + minExecutionPercent: z + .number() + .nonnegative() + .max(100) + .default(1.0) + .describe( + 'Minimum execution percentage threshold for including functions in results', + ), + outputFile: z + .string() + .optional() + .describe('Path to write profile report to file'), + smartDetection: z + .boolean() + .default(true) + .describe( + 'Automatically detect and focus on user code, excluding node_modules and Node.js internals', + ), + topN: z + .number() + .int() + .positive() + .default(25) + .describe('Maximum number of top functions to show in results'), + }) + .optional() + .describe( + 'Configuration for profile command to identify benchmark candidates', + ), + quiet: z + .boolean() + .describe( + 'Run in quiet mode with minimal console output (only errors and final results)', + ), + reporterConfig: reporterConfigSchema, + reporters: z + .array(z.string()) + .min(1) + .describe( + 'List of reporter names to use for output. Available reporters: "human", "json", "csv"', + ), + tags: z + .array(z.string()) + .describe( + 'Tags to filter which benchmarks to run. If empty, all benchmarks are included. Only benchmarks with matching tags will execute.', + ), + thresholds: thresholdConfigSchema, + time: z + .number() + .int() + .positive() + .describe( + 'Maximum time to spend on each benchmark task in milliseconds. Tasks will run at least until this duration or iteration count is reached, depending on limitBy setting.', + ), + timeout: z + .number() + .int() + .positive() + .describe( + 'Timeout for individual benchmark tasks in milliseconds. Tasks exceeding this duration will be terminated and marked as failed.', + ), + verbose: z + .boolean() + .describe( + 'Enable verbose output. Provides more detailed console output including progress, intermediate results, and diagnostic information', + ), + warmup: z + .number() + .int() + .nonnegative() + .describe( + 'Number of warmup iterations to run before measurement begins. Warmup helps stabilize performance by allowing JIT compilation and caching to occur.', + ), }; +/** Description for the budgets field */ +const budgetsDescription = + 'Performance budgets organized by file → suite → task. Budgets define acceptable performance thresholds.'; + +/** Description and metadata for the config schema */ +const configSchemaDescription = + 'ModestBench configuration for controlling benchmark discovery, execution, and reporting'; +const configSchemaMeta = { title: 'ModestBench Configuration' }; + /** - * Transform nested budget structure to flat TaskId → Budget mapping Also parses - * time and percentage strings + * Schema for the main ModestBench configuration + * + * This is the complete configuration schema used for validating benchmark + * configuration from all sources (files, CLI args, defaults). * - * @internal + * The budgets field uses transforms to: + * + * 1. Parse string values like "10ms" or "10%" to numbers + * 2. Flatten nested structure to flat taskId → Budget mapping */ -const transformBudgets = ( - nested: Record>> | undefined, -): Record | undefined => { - if (!nested) { - return undefined; - } +const modestBenchConfigSchema = z + .object({ + ...baseConfigProperties, + budgets: budgetsSchema.optional().describe(budgetsDescription), + }) + .strict() + .describe(configSchemaDescription) + .meta(configSchemaMeta); - const flat: Record = {}; +/** + * Input schema for configuration (without transforms) + * + * This schema is used for JSON Schema generation. It validates the same input + * structure but without transforms, which can't be represented in JSON Schema + * format. + */ +const modestBenchConfigInputSchema = z + .object({ + ...baseConfigProperties, + budgets: budgetsRawInputSchema.optional().describe(budgetsDescription), + }) + .strict() + .describe(configSchemaDescription) + .meta(configSchemaMeta); - for (const [file, suites] of Object.entries(nested)) { - for (const [suite, tasks] of Object.entries(suites)) { - for (const [task, budget] of Object.entries(tasks)) { - const taskId = `${file}/${suite}/${task}`; - // Transform budget values (parse strings) - flat[taskId] = transformBudgetValues(budget as BudgetInput); - } - } - } +/** + * Partial input schema for JSON Schema generation + * + * This is used for generating JSON Schema for IDE autocomplete in config files. + */ +export const partialModestBenchConfigInputSchema = + modestBenchConfigInputSchema.partial(); - return flat; -}; +/** + * Validate a partial configuration object + * + * This is used for validating configuration from files or CLI args before + * merging with defaults. + */ +export const partialModestBenchConfigSchema: z.ZodType< + Partial +> = modestBenchConfigSchema.partial(); /** - * Safely parse and validate a partial configuration object with budget - * transformation + * Safely parse and validate a partial configuration object * * @param config - The configuration object to validate * @returns A result object with either success: true and data, or success: * false and error */ export const safeParsePartialConfig = (config: unknown) => { - const result = partialModestBenchConfigSchema.safeParse(config); - - // Transform nested budgets to flat structure after validation - if (result.success && result.data.budgets) { - return { - ...result, - data: { - ...result.data, - budgets: transformBudgets( - result.data.budgets as Record< - string, - Record> - >, - ), - }, - }; - } - - return result; + return partialModestBenchConfigSchema.safeParse(config); }; /** @@ -447,25 +466,21 @@ export const safeParsePartialConfig = (config: unknown) => { * false and error */ export const safeParseConfig = (config: unknown) => { - const result = modestBenchConfigSchema.safeParse(config); - - // Transform nested budgets to flat structure after validation - if (result.success && result.data.budgets) { - return { - ...result, - data: { - ...result.data, - budgets: transformBudgets( - result.data.budgets as Record< - string, - Record> - >, - ), - }, - }; - } - - return result; + return modestBenchConfigSchema.safeParse(config); }; +/** + * Configuration type after parsing (output type) + * + * This is the type you get after parsing a config file - budgets are flattened + * to taskId keys and string values are converted to numbers. + */ export type ModestBenchConfig = z.infer; + +/** + * Configuration type before parsing (input type) + * + * This is the type of config files written by users - budgets are nested (file + * → suite → task) and values can be strings like "10ms" or "10%". + */ +export type ModestBenchConfigInput = z.input; diff --git a/src/core/engine.ts b/src/core/engine.ts index e412b88..7bd3857 100644 --- a/src/core/engine.ts +++ b/src/core/engine.ts @@ -16,7 +16,6 @@ import type { BenchmarkRun, BenchmarkSuite, BenchmarkTask, - Budget, BudgetSummary, CiInfo, ConfigurationManager, @@ -382,7 +381,7 @@ export abstract class ModestBenchEngine implements BenchmarkEngine { // Check if any budgets use relative thresholds const hasRelativeBudgets = Object.values(config.budgets).some( - (budget) => (budget as Budget).relative, + (budget) => budget.relative, ); if (hasRelativeBudgets) { @@ -414,7 +413,7 @@ export abstract class ModestBenchEngine implements BenchmarkEngine { // Evaluate budgets budgetSummary = evaluator.evaluateRun( - config.budgets as Record, + config.budgets, taskResults, baselineData, ); diff --git a/src/services/config-manager.ts b/src/services/config-manager.ts index 1ab1045..f1867c0 100644 --- a/src/services/config-manager.ts +++ b/src/services/config-manager.ts @@ -198,10 +198,7 @@ export class ModestBenchConfigurationManager implements ConfigurationManager { ); } - // TODO: safeParseConfig transforms budgets from nested to flat format, - // but the schema type doesn't reflect this. Fix by defining a separate - // output type for the transformed config. (see #15) - return validation.data as ModestBenchConfig; + return validation.data; } catch (error) { // Re-throw our custom errors if ( diff --git a/src/services/file-loader.ts b/src/services/file-loader.ts index 3f290dc..4fd8588 100644 --- a/src/services/file-loader.ts +++ b/src/services/file-loader.ts @@ -18,11 +18,11 @@ import type { ValidationWarning, } from '../types/index.js'; +import { benchmarkFileSchema } from '../config/benchmark-schema.js'; import { BENCHMARK_FILE_EXTENSIONS, BENCHMARK_FILE_PATTERN, } from '../constants.js'; -import { benchmarkFileSchema } from '../core/benchmark-schema.js'; import { FileDiscoveryError, FileLoadError, diff --git a/src/types/core.ts b/src/types/core.ts index f3a3c31..8c75068 100644 --- a/src/types/core.ts +++ b/src/types/core.ts @@ -3,6 +3,7 @@ import type { z } from 'zod'; import type { jsonReporterConfigSchema, ModestBenchConfig, + ModestBenchConfigInput, reporterConfigSchema, } from '../config/schema.js'; // Budget-related types @@ -80,7 +81,7 @@ export type { TaskId, }; -export type { ModestBenchConfig }; +export type { ModestBenchConfig, ModestBenchConfigInput }; // Re-export schema-derived types export type { @@ -90,7 +91,7 @@ export type { BenchmarkSuiteInput, BenchmarkTask, BenchmarkTaskInput, -} from '../core/benchmark-schema.js'; +} from '../config/benchmark-schema.js'; /** * CI/CD environment information @@ -399,6 +400,8 @@ export interface TaskResult { /** * Threshold configuration for performance assertions + * + * TODO: Derive from thresholdConfigSchema via z.infer (see #15) */ export interface ThresholdConfig { /** Maximum allowed margin of error percentage */ diff --git a/test/contract/loader-format-validation.test.ts b/test/contract/loader-format-validation.test.ts index 9e1ac7d..11e7ba9 100644 --- a/test/contract/loader-format-validation.test.ts +++ b/test/contract/loader-format-validation.test.ts @@ -10,7 +10,7 @@ import { describe, it } from 'node:test'; import type { BenchmarkSuite } from '../../src/types/core.js'; -import { benchmarkFileSchema } from '../../src/core/benchmark-schema.js'; +import { benchmarkFileSchema } from '../../src/config/benchmark-schema.js'; describe('Benchmark file format validation', () => { describe('traditional suite-based format', () => {