Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions src/cli/commands/run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import type {
BenchmarkRun,
ModestBenchConfig,
Reporter,
ReporterConfig,
} from '../../types/index.js';
import type { CliContext } from '../index.js';

Expand Down Expand Up @@ -61,6 +62,7 @@ interface RunOptions {
excludeTags?: string[] | undefined;
iterations?: number | undefined;
json?: boolean | undefined;
jsonPretty?: boolean | undefined;
noColor?: boolean | undefined;
outputDir?: string | undefined;
outputFile?: string | undefined;
Expand Down Expand Up @@ -126,6 +128,7 @@ export const handleRunCommand = async (
options.outputDir,
options.outputFile,
options.progress,
options.jsonPretty,
);

// Step 3: Discovery phase
Expand Down Expand Up @@ -389,7 +392,7 @@ const setupReporters = async (
context: CliContext,
config: {
outputDir?: string;
reporterConfig?: Record<string, unknown>;
reporterConfig?: ReporterConfig;
reporters?: string[];
},
isVerbose: boolean,
Expand All @@ -398,6 +401,7 @@ const setupReporters = async (
explicitOutputDir?: string,
explicitOutputFile?: string,
progressOption?: boolean,
explicitJsonPretty?: boolean,
): Promise<Reporter[]> => {
try {
const reporters: Reporter[] = [];
Expand Down Expand Up @@ -453,9 +457,14 @@ const setupReporters = async (
explicitOutputFile,
'results.json',
);
// Precedence: CLI flag > config file > default (false)
const prettyPrint =
explicitJsonPretty ??
config.reporterConfig?.json?.prettyPrint ??
false;
reporter = new JsonReporter({
...(outputPath ? { outputPath } : {}),
prettyPrint: true,
prettyPrint,
});
break;
}
Expand Down
9 changes: 8 additions & 1 deletion src/cli/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,12 @@ export const main = async (
'Benchmark engine: tinybench (default) or accurate (requires --allow-natives-syntax)',
type: 'string',
})
.option('json-pretty', {
defaultDescription: 'false',
description:
'Pretty-print JSON output (only affects json reporter)',
type: 'boolean',
})
.example([
['$0 run', 'Run benchmarks in current directory and bench/'],
['$0 run benchmarks/', 'Run all benchmarks in a directory'],
Expand Down Expand Up @@ -315,6 +321,7 @@ export const main = async (
excludeTags: argv['exclude-tag'],
iterations: argv.iterations,
json: argv.json,
jsonPretty: argv['json-pretty'],
noColor: argv.noColor,
outputDir: argv.output,
outputFile: argv['output-file'],
Expand Down Expand Up @@ -1133,7 +1140,7 @@ const createCliContext = async (
engine.registerReporter(
'json',
new JsonReporter({
prettyPrint: true,
prettyPrint: false,
}),
);

Expand Down
37 changes: 30 additions & 7 deletions src/config/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,36 @@

import * as z from 'zod';

import type { ModestBenchConfig } from '../types/core.js';

import { BENCHMARK_FILE_PATTERN } from '../constants.js';
import { parsePercentageString, parseTimeString } from './budget-schema.js';

/**
* Schema for JSON reporter configuration options
*/
export const jsonReporterConfigSchema = z.object({
prettyPrint: z
.boolean()
.optional()
.describe('Whether to pretty-print JSON output (default: false)'),
});

/**
* Schema for reporter-specific configuration
*
* Allows typed configuration for known reporters while permitting unknown
* reporter configs via catchall.
*/
export const reporterConfigSchema = z
.object({
json: jsonReporterConfigSchema
.optional()
.describe('Configuration options for the JSON reporter'),
})
.catchall(z.unknown())
.describe(
'Configuration options specific to individual reporters, keyed by reporter name',
);

/**
* Schema for threshold configuration
*
Expand Down Expand Up @@ -226,11 +251,7 @@ const modestBenchConfigSchema = z
.describe(
'Run in quiet mode with minimal console output (only errors and final results)',
),
reporterConfig: z
.record(z.string(), z.unknown())
.describe(
'Configuration options specific to individual reporters, keyed by reporter name',
),
reporterConfig: reporterConfigSchema,
reporters: z
.array(z.string())
.min(1)
Expand Down Expand Up @@ -446,3 +467,5 @@ export const safeParseConfig = (config: unknown) => {

return result;
};

export type ModestBenchConfig = z.infer<typeof modestBenchConfigSchema>;
2 changes: 1 addition & 1 deletion src/reporters/json.ts
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ export class JsonReporter extends BaseReporter {
super('json', options);

this.outputPath = options.outputPath;
this.prettyPrint = options.prettyPrint ?? true;
this.prettyPrint = options.prettyPrint ?? false;
this.includeStatistics = options.includeStatistics ?? true;
this.includeMetadata = options.includeMetadata ?? true;
}
Expand Down
5 changes: 4 additions & 1 deletion src/services/config-manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,10 @@ export class ModestBenchConfigurationManager implements ConfigurationManager {
);
}

return validation.data;
// TODO: safeParseConfig transforms budgets from nested to flat format,
// but the schema type doesn't reflect this. Fix by defining a separate
// output type for the transformed config. (see #15)
return validation.data as ModestBenchConfig;
} catch (error) {
// Re-throw our custom errors
if (
Expand Down
156 changes: 57 additions & 99 deletions src/types/core.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,10 @@
import type { z } from 'zod';

import type {
jsonReporterConfigSchema,
ModestBenchConfig,
reporterConfigSchema,
} from '../config/schema.js';
// Budget-related types
import type {
AbsoluteBudget,
Expand All @@ -13,30 +20,6 @@ import type {
TaskId,
} from './budgets.js';

export type {
AbsoluteBudget,
BaselineReference,
BaselineStorage,
BaselineSummaryData,
Budget,
BudgetResult,
BudgetSummary,
BudgetViolation,
RelativeBudget,
RunId,
TaskId,
};

// Re-export schema-derived types
export type {
BenchmarkDefinition,
BenchmarkDefinitionInput,
BenchmarkSuite,
BenchmarkSuiteInput,
BenchmarkTask,
BenchmarkTaskInput,
} from '../core/benchmark-schema.js';

/**
* Benchmark file structure after parsing
*/
Expand All @@ -51,21 +34,6 @@ export interface BenchmarkFile {
readonly metadata: FileMetadata;
}

/**
* ModestBench Core Types
*
* Defines the fundamental data structures used throughout the ModestBench
* system. These types represent benchmark results, metadata, configuration, and
* system state.
*
* Note: BenchmarkDefinition, BenchmarkSuite, and BenchmarkTask types are
* derived from Zod schemas and re-exported from benchmark-schema.ts for type
* safety and consistency.
*/

// Re-export identifier helper functions
export { createRunId, createTaskId } from '../utils/identifiers.js';

/**
* Represents a complete benchmark run across multiple files
*/
Expand Down Expand Up @@ -98,6 +66,32 @@ export interface BenchmarkRun {
readonly tags?: string[];
}

export type {
AbsoluteBudget,
BaselineReference,
BaselineStorage,
BaselineSummaryData,
Budget,
BudgetResult,
BudgetSummary,
BudgetViolation,
RelativeBudget,
RunId,
TaskId,
};

export type { ModestBenchConfig };

// Re-export schema-derived types
export type {
BenchmarkDefinition,
BenchmarkDefinitionInput,
BenchmarkSuite,
BenchmarkSuiteInput,
BenchmarkTask,
BenchmarkTaskInput,
} from '../core/benchmark-schema.js';

/**
* CI/CD environment information
*/
Expand All @@ -116,6 +110,21 @@ export interface CiInfo {
readonly pullRequest?: string;
}

/**
* ModestBench Core Types
*
* Defines the fundamental data structures used throughout the ModestBench
* system. These types represent benchmark results, metadata, configuration, and
* system state.
*
* Note: BenchmarkDefinition, BenchmarkSuite, and BenchmarkTask types are
* derived from Zod schemas and re-exported from benchmark-schema.ts for type
* safety and consistency.
*/

// Re-export identifier helper functions
export { createRunId, createTaskId } from '../utils/identifiers.js';

/**
* CPU information
*/
Expand Down Expand Up @@ -277,6 +286,11 @@ export interface GitInfo {
readonly timestamp: Date;
}

/**
* Configuration options for the JSON reporter
*/
export type JsonReporterConfig = z.infer<typeof jsonReporterConfigSchema>;

/**
* Memory information
*/
Expand All @@ -290,68 +304,12 @@ export interface MemoryInfo {
}

/**
* Benchmark configuration
* Reporter-specific configuration
*
* The JSON Schema for this configuration is available at
* `dist/schema/modestbench-config.schema.json` after building the project.
*
* Config files can optionally include a `$schema` property pointing to the
* schema file for IDE autocomplete and validation support.
*
* @example
*
* ```json
* {
* "$schema": "./node_modules/modestbench/dist/schema/modestbench-config.schema.json",
* "iterations": 1000,
* "reporters": ["human", "json"],
* "time": 5000
* }
* ```
* Provides typed configuration for known reporters. Unknown reporter configs
* are allowed via index signature.
*/
export interface ModestBenchConfig {
readonly $schema?: string | undefined;
/** Whether to stop on first failure */
readonly bail: boolean;
/** Name of baseline to use for relative budget comparisons */
readonly baseline?: string | undefined;
/** How to handle budget violations: 'fail', 'warn', or 'report' */
readonly budgetMode?: 'fail' | 'report' | 'warn' | undefined;
/** Performance budgets mapped by task identifier */
readonly budgets?: Record<string, unknown> | undefined;
/** Patterns to exclude from discovery */
readonly exclude: string[];
/** Tags to exclude from execution */
readonly excludeTags: string[];
/** Default number of iterations per task */
readonly iterations: number;
/** How to limit benchmark execution: 'time', 'iterations', 'any', or 'all' */
readonly limitBy: 'all' | 'any' | 'iterations' | 'time';
/** Custom metadata to attach to runs */
readonly metadata: Record<string, unknown>;
/** Output directory for reports (undefined means stdout for data reporters) */
readonly outputDir?: string;
/** Pattern(s) for discovering benchmark files */
readonly pattern: string | string[];
/** Whether to run in quiet mode */
readonly quiet: boolean;
/** Configuration for specific reporters */
readonly reporterConfig: Record<string, unknown>;
/** Reporters to use for output */
readonly reporters: string[];
/** Tags to include (if empty, include all) */
readonly tags: string[];
/** Threshold configuration for performance assertions */
readonly thresholds: ThresholdConfig;
/** Maximum time to spend on each task in milliseconds */
readonly time: number;
/** Timeout for individual tasks in milliseconds */
readonly timeout: number;
/** Whether to run in verbose mode */
readonly verbose: boolean;
/** Number of warmup iterations before measurement */
readonly warmup: number;
}
export type ReporterConfig = z.infer<typeof reporterConfigSchema>;

/**
* Summary statistics for a benchmark run
Expand Down
Loading