diff --git a/src/cli/commands/run.ts b/src/cli/commands/run.ts index 2e413ab..d93dbaf 100644 --- a/src/cli/commands/run.ts +++ b/src/cli/commands/run.ts @@ -11,6 +11,7 @@ import type { BenchmarkRun, ModestBenchConfig, Reporter, + ReporterConfig, } from '../../types/index.js'; import type { CliContext } from '../index.js'; @@ -61,6 +62,7 @@ interface RunOptions { excludeTags?: string[] | undefined; iterations?: number | undefined; json?: boolean | undefined; + jsonPretty?: boolean | undefined; noColor?: boolean | undefined; outputDir?: string | undefined; outputFile?: string | undefined; @@ -126,6 +128,7 @@ export const handleRunCommand = async ( options.outputDir, options.outputFile, options.progress, + options.jsonPretty, ); // Step 3: Discovery phase @@ -389,7 +392,7 @@ const setupReporters = async ( context: CliContext, config: { outputDir?: string; - reporterConfig?: Record; + reporterConfig?: ReporterConfig; reporters?: string[]; }, isVerbose: boolean, @@ -398,6 +401,7 @@ const setupReporters = async ( explicitOutputDir?: string, explicitOutputFile?: string, progressOption?: boolean, + explicitJsonPretty?: boolean, ): Promise => { try { const reporters: Reporter[] = []; @@ -453,9 +457,14 @@ const setupReporters = async ( explicitOutputFile, 'results.json', ); + // Precedence: CLI flag > config file > default (false) + const prettyPrint = + explicitJsonPretty ?? + config.reporterConfig?.json?.prettyPrint ?? + false; reporter = new JsonReporter({ ...(outputPath ? { outputPath } : {}), - prettyPrint: true, + prettyPrint, }); break; } diff --git a/src/cli/index.ts b/src/cli/index.ts index 05d359c..d7b4de0 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -276,6 +276,12 @@ export const main = async ( 'Benchmark engine: tinybench (default) or accurate (requires --allow-natives-syntax)', type: 'string', }) + .option('json-pretty', { + defaultDescription: 'false', + description: + 'Pretty-print JSON output (only affects json reporter)', + type: 'boolean', + }) .example([ ['$0 run', 'Run benchmarks in current directory and bench/'], ['$0 run benchmarks/', 'Run all benchmarks in a directory'], @@ -315,6 +321,7 @@ export const main = async ( excludeTags: argv['exclude-tag'], iterations: argv.iterations, json: argv.json, + jsonPretty: argv['json-pretty'], noColor: argv.noColor, outputDir: argv.output, outputFile: argv['output-file'], @@ -1133,7 +1140,7 @@ const createCliContext = async ( engine.registerReporter( 'json', new JsonReporter({ - prettyPrint: true, + prettyPrint: false, }), ); diff --git a/src/config/schema.ts b/src/config/schema.ts index 2fafec7..73bd582 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -8,11 +8,36 @@ import * as z from 'zod'; -import type { ModestBenchConfig } from '../types/core.js'; - import { BENCHMARK_FILE_PATTERN } from '../constants.js'; import { parsePercentageString, parseTimeString } from './budget-schema.js'; +/** + * Schema for JSON reporter configuration options + */ +export const jsonReporterConfigSchema = z.object({ + prettyPrint: z + .boolean() + .optional() + .describe('Whether to pretty-print JSON output (default: false)'), +}); + +/** + * Schema for reporter-specific configuration + * + * Allows typed configuration for known reporters while permitting unknown + * reporter configs via catchall. + */ +export const reporterConfigSchema = z + .object({ + json: jsonReporterConfigSchema + .optional() + .describe('Configuration options for the JSON reporter'), + }) + .catchall(z.unknown()) + .describe( + 'Configuration options specific to individual reporters, keyed by reporter name', + ); + /** * Schema for threshold configuration * @@ -226,11 +251,7 @@ const modestBenchConfigSchema = z .describe( 'Run in quiet mode with minimal console output (only errors and final results)', ), - reporterConfig: z - .record(z.string(), z.unknown()) - .describe( - 'Configuration options specific to individual reporters, keyed by reporter name', - ), + reporterConfig: reporterConfigSchema, reporters: z .array(z.string()) .min(1) @@ -446,3 +467,5 @@ export const safeParseConfig = (config: unknown) => { return result; }; + +export type ModestBenchConfig = z.infer; diff --git a/src/reporters/json.ts b/src/reporters/json.ts index 6129871..28930e1 100644 --- a/src/reporters/json.ts +++ b/src/reporters/json.ts @@ -90,7 +90,7 @@ export class JsonReporter extends BaseReporter { super('json', options); this.outputPath = options.outputPath; - this.prettyPrint = options.prettyPrint ?? true; + this.prettyPrint = options.prettyPrint ?? false; this.includeStatistics = options.includeStatistics ?? true; this.includeMetadata = options.includeMetadata ?? true; } diff --git a/src/services/config-manager.ts b/src/services/config-manager.ts index f1867c0..1ab1045 100644 --- a/src/services/config-manager.ts +++ b/src/services/config-manager.ts @@ -198,7 +198,10 @@ export class ModestBenchConfigurationManager implements ConfigurationManager { ); } - return validation.data; + // TODO: safeParseConfig transforms budgets from nested to flat format, + // but the schema type doesn't reflect this. Fix by defining a separate + // output type for the transformed config. (see #15) + return validation.data as ModestBenchConfig; } catch (error) { // Re-throw our custom errors if ( diff --git a/src/types/core.ts b/src/types/core.ts index abddcc4..f3a3c31 100644 --- a/src/types/core.ts +++ b/src/types/core.ts @@ -1,3 +1,10 @@ +import type { z } from 'zod'; + +import type { + jsonReporterConfigSchema, + ModestBenchConfig, + reporterConfigSchema, +} from '../config/schema.js'; // Budget-related types import type { AbsoluteBudget, @@ -13,30 +20,6 @@ import type { TaskId, } from './budgets.js'; -export type { - AbsoluteBudget, - BaselineReference, - BaselineStorage, - BaselineSummaryData, - Budget, - BudgetResult, - BudgetSummary, - BudgetViolation, - RelativeBudget, - RunId, - TaskId, -}; - -// Re-export schema-derived types -export type { - BenchmarkDefinition, - BenchmarkDefinitionInput, - BenchmarkSuite, - BenchmarkSuiteInput, - BenchmarkTask, - BenchmarkTaskInput, -} from '../core/benchmark-schema.js'; - /** * Benchmark file structure after parsing */ @@ -51,21 +34,6 @@ export interface BenchmarkFile { readonly metadata: FileMetadata; } -/** - * ModestBench Core Types - * - * Defines the fundamental data structures used throughout the ModestBench - * system. These types represent benchmark results, metadata, configuration, and - * system state. - * - * Note: BenchmarkDefinition, BenchmarkSuite, and BenchmarkTask types are - * derived from Zod schemas and re-exported from benchmark-schema.ts for type - * safety and consistency. - */ - -// Re-export identifier helper functions -export { createRunId, createTaskId } from '../utils/identifiers.js'; - /** * Represents a complete benchmark run across multiple files */ @@ -98,6 +66,32 @@ export interface BenchmarkRun { readonly tags?: string[]; } +export type { + AbsoluteBudget, + BaselineReference, + BaselineStorage, + BaselineSummaryData, + Budget, + BudgetResult, + BudgetSummary, + BudgetViolation, + RelativeBudget, + RunId, + TaskId, +}; + +export type { ModestBenchConfig }; + +// Re-export schema-derived types +export type { + BenchmarkDefinition, + BenchmarkDefinitionInput, + BenchmarkSuite, + BenchmarkSuiteInput, + BenchmarkTask, + BenchmarkTaskInput, +} from '../core/benchmark-schema.js'; + /** * CI/CD environment information */ @@ -116,6 +110,21 @@ export interface CiInfo { readonly pullRequest?: string; } +/** + * ModestBench Core Types + * + * Defines the fundamental data structures used throughout the ModestBench + * system. These types represent benchmark results, metadata, configuration, and + * system state. + * + * Note: BenchmarkDefinition, BenchmarkSuite, and BenchmarkTask types are + * derived from Zod schemas and re-exported from benchmark-schema.ts for type + * safety and consistency. + */ + +// Re-export identifier helper functions +export { createRunId, createTaskId } from '../utils/identifiers.js'; + /** * CPU information */ @@ -277,6 +286,11 @@ export interface GitInfo { readonly timestamp: Date; } +/** + * Configuration options for the JSON reporter + */ +export type JsonReporterConfig = z.infer; + /** * Memory information */ @@ -290,68 +304,12 @@ export interface MemoryInfo { } /** - * Benchmark configuration + * Reporter-specific configuration * - * The JSON Schema for this configuration is available at - * `dist/schema/modestbench-config.schema.json` after building the project. - * - * Config files can optionally include a `$schema` property pointing to the - * schema file for IDE autocomplete and validation support. - * - * @example - * - * ```json - * { - * "$schema": "./node_modules/modestbench/dist/schema/modestbench-config.schema.json", - * "iterations": 1000, - * "reporters": ["human", "json"], - * "time": 5000 - * } - * ``` + * Provides typed configuration for known reporters. Unknown reporter configs + * are allowed via index signature. */ -export interface ModestBenchConfig { - readonly $schema?: string | undefined; - /** Whether to stop on first failure */ - readonly bail: boolean; - /** Name of baseline to use for relative budget comparisons */ - readonly baseline?: string | undefined; - /** How to handle budget violations: 'fail', 'warn', or 'report' */ - readonly budgetMode?: 'fail' | 'report' | 'warn' | undefined; - /** Performance budgets mapped by task identifier */ - readonly budgets?: Record | undefined; - /** Patterns to exclude from discovery */ - readonly exclude: string[]; - /** Tags to exclude from execution */ - readonly excludeTags: string[]; - /** Default number of iterations per task */ - readonly iterations: number; - /** How to limit benchmark execution: 'time', 'iterations', 'any', or 'all' */ - readonly limitBy: 'all' | 'any' | 'iterations' | 'time'; - /** Custom metadata to attach to runs */ - readonly metadata: Record; - /** Output directory for reports (undefined means stdout for data reporters) */ - readonly outputDir?: string; - /** Pattern(s) for discovering benchmark files */ - readonly pattern: string | string[]; - /** Whether to run in quiet mode */ - readonly quiet: boolean; - /** Configuration for specific reporters */ - readonly reporterConfig: Record; - /** Reporters to use for output */ - readonly reporters: string[]; - /** Tags to include (if empty, include all) */ - readonly tags: string[]; - /** Threshold configuration for performance assertions */ - readonly thresholds: ThresholdConfig; - /** Maximum time to spend on each task in milliseconds */ - readonly time: number; - /** Timeout for individual tasks in milliseconds */ - readonly timeout: number; - /** Whether to run in verbose mode */ - readonly verbose: boolean; - /** Number of warmup iterations before measurement */ - readonly warmup: number; -} +export type ReporterConfig = z.infer; /** * Summary statistics for a benchmark run diff --git a/test/integration/json-pretty.test.ts b/test/integration/json-pretty.test.ts new file mode 100644 index 0000000..80d7c30 --- /dev/null +++ b/test/integration/json-pretty.test.ts @@ -0,0 +1,247 @@ +/** + * Integration tests for JSON reporter pretty-print configuration + * + * Tests CLI flag `--json-pretty` and config file + * `reporterConfig.json.prettyPrint` options with proper precedence handling. + * + * @packageDocumentation + */ + +import { expect } from 'bupkis'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, it } from 'node:test'; + +import { runCommand } from '../util.js'; +import { fixtures } from './fixture-paths.js'; + +describe('JSON reporter prettyPrint configuration', () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'modestbench-json-pretty-')); + }); + + afterEach(async () => { + await rm(tempDir, { force: true, recursive: true }); + }); + + describe('default behavior', () => { + it('should output compact JSON by default (no newlines in JSON body)', async () => { + const outputFile = join(tempDir, 'results.json'); + + // Create an explicit, empty config file and pass it via `--config` so that: + // - no project-/repo-level modestbench.config.json is picked up, and + // - the JSON reporter behavior observed here reflects the built-in defaults. + // The tempDir is a fresh directory created for this test, so this empty file + // effectively neutralizes any parent config and lets us validate default behavior. + const configPath = join(tempDir, 'modestbench.config.json'); + await writeFile(configPath, JSON.stringify({})); + + const result = await runCommand([ + 'run', + fixtures.simple, + '--config', + configPath, + '--reporter', + 'json', + '--output-file', + outputFile, + ]); + + expect(result.exitCode, 'to equal', 0); + + const jsonContent = await readFile(outputFile, 'utf-8'); + + // Compact JSON should be a single line (no newlines except possibly at the end) + const lines = jsonContent.trim().split('\n'); + expect(lines.length, 'to equal', 1); + + // Should still be valid JSON + const data = JSON.parse(jsonContent); + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + expect(data, 'to have keys', ['meta', 'run', 'statistics']); + }); + }); + + describe('--json-pretty CLI flag', () => { + it('should output formatted JSON when --json-pretty is specified', async () => { + const outputFile = join(tempDir, 'results.json'); + const result = await runCommand([ + 'run', + fixtures.simple, + '--reporter', + 'json', + '--json-pretty', + '--output-file', + outputFile, + ]); + + expect(result.exitCode, 'to equal', 0); + + const jsonContent = await readFile(outputFile, 'utf-8'); + + // Pretty-printed JSON should have multiple lines + const lines = jsonContent.trim().split('\n'); + expect(lines.length, 'to be greater than', 1); + + // Should still be valid JSON + const data = JSON.parse(jsonContent); + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + expect(data, 'to have keys', ['meta', 'run', 'statistics']); + }); + }); + + describe('config file reporterConfig.json.prettyPrint', () => { + it('should output formatted JSON when config sets prettyPrint: true', async () => { + const outputFile = join(tempDir, 'results.json'); + + // Create a config file with prettyPrint: true + const configPath = join(tempDir, 'modestbench.config.json'); + await writeFile( + configPath, + JSON.stringify({ + reporterConfig: { + json: { + prettyPrint: true, + }, + }, + }), + ); + + const result = await runCommand([ + 'run', + fixtures.simple, + '--config', + configPath, + '--reporter', + 'json', + '--output-file', + outputFile, + ]); + + expect(result.exitCode, 'to equal', 0); + + const jsonContent = await readFile(outputFile, 'utf-8'); + + // Pretty-printed JSON should have multiple lines + const lines = jsonContent.trim().split('\n'); + expect(lines.length, 'to be greater than', 1); + }); + + it('should output compact JSON when config sets prettyPrint: false', async () => { + const outputFile = join(tempDir, 'results.json'); + + // Create a config file with prettyPrint: false + const configPath = join(tempDir, 'modestbench.config.json'); + await writeFile( + configPath, + JSON.stringify({ + reporterConfig: { + json: { + prettyPrint: false, + }, + }, + }), + ); + + const result = await runCommand([ + 'run', + fixtures.simple, + '--config', + configPath, + '--reporter', + 'json', + '--output-file', + outputFile, + ]); + + expect(result.exitCode, 'to equal', 0); + + const jsonContent = await readFile(outputFile, 'utf-8'); + + // Compact JSON should be a single line + const lines = jsonContent.trim().split('\n'); + expect(lines.length, 'to equal', 1); + }); + }); + + describe('precedence: CLI flag overrides config', () => { + it('--json-pretty should override config prettyPrint: false', async () => { + const outputFile = join(tempDir, 'results.json'); + + // Create a config file with prettyPrint: false + const configPath = join(tempDir, 'modestbench.config.json'); + await writeFile( + configPath, + JSON.stringify({ + reporterConfig: { + json: { + prettyPrint: false, + }, + }, + }), + ); + + // CLI flag --json-pretty should override the config + const result = await runCommand([ + 'run', + fixtures.simple, + '--config', + configPath, + '--reporter', + 'json', + '--json-pretty', + '--output-file', + outputFile, + ]); + + expect(result.exitCode, 'to equal', 0); + + const jsonContent = await readFile(outputFile, 'utf-8'); + + // Should be pretty-printed despite config setting prettyPrint: false + const lines = jsonContent.trim().split('\n'); + expect(lines.length, 'to be greater than', 1); + }); + + it('--no-json-pretty should override config prettyPrint: true', async () => { + const outputFile = join(tempDir, 'results.json'); + + // Create a config file with prettyPrint: true + const configPath = join(tempDir, 'modestbench.config.json'); + await writeFile( + configPath, + JSON.stringify({ + reporterConfig: { + json: { + prettyPrint: true, + }, + }, + }), + ); + + // CLI flag --no-json-pretty should override the config + const result = await runCommand([ + 'run', + fixtures.simple, + '--config', + configPath, + '--reporter', + 'json', + '--no-json-pretty', + '--output-file', + outputFile, + ]); + + expect(result.exitCode, 'to equal', 0); + + const jsonContent = await readFile(outputFile, 'utf-8'); + + // Should be compact despite config setting prettyPrint: true + const lines = jsonContent.trim().split('\n'); + expect(lines.length, 'to equal', 1); + }); + }); +}); diff --git a/test/unit/services/config-manager.test.ts b/test/unit/services/config-manager.test.ts index 7c602d5..dcf4c2b 100644 --- a/test/unit/services/config-manager.test.ts +++ b/test/unit/services/config-manager.test.ts @@ -66,12 +66,12 @@ describe('ModestBenchConfigurationManager', () => { const manager = new ModestBenchConfigurationManager(); const result = manager.merge( { reporterConfig: { human: { color: true } } }, - { reporterConfig: { json: { pretty: false } } }, + { reporterConfig: { json: { prettyPrint: false } } }, ); expect(result.reporterConfig, 'to satisfy', { human: { color: true }, - json: { pretty: false }, + json: { prettyPrint: false }, }); });