diff --git a/README.md b/README.md index 65a7b67..877a767 100644 --- a/README.md +++ b/README.md @@ -324,10 +324,14 @@ Define budgets in your `modestbench.config.json`: { "budgetMode": "fail", "budgets": { - "benchmarks/critical.bench.js/default/parseConfig": { - "absolute": { - "maxTime": "10ms", - "minOpsPerSec": 100000 + "benchmarks/critical.bench.js": { + "default": { + "parseConfig": { + "absolute": { + "maxTime": "10ms", + "minOpsPerSec": 100000 + } + } } } } @@ -344,6 +348,35 @@ Define budgets in your `modestbench.config.json`: - **Relative Budgets**: Comparison against baseline - `maxRegression` - Maximum performance degradation (e.g., `"10%"`, `0.1`) +#### Wildcard Patterns + +Apply budgets broadly using wildcards: + +```json +{ + "budgets": { + "**/*.bench.js": { + "*": { + "*": { + "relative": { "maxRegression": "15%" } + } + } + }, + "benchmarks/critical.bench.js": { + "*": { + "*": { + "relative": { "maxRegression": "5%" } + } + } + } + } +} +``` + +- **Files**: Use glob patterns (`**/*.bench.js`, `benchmarks/*.bench.js`) +- **Suites/Tasks**: Use `*` to match any name +- **Precedence**: Most specific pattern wins (exact matches override wildcards) + **Budget Modes:** - `fail` (default) - Exit with error code if budgets fail diff --git a/astro.config.js b/astro.config.js index 88fd486..c6a3ab6 100644 --- a/astro.config.js +++ b/astro.config.js @@ -28,6 +28,12 @@ export default defineConfig({ { label: 'CLI Reference', link: '/guides/cli/' }, { label: 'Output Formats', link: '/guides/output/' }, { label: 'Understanding Statistics', link: '/guides/statistics/' }, + { + label: 'Performance Budgets', + link: '/guides/performance-budgets/', + }, + { label: 'Profiling', link: '/guides/profiling/' }, + { label: 'Test Adapters', link: '/guides/test-adapters/' }, { label: 'Advanced Usage', link: '/guides/advanced/' }, { label: 'Custom Reporters', link: '/guides/custom-reporters/' }, ], diff --git a/site/src/content/docs/guides/performance-budgets.mdx b/site/src/content/docs/guides/performance-budgets.mdx index f016724..6d288bc 100644 --- a/site/src/content/docs/guides/performance-budgets.mdx +++ b/site/src/content/docs/guides/performance-budgets.mdx @@ -149,6 +149,127 @@ Use both absolute and relative: } ``` +## Wildcard Patterns + +Instead of specifying every file, suite, and task individually, you can use wildcard patterns to apply budgets broadly: + +### Pattern Syntax + +| Level | Exact Match | Wildcard | +|-------|-------------|----------| +| File | `"api.bench.js"` | `"**/*.bench.js"` (glob) | +| Suite | `"String Operations"` | `"*"` (any suite) | +| Task | `"parseJSON"` | `"*"` (any task) | + +- **Files**: Use glob patterns (e.g., `**/*.bench.js`, `benchmarks/*.bench.js`) +- **Suites/Tasks**: Use `*` to match any name + +### Example: Global Default Budget + +Apply a budget to all benchmarks: + +```json +{ + "budgets": { + "**/*.bench.js": { + "*": { + "*": { + "relative": { + "maxRegression": "15%" + } + } + } + } + } +} +``` + +### Example: Stricter Budget for Specific Files + +Override the global default with stricter limits for critical paths: + +```json +{ + "budgets": { + "**/*.bench.js": { + "*": { + "*": { + "relative": { "maxRegression": "15%" } + } + } + }, + "benchmarks/critical.bench.js": { + "*": { + "*": { + "relative": { "maxRegression": "5%" } + } + } + } + } +} +``` + +### Example: Mixed Specificity + +Combine wildcards with exact matches for fine-grained control: + +```json +{ + "budgets": { + "**/*.bench.js": { + "*": { + "*": { + "relative": { "maxRegression": "20%" } + } + } + }, + "api.bench.js": { + "Authentication": { + "*": { + "relative": { "maxRegression": "10%" } + }, + "login": { + "absolute": { "maxTime": "50ms" }, + "relative": { "maxRegression": "5%" } + } + } + } + } +} +``` + +### Precedence Rules + +When multiple patterns match a task, the **most specific** pattern wins: + +1. **Exact matches** always take precedence over wildcards +2. **File specificity**: Exact file > partial glob > full glob +3. **Suite/task specificity**: Exact name > `*` wildcard + +**Specificity scoring:** + +| Pattern Component | Points | +|-------------------|--------| +| Exact file name | +2 | +| Glob with specific parts (e.g., `**/api/**/*.bench.js`) | +1 | +| Full glob (`**/*` or `*`) | +0 | +| Exact suite name | +1 | +| Exact task name | +1 | +| Wildcard (`*`) | +0 | + +**Example:** + +For task `api.bench.js/Authentication/login`: + +| Pattern | Specificity | Result | +|---------|-------------|--------| +| `**/*.bench.js/*/*` | 1 | Lowest priority | +| `api.bench.js/*/*` | 2 | Medium priority | +| `api.bench.js/Authentication/*` | 3 | Higher priority | +| `api.bench.js/Authentication/login` | 4 | **Wins** | + +When patterns have equal specificity, the budgets are **merged**, with later definitions overriding earlier ones. + ## Baseline Management ### Creating Baselines @@ -472,29 +593,59 @@ Adjust budget if current performance is legitimately slower. } ``` -### Example 2: Regression Guard +### Example 2: Regression Guard for All Tasks -:::note -To apply budgets to all tasks, you'll need to specify each file/suite/task. Wildcard patterns are not currently supported, but are being considered for a future release. See [issue #72](https://github.com/boneskull/modestbench/issues/72) for details. -::: +Apply a regression guard to all benchmarks using wildcards: + +```json +{ + "baseline": "production", + "budgets": { + "**/*.bench.js": { + "*": { + "*": { + "relative": { "maxRegression": "15%" } + } + } + } + } +} +``` + +### Example 3: Regression Guard with Overrides + +Apply different thresholds to different areas: ```json { "baseline": "production", "budgets": { + "**/*.bench.js": { + "*": { + "*": { + "relative": { "maxRegression": "20%" } + } + } + }, "api.bench.js": { + "*": { + "*": { + "relative": { "maxRegression": "10%" } + } + } + }, + "utils.bench.js": { "default": { "parseRequest": { - "relative": { "maxRegression": "15%" } + "relative": { "maxRegression": "5%" } } } } - // Repeat for other files/suites/tasks... } } ``` -### Example 3: Per-Task Budgets +### Example 4: Per-Task Budgets ```json { diff --git a/src/config/schema.ts b/src/config/schema.ts index e5de808..60fd10a 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -8,9 +8,17 @@ import * as z from 'zod'; -import type { Budget } from '../types/budgets.js'; +import type { + Budget, + BudgetPattern, + ResolvedBudgets, +} from '../types/budgets.js'; import { BENCHMARK_FILE_PATTERN } from '../constants.js'; +import { + createBudgetPattern, + isGlobPattern, +} from '../services/budget-resolver.js'; import { parsePercentageString, parseTimeString } from './budget-schema.js'; /** @@ -194,33 +202,64 @@ const budgetsInputSchema = z.record( ); /** - * Transform nested budget structure to flat TaskId → Budget mapping + * Check if a suite or task name is a wildcard + * + * @param name - The suite or task name + * @returns True if the name is a wildcard (`*`) + */ +const isWildcard = (name: string): boolean => name === '*'; + +/** + * Check if a budget entry contains any wildcards or glob patterns + * + * @param file - File pattern + * @param suite - Suite name or wildcard + * @param task - Task name or wildcard + * @returns True if any part contains wildcards + */ +const hasWildcards = (file: string, suite: string, task: string): boolean => { + return isGlobPattern(file) || isWildcard(suite) || isWildcard(task); +}; + +/** + * Transform nested budget structure to ResolvedBudgets with exact matches and + * patterns separated * * @param nested - Nested budgets structure (file → suite → task → budget) - * @returns Flat budgets map (taskId → budget) + * @returns ResolvedBudgets with exact matches and wildcard patterns */ const flattenBudgets = ( nested: z.infer, -): Record => { - const flat: Record = {}; +): ResolvedBudgets => { + const exact: Record = {}; + const patterns: BudgetPattern[] = []; for (const [file, suites] of Object.entries(nested)) { for (const [suite, tasks] of Object.entries(suites)) { for (const [task, budget] of Object.entries(tasks)) { - const taskId = `${file}/${suite}/${task}`; - flat[taskId] = budget; + if (hasWildcards(file, suite, task)) { + // This is a pattern budget + patterns.push(createBudgetPattern(file, suite, task, budget)); + } else { + // This is an exact match + const taskId = `${file}/${suite}/${task}`; + exact[taskId] = budget; + } } } } - return flat; + // Sort patterns by specificity descending for consistent iteration order + patterns.sort((a, b) => b.specificity - a.specificity); + + return { exact, patterns }; }; /** - * Budgets schema with transform for nested-to-flat conversion + * Budgets schema with transform for nested-to-ResolvedBudgets conversion * - * Input: { [file]: { [suite]: { [task]: Budget } } } Output: { [taskId]: Budget - * } where taskId = "file/suite/task" + * Input: { [file]: { [suite]: { [task]: Budget } } } Output: ResolvedBudgets { + * exact: { [taskId]: Budget }, patterns: BudgetPattern[] } */ const budgetsSchema = budgetsInputSchema.transform(flattenBudgets); @@ -386,7 +425,7 @@ const baseConfigProperties = { /** Description for the budgets field */ const budgetsDescription = - 'Performance budgets organized by file → suite → task. Budgets define acceptable performance thresholds.'; + 'Performance budgets organized by file → suite → task. Budgets define acceptable performance thresholds. Supports wildcards (* for suite/task, glob patterns for files).'; /** Description and metadata for the config schema */ const configSchemaDescription = @@ -402,7 +441,7 @@ const configSchemaMeta = { title: 'ModestBench Configuration' }; * The budgets field uses transforms to: * * 1. Parse string values like "10ms" or "10%" to numbers - * 2. Flatten nested structure to flat taskId → Budget mapping + * 2. Separate exact matches from wildcard patterns into ResolvedBudgets */ const modestBenchConfigSchema = z .object({ @@ -472,8 +511,8 @@ export const safeParseConfig = (config: unknown) => { /** * Configuration type after parsing (output type) * - * This is the type you get after parsing a config file - budgets are flattened - * to taskId keys and string values are converted to numbers. + * This is the type you get after parsing a config file - budgets are + * transformed to ResolvedBudgets and string values are converted to numbers. */ export type ModestBenchConfig = z.infer; diff --git a/src/core/engine.ts b/src/core/engine.ts index 7bd3857..e31433c 100644 --- a/src/core/engine.ts +++ b/src/core/engine.ts @@ -353,93 +353,99 @@ export abstract class ModestBenchEngine implements BenchmarkEngine { // Evaluate budgets if configured let budgetSummary: BudgetSummary | undefined; - if (config.budgets && Object.keys(config.budgets).length > 0) { - const evaluator = new BudgetEvaluator(); - const baselineStorage = new BaselineStorageService(process.cwd()); - - // Collect task results - const taskResults = new Map(); - - for (const file of fileResults) { - for (const suite of file.suites) { - for (const task of suite.tasks) { - if (!task.error) { - // file.filePath is already relative to cwd - const taskId = createTaskId( - file.filePath, - suite.name, - task.name, - ); - taskResults.set(taskId, task); + if (config.budgets) { + const budgets = config.budgets; + const hasBudgets = + Object.keys(budgets.exact).length > 0 || budgets.patterns.length > 0; + + if (hasBudgets) { + const evaluator = new BudgetEvaluator(); + const baselineStorage = new BaselineStorageService(process.cwd()); + + // Collect task results + const taskResults = new Map(); + + for (const file of fileResults) { + for (const suite of file.suites) { + for (const task of suite.tasks) { + if (!task.error) { + // file.filePath is already relative to cwd + const taskId = createTaskId( + file.filePath, + suite.name, + task.name, + ); + taskResults.set(taskId, task); + } } } } - } - // Load baseline data if needed for relative budgets - let baselineData: Map | undefined; + // Load baseline data if needed for relative budgets + let baselineData: Map | undefined; - // Check if any budgets use relative thresholds - const hasRelativeBudgets = Object.values(config.budgets).some( - (budget) => budget.relative, - ); + // Check if any budgets use relative thresholds + const hasRelativeBudgets = + Object.values(budgets.exact).some((budget) => budget.relative) || + budgets.patterns.some((pattern) => pattern.budget.relative); - if (hasRelativeBudgets) { - const baselineName = - config.baseline || (await baselineStorage.getDefault()); + if (hasRelativeBudgets) { + const baselineName = + config.baseline || (await baselineStorage.getDefault()); - if (baselineName) { - const baseline = await baselineStorage.getBaseline(baselineName); + if (baselineName) { + const baseline = await baselineStorage.getBaseline(baselineName); - if (baseline) { - // Cast keys to TaskId since they come from validated baseline storage - baselineData = new Map( - Object.entries(baseline.summary) as [ - TaskId, - BaselineSummaryData, - ][], - ); + if (baseline) { + // Cast keys to TaskId since they come from validated baseline storage + baselineData = new Map( + Object.entries(baseline.summary) as [ + TaskId, + BaselineSummaryData, + ][], + ); + } else { + console.warn( + `Warning: Baseline "${baselineName}" not found. Relative budgets will be skipped.`, + ); + } } else { console.warn( - `Warning: Baseline "${baselineName}" not found. Relative budgets will be skipped.`, + 'Warning: Relative budgets configured but no baseline specified. Relative budgets will be skipped.', ); } - } else { - console.warn( - 'Warning: Relative budgets configured but no baseline specified. Relative budgets will be skipped.', - ); } - } - // Evaluate budgets - budgetSummary = evaluator.evaluateRun( - config.budgets, - taskResults, - baselineData, - ); + // Evaluate budgets + budgetSummary = evaluator.evaluateRun( + budgets, + taskResults, + baselineData, + ); - // Notify reporters of budget results - for (const reporter of reporters) { - if (reporter.onBudgetResult) { - await reporter.onBudgetResult(budgetSummary); + // Notify reporters of budget results + for (const reporter of reporters) { + if (reporter.onBudgetResult) { + await reporter.onBudgetResult(budgetSummary); + } } - } - // Handle budget failures based on budgetMode - if (budgetSummary.failed > 0) { - const mode = config.budgetMode || 'fail'; + // Handle budget failures based on budgetMode + if (budgetSummary.failed > 0) { + const mode = config.budgetMode || 'fail'; - if (mode === 'fail') { - throw new BudgetExceededError( - `${budgetSummary.failed} of ${budgetSummary.total} budget(s) exceeded`, - budgetSummary, - ); - } else if (mode === 'warn') { - console.warn( - `Warning: ${budgetSummary.failed} of ${budgetSummary.total} budget(s) exceeded`, - ); + if (mode === 'fail') { + throw new BudgetExceededError( + `${budgetSummary.failed} of ${budgetSummary.total} budget(s) exceeded`, + budgetSummary, + ); + } else if (mode === 'warn') { + console.warn( + `Warning: ${budgetSummary.failed} of ${budgetSummary.total} budget(s) exceeded`, + ); + } + // mode === 'report': just include in output, don't fail } - // mode === 'report': just include in output, don't fail } } diff --git a/src/services/budget-evaluator.ts b/src/services/budget-evaluator.ts index 592e715..9c38ead 100644 --- a/src/services/budget-evaluator.ts +++ b/src/services/budget-evaluator.ts @@ -4,10 +4,13 @@ import type { BudgetResult, BudgetSummary, BudgetViolation, + ResolvedBudgets, TaskId, TaskResult, } from '../types/core.js'; +import { resolveBudget } from './budget-resolver.js'; + /** * Service for evaluating performance budgets * @@ -49,30 +52,31 @@ export class BudgetEvaluator { * Evaluate budgets for an entire benchmark run */ evaluateRun( - budgets: Record, + budgets: ResolvedBudgets, taskResults: Map, baselineData?: Map, ): BudgetSummary { const results: BudgetResult[] = []; - for (const [taskId, budget] of Object.entries(budgets)) { - const taskResult = taskResults.get(taskId as TaskId); + for (const [taskId, taskResult] of taskResults) { + const budget = resolveBudget(taskId, budgets); - // Skip if no result for this task - if (!taskResult) { + // Skip if no budget matches this task + if (!budget) { continue; } - // Skip relative budgets if no baseline data - if (budget.relative && !baselineData) { + // Skip if budget has ONLY relative thresholds and no baseline data + // (absolute budgets can still be evaluated without baseline) + if (budget.relative && !budget.absolute && !baselineData) { continue; } const budgetResult = this.evaluateTask( - taskId as TaskId, + taskId, budget, taskResult, - baselineData?.get(taskId as TaskId), + baselineData?.get(taskId), ); results.push(budgetResult); diff --git a/src/services/budget-resolver.ts b/src/services/budget-resolver.ts new file mode 100644 index 0000000..6eedd08 --- /dev/null +++ b/src/services/budget-resolver.ts @@ -0,0 +1,254 @@ +/** + * Budget resolution service for matching budgets to tasks + * + * This module provides pattern-based budget resolution using: + * + * - Minimatch glob patterns for file matching + * - Simple `*` wildcards for suite/task matching + * + * @packageDocumentation + */ + +import { minimatch } from 'minimatch'; + +import type { + Budget, + BudgetPattern, + ResolvedBudgets, + TaskId, +} from '../types/core.js'; + +/** + * Check if a glob pattern contains wildcards + * + * @param pattern - The pattern to check + * @returns True if the pattern contains glob metacharacters + */ +export const isGlobPattern = (pattern: string): boolean => { + return /[*?[\]]/.test(pattern); +}; + +/** + * Check if a file path matches a glob pattern + * + * @param pattern - Minimatch glob pattern + * @param filePath - File path to match against + * @returns True if the pattern matches the file path + */ +export const matchesFile = (pattern: string, filePath: string): boolean => { + // Exact match fast path + if (pattern === filePath) { + return true; + } + + return minimatch(filePath, pattern, { matchBase: true }); +}; + +/** + * Check if a suite or task name matches a pattern + * + * @param pattern - Either an exact name or `*` for wildcard + * @param value - The value to match against + * @returns True if the pattern matches the value + */ +export const matchesSuiteOrTask = (pattern: string, value: string): boolean => { + return pattern === '*' || pattern === value; +}; + +/** + * Calculate specificity score for a budget pattern + * + * Higher scores indicate more specific patterns. Scoring: + * + * - File: +2 for exact match, +1 for glob with specific parts, +0 for `**\/*` + * - Suite: +1 for exact match, +0 for `*` + * - Task: +1 for exact match, +0 for `*` + * + * @param pattern - The budget pattern to score + * @returns Specificity score (0-4) + */ +export const calculateSpecificity = ( + pattern: Pick, +): number => { + let score = 0; + + // File specificity + if (!isGlobPattern(pattern.filePattern)) { + // Exact file match + score += 2; + } else if (pattern.filePattern !== '**/*' && pattern.filePattern !== '*') { + // Glob with some specificity (e.g., "**/api/**/*.bench.js") + score += 1; + } + // else: fully generic glob like "**/*" gets +0 + + // Suite specificity + if (pattern.suitePattern !== '*') { + score += 1; + } + + // Task specificity + if (pattern.taskPattern !== '*') { + score += 1; + } + + return score; +}; + +/** + * Parse a TaskId into its components + * + * TaskIds have the format: `{filePath}/{suiteName}/{taskName}` The file path + * can contain slashes, so we parse from the end. + * + * @param taskId - The TaskId to parse + * @returns Object with file, suite, and task components + */ +export const parseTaskId = ( + taskId: TaskId, +): { file: string; suite: string; task: string } => { + const str = taskId as string; + const lastSlash = str.lastIndexOf('/'); + const secondLastSlash = str.lastIndexOf('/', lastSlash - 1); + + return { + file: str.substring(0, secondLastSlash), + suite: str.substring(secondLastSlash + 1, lastSlash), + task: str.substring(lastSlash + 1), + }; +}; + +/** + * Deep merge two budget objects + * + * More specific (second) budget values override less specific (first) values. + * Merges at the absolute/relative level. + * + * @param base - Base budget (less specific) + * @param override - Override budget (more specific) + * @returns Merged budget + */ +export const mergeBudgets = (base: Budget, override: Budget): Budget => { + return { + ...(base.absolute || override.absolute + ? { + absolute: { + ...base.absolute, + ...override.absolute, + }, + } + : {}), + ...(base.relative || override.relative + ? { + relative: { + ...base.relative, + ...override.relative, + }, + } + : {}), + }; +}; + +/** + * Resolve the appropriate budget for a task + * + * Resolution order: + * + * 1. Check exact match in `exact` map (highest priority) + * 2. Find all matching patterns + * 3. Sort by specificity (ascending) + * 4. Merge matched budgets (more specific overrides less specific) + * + * @param taskId - The task identifier to resolve a budget for + * @param budgets - The resolved budgets structure + * @returns The resolved budget, or undefined if no budget matches + */ +export const resolveBudget = ( + taskId: TaskId, + budgets: ResolvedBudgets, +): Budget | undefined => { + // Fast path: exact match + const exactMatch = budgets.exact[taskId as string]; + if (exactMatch) { + // Still need to check patterns and merge if there are less-specific matches + const parsed = parseTaskId(taskId); + const matchingPatterns = budgets.patterns.filter( + (p) => + matchesFile(p.filePattern, parsed.file) && + matchesSuiteOrTask(p.suitePattern, parsed.suite) && + matchesSuiteOrTask(p.taskPattern, parsed.task), + ); + + if (matchingPatterns.length === 0) { + return exactMatch; + } + + // Sort by specificity ascending (least specific first, so more specific can override) + const sorted = [...matchingPatterns].sort( + (a, b) => a.specificity - b.specificity, + ); + + // Merge all patterns, then apply exact match last + let merged = sorted[0]!.budget; + for (let i = 1; i < sorted.length; i++) { + merged = mergeBudgets(merged, sorted[i]!.budget); + } + + // Exact match has highest priority + return mergeBudgets(merged, exactMatch); + } + + // Find all matching patterns + const parsed = parseTaskId(taskId); + const matchingPatterns = budgets.patterns.filter( + (p) => + matchesFile(p.filePattern, parsed.file) && + matchesSuiteOrTask(p.suitePattern, parsed.suite) && + matchesSuiteOrTask(p.taskPattern, parsed.task), + ); + + if (matchingPatterns.length === 0) { + return undefined; + } + + // Sort by specificity ascending (least specific first) + const sorted = [...matchingPatterns].sort( + (a, b) => a.specificity - b.specificity, + ); + + // Merge all matches (more specific overrides less specific) + let merged = sorted[0]!.budget; + for (let i = 1; i < sorted.length; i++) { + merged = mergeBudgets(merged, sorted[i]!.budget); + } + + return merged; +}; + +/** + * Create a BudgetPattern from its components + * + * @param filePattern - Glob pattern for file matching + * @param suitePattern - Suite name or `*` for wildcard + * @param taskPattern - Task name or `*` for wildcard + * @param budget - The budget to apply + * @returns A BudgetPattern with computed specificity + */ +export const createBudgetPattern = ( + filePattern: string, + suitePattern: string, + taskPattern: string, + budget: Budget, +): BudgetPattern => { + return { + budget, + filePattern, + specificity: calculateSpecificity({ + filePattern, + suitePattern, + taskPattern, + }), + suitePattern, + taskPattern, + }; +}; diff --git a/src/types/budgets.ts b/src/types/budgets.ts index 7da2443..78c6e18 100644 --- a/src/types/budgets.ts +++ b/src/types/budgets.ts @@ -80,6 +80,30 @@ export interface Budget { readonly relative?: RelativeBudget; } +/** + * A budget pattern with glob support for files and simple wildcards for + * suite/task + * + * File patterns use minimatch glob syntax (e.g., `**\/*.bench.js`). Suite/task + * patterns use simple `*` wildcard (matches any value). + */ +export interface BudgetPattern { + /** The budget to apply when this pattern matches */ + readonly budget: Budget; + + /** Glob pattern for file matching (minimatch syntax) */ + readonly filePattern: string; + + /** Computed specificity score (higher = more specific) */ + readonly specificity: number; + + /** Suite name or `*` for wildcard */ + readonly suitePattern: string; + + /** Task name or `*` for wildcard */ + readonly taskPattern: string; +} + /** * Budget evaluation result for a single task */ @@ -159,6 +183,20 @@ export interface RelativeBudget { readonly maxRegression?: number; } +/** + * Resolved budgets structure with exact matches and patterns separated + * + * During evaluation, exact matches are checked first, then patterns are matched + * in order of specificity (highest first). + */ +export interface ResolvedBudgets { + /** Exact TaskId matches (no wildcards or globs) */ + readonly exact: Record; + + /** Patterns with wildcards/globs, sorted by specificity descending */ + readonly patterns: readonly BudgetPattern[]; +} + /** * Branded type for benchmark run identifiers * diff --git a/src/types/core.ts b/src/types/core.ts index 8c75068..69fdc17 100644 --- a/src/types/core.ts +++ b/src/types/core.ts @@ -13,10 +13,12 @@ import type { BaselineStorage, BaselineSummaryData, Budget, + BudgetPattern, BudgetResult, BudgetSummary, BudgetViolation, RelativeBudget, + ResolvedBudgets, RunId, TaskId, } from './budgets.js'; @@ -73,10 +75,12 @@ export type { BaselineStorage, BaselineSummaryData, Budget, + BudgetPattern, BudgetResult, BudgetSummary, BudgetViolation, RelativeBudget, + ResolvedBudgets, RunId, TaskId, }; diff --git a/test/integration/budget-wildcards.test.ts b/test/integration/budget-wildcards.test.ts new file mode 100644 index 0000000..08b1016 --- /dev/null +++ b/test/integration/budget-wildcards.test.ts @@ -0,0 +1,326 @@ +/** + * Integration tests for budget wildcard patterns + * + * These tests verify the end-to-end flow of budget wildcard configuration, from + * parsing through to evaluation. + */ + +import { expect } from 'bupkis'; +import { describe, it } from 'node:test'; + +import type { TaskId, TaskResult } from '../../src/types/core.js'; + +import { safeParsePartialConfig } from '../../src/config/schema.js'; +import { BudgetEvaluator } from '../../src/services/budget-evaluator.js'; +import { createTaskId } from '../../src/utils/identifiers.js'; + +describe('Budget Wildcards Integration', () => { + describe('configuration transformation', () => { + it('should transform wildcard config with both exact and pattern budgets', () => { + const config = { + budgets: { + '**/*': { + '*': { + '*': { + relative: { maxRegression: 0.15 }, + }, + }, + }, + 'test.bench.js': { + default: { + fastTask: { + absolute: { maxTime: 5_000_000 }, + }, + }, + }, + }, + }; + + const result = safeParsePartialConfig(config); + expect(result.success, 'to be true'); + + if (result.success && result.data.budgets) { + const resolved = result.data.budgets; + + // Should have one exact match + expect(Object.keys(resolved.exact).length, 'to equal', 1); + expect( + resolved.exact['test.bench.js/default/fastTask'], + 'to be defined', + ); + expect( + resolved.exact['test.bench.js/default/fastTask']?.absolute?.maxTime, + 'to equal', + 5_000_000, + ); + + // Should have one pattern + expect(resolved.patterns.length, 'to equal', 1); + expect(resolved.patterns[0]!.filePattern, 'to equal', '**/*'); + expect( + resolved.patterns[0]!.budget.relative?.maxRegression, + 'to equal', + 0.15, + ); + } + }); + }); + + describe('pattern precedence in configuration', () => { + it('should correctly order patterns by specificity', () => { + const config = { + budgets: { + // Most generic (specificity 0) + '**/*': { + '*': { + '*': { relative: { maxRegression: 0.2 } }, + }, + }, + // Glob with specificity (specificity 1) + '**/*.bench.js': { + '*': { + '*': { absolute: { maxTime: 100_000_000 } }, + }, + }, + // Exact file with wildcard suite (specificity 2) + 'test.bench.js': { + '*': { + '*': { absolute: { maxTime: 50_000_000 } }, + }, + // Exact file + exact suite (specificity 3) + Performance: { + '*': { absolute: { maxTime: 10_000_000 } }, + }, + }, + }, + }; + + const result = safeParsePartialConfig(config); + expect(result.success, 'to be true'); + + if (result.success && result.data.budgets) { + const resolved = result.data.budgets; + + // All are patterns (have wildcards in suite or task) + expect(resolved.patterns.length, 'to be greater than', 0); + + // Verify sorted by specificity descending + for (let i = 0; i < resolved.patterns.length - 1; i++) { + expect( + resolved.patterns[i]!.specificity, + 'to be greater than or equal to', + resolved.patterns[i + 1]!.specificity, + ); + } + } + }); + }); + + describe('budget evaluation with wildcards', () => { + it('should apply wildcard budgets to matching tasks', () => { + const config = { + budgets: { + '**/*': { + '*': { + '*': { + absolute: { maxTime: 100_000_000 }, // 100ms for all + }, + }, + }, + }, + }; + + const result = safeParsePartialConfig(config); + expect(result.success, 'to be true'); + + if (result.success && result.data.budgets) { + const resolved = result.data.budgets; + const evaluator = new BudgetEvaluator(); + + // Create some task results + const taskResults = new Map([ + [ + createTaskId('api.bench.js/User Routes/getUser'), + { + mean: 50_000_000, // 50ms - under budget + opsPerSecond: 20, + p99: 60_000_000, + } as TaskResult, + ], + [ + createTaskId('db.bench.js/Query/selectAll'), + { + mean: 150_000_000, // 150ms - over budget + opsPerSecond: 6.67, + p99: 180_000_000, + } as TaskResult, + ], + ]); + + const summary = evaluator.evaluateRun(resolved, taskResults); + + // One should pass, one should fail + expect(summary.total, 'to equal', 2); + expect(summary.passed, 'to equal', 1); + expect(summary.failed, 'to equal', 1); + } + }); + + it('should merge budgets from multiple matching patterns', () => { + const config = { + budgets: { + // Global: set max regression + '**/*': { + '*': { + '*': { + relative: { maxRegression: 0.15 }, + }, + }, + }, + // More specific: add absolute time constraint + 'api.bench.js': { + '*': { + '*': { + absolute: { maxTime: 50_000_000 }, + }, + }, + }, + }, + }; + + const result = safeParsePartialConfig(config); + expect(result.success, 'to be true'); + + if (result.success && result.data.budgets) { + const resolved = result.data.budgets; + const evaluator = new BudgetEvaluator(); + + const taskResults = new Map([ + [ + createTaskId('api.bench.js/User Routes/getUser'), + { + mean: 100_000_000, // 100ms - over absolute but under regression + opsPerSecond: 10, + p99: 120_000_000, + } as TaskResult, + ], + ]); + + // Note: Without baseline data, relative budgets are skipped + // So only absolute budget (maxTime) should be evaluated + const summary = evaluator.evaluateRun(resolved, taskResults); + + expect(summary.total, 'to equal', 1); + expect(summary.failed, 'to equal', 1); // Should fail absolute budget + + // Verify violation is for maxTime + const violation = summary.results[0]!.violations[0]; + expect(violation?.type, 'to equal', 'maxTime'); + } + }); + + it('should let exact match override pattern budgets', () => { + const config = { + budgets: { + // Global: strict time limit + '**/*': { + '*': { + '*': { + absolute: { maxTime: 10_000_000 }, // 10ms + }, + }, + }, + // Specific task: more lenient + 'slow.bench.js': { + Performance: { + heavyTask: { + absolute: { maxTime: 100_000_000 }, // 100ms + }, + }, + }, + }, + }; + + const result = safeParsePartialConfig(config); + expect(result.success, 'to be true'); + + if (result.success && result.data.budgets) { + const resolved = result.data.budgets; + const evaluator = new BudgetEvaluator(); + + const taskResults = new Map([ + [ + createTaskId('slow.bench.js/Performance/heavyTask'), + { + mean: 50_000_000, // 50ms - over global, under specific + opsPerSecond: 20, + p99: 60_000_000, + } as TaskResult, + ], + ]); + + const summary = evaluator.evaluateRun(resolved, taskResults); + + // Should pass because exact match (100ms) overrides global (10ms) + expect(summary.total, 'to equal', 1); + expect(summary.passed, 'to equal', 1); + } + }); + + it('should match file patterns with minimatch glob syntax', () => { + const config = { + budgets: { + '**/api/**/*.bench.js': { + '*': { + '*': { + absolute: { maxTime: 50_000_000 }, + }, + }, + }, + }, + }; + + const result = safeParsePartialConfig(config); + expect(result.success, 'to be true'); + + if (result.success && result.data.budgets) { + const resolved = result.data.budgets; + const evaluator = new BudgetEvaluator(); + + const taskResults = new Map([ + // Should match + [ + createTaskId('benchmarks/api/users.bench.js/Routes/getUser'), + { + mean: 100_000_000, // Over budget + opsPerSecond: 10, + p99: 120_000_000, + } as TaskResult, + ], + // Should NOT match (not in api directory) + [ + createTaskId('benchmarks/db/queries.bench.js/Queries/selectAll'), + { + mean: 100_000_000, + opsPerSecond: 10, + p99: 120_000_000, + } as TaskResult, + ], + ]); + + const summary = evaluator.evaluateRun(resolved, taskResults); + + // Only the api benchmark should be evaluated + expect(summary.total, 'to equal', 1); + expect(summary.failed, 'to equal', 1); + + // Verify it's the api task + expect( + summary.results[0]!.taskId, + 'to equal', + 'benchmarks/api/users.bench.js/Routes/getUser', + ); + } + }); + }); +}); diff --git a/test/unit/config/budget-config-integration.test.ts b/test/unit/config/budget-config-integration.test.ts index 765c1c4..5a8ccf1 100644 --- a/test/unit/config/budget-config-integration.test.ts +++ b/test/unit/config/budget-config-integration.test.ts @@ -24,12 +24,10 @@ describe('Budget Configuration Schema Integration', () => { const result = safeParsePartialConfig(config); expect(result.success, 'to be true'); - // Verify transform flattened to TaskId keys + // Verify transform created ResolvedBudgets with exact matches if (result.success && result.data.budgets) { - expect( - result.data.budgets['test.bench.js/default/task'], - 'to be defined', - ); + const resolved = result.data.budgets; + expect(resolved.exact['test.bench.js/default/task'], 'to be defined'); } }); @@ -265,10 +263,11 @@ describe('Budget Configuration Schema Integration', () => { const result = safeParsePartialConfig(config); expect(result.success, 'to be true'); - // Verify transform worked + // Verify transform created ResolvedBudgets with exact matches if (result.success && result.data.budgets) { + const resolved = result.data.budgets; expect( - result.data.budgets['test.bench.js/default/criticalPath'], + resolved.exact['test.bench.js/default/criticalPath'], 'to be defined', ); } @@ -305,22 +304,23 @@ describe('Budget Configuration Schema Integration', () => { const result = safeParsePartialConfig(config); expect(result.success, 'to be true'); - // Verify all tasks were flattened + // Verify all tasks were transformed to ResolvedBudgets if (result.success && result.data.budgets) { + const resolved = result.data.budgets; expect( - result.data.budgets['api.bench.js/HTTP Routes/getUser'], + resolved.exact['api.bench.js/HTTP Routes/getUser'], 'to be defined', ); expect( - result.data.budgets['api.bench.js/HTTP Routes/createUser'], + resolved.exact['api.bench.js/HTTP Routes/createUser'], 'to be defined', ); expect( - result.data.budgets['api.bench.js/default/healthCheck'], + resolved.exact['api.bench.js/default/healthCheck'], 'to be defined', ); expect( - result.data.budgets['db.bench.js/Query Performance/selectOne'], + resolved.exact['db.bench.js/Query Performance/selectOne'], 'to be defined', ); } diff --git a/test/unit/config/budget-wildcards.test.ts b/test/unit/config/budget-wildcards.test.ts new file mode 100644 index 0000000..c011402 --- /dev/null +++ b/test/unit/config/budget-wildcards.test.ts @@ -0,0 +1,312 @@ +import { expect } from 'bupkis'; +import { describe, it } from 'node:test'; + +import { safeParsePartialConfig } from '../../../src/config/schema.js'; + +describe('Budget Wildcard Configuration', () => { + describe('pattern detection', () => { + it('should categorize exact file/suite/task as exact match', () => { + const config = { + budgets: { + 'test.bench.js': { + default: { + myTask: { + absolute: { maxTime: 100 }, + }, + }, + }, + }, + }; + + const result = safeParsePartialConfig(config); + expect(result.success, 'to be true'); + + if (result.success && result.data.budgets) { + const resolved = result.data.budgets; + expect(Object.keys(resolved.exact).length, 'to equal', 1); + expect(resolved.exact['test.bench.js/default/myTask'], 'to be defined'); + expect(resolved.patterns.length, 'to equal', 0); + } + }); + + it('should categorize glob file pattern as pattern', () => { + const config = { + budgets: { + '**/*.bench.js': { + '*': { + '*': { + absolute: { maxTime: 100 }, + }, + }, + }, + }, + }; + + const result = safeParsePartialConfig(config); + expect(result.success, 'to be true'); + + if (result.success && result.data.budgets) { + const resolved = result.data.budgets; + expect(Object.keys(resolved.exact).length, 'to equal', 0); + expect(resolved.patterns.length, 'to equal', 1); + expect(resolved.patterns[0]!.filePattern, 'to equal', '**/*.bench.js'); + expect(resolved.patterns[0]!.suitePattern, 'to equal', '*'); + expect(resolved.patterns[0]!.taskPattern, 'to equal', '*'); + } + }); + + it('should categorize wildcard suite as pattern', () => { + const config = { + budgets: { + 'test.bench.js': { + '*': { + myTask: { + absolute: { maxTime: 100 }, + }, + }, + }, + }, + }; + + const result = safeParsePartialConfig(config); + expect(result.success, 'to be true'); + + if (result.success && result.data.budgets) { + const resolved = result.data.budgets; + expect(Object.keys(resolved.exact).length, 'to equal', 0); + expect(resolved.patterns.length, 'to equal', 1); + expect(resolved.patterns[0]!.suitePattern, 'to equal', '*'); + expect(resolved.patterns[0]!.taskPattern, 'to equal', 'myTask'); + } + }); + + it('should categorize wildcard task as pattern', () => { + const config = { + budgets: { + 'test.bench.js': { + default: { + '*': { + absolute: { maxTime: 100 }, + }, + }, + }, + }, + }; + + const result = safeParsePartialConfig(config); + expect(result.success, 'to be true'); + + if (result.success && result.data.budgets) { + const resolved = result.data.budgets; + expect(Object.keys(resolved.exact).length, 'to equal', 0); + expect(resolved.patterns.length, 'to equal', 1); + expect(resolved.patterns[0]!.taskPattern, 'to equal', '*'); + } + }); + }); + + describe('specificity calculation', () => { + it('should sort patterns by specificity descending', () => { + const config = { + budgets: { + '**/*': { + '*': { + '*': { absolute: { maxTime: 1000 } }, + }, + }, + '**/*.bench.js': { + '*': { + '*': { absolute: { maxTime: 500 } }, + }, + }, + 'test.bench.js': { + '*': { + '*': { absolute: { maxTime: 100 } }, + }, + }, + }, + }; + + const result = safeParsePartialConfig(config); + expect(result.success, 'to be true'); + + if (result.success && result.data.budgets) { + const resolved = result.data.budgets; + expect(resolved.patterns.length, 'to equal', 3); + + // Should be sorted by specificity descending + // Exact file (2) > glob with specificity (1) > generic glob (0) + expect(resolved.patterns[0]!.filePattern, 'to equal', 'test.bench.js'); + expect(resolved.patterns[0]!.specificity, 'to equal', 2); + + expect(resolved.patterns[1]!.filePattern, 'to equal', '**/*.bench.js'); + expect(resolved.patterns[1]!.specificity, 'to equal', 1); + + expect(resolved.patterns[2]!.filePattern, 'to equal', '**/*'); + expect(resolved.patterns[2]!.specificity, 'to equal', 0); + } + }); + }); + + describe('budget value transformation', () => { + it('should transform time strings in pattern budgets', () => { + const config = { + budgets: { + '**/*': { + '*': { + '*': { + absolute: { maxTime: '10ms' }, + }, + }, + }, + }, + }; + + const result = safeParsePartialConfig(config); + expect(result.success, 'to be true'); + + if (result.success && result.data.budgets) { + const resolved = result.data.budgets; + expect( + resolved.patterns[0]!.budget.absolute?.maxTime, + 'to equal', + 10_000_000, + ); + } + }); + + it('should transform percentage strings in pattern budgets', () => { + const config = { + budgets: { + '**/*': { + '*': { + '*': { + relative: { maxRegression: '15%' }, + }, + }, + }, + }, + }; + + const result = safeParsePartialConfig(config); + expect(result.success, 'to be true'); + + if (result.success && result.data.budgets) { + const resolved = result.data.budgets; + expect( + resolved.patterns[0]!.budget.relative?.maxRegression, + 'to equal', + 0.15, + ); + } + }); + }); + + describe('mixed exact and pattern budgets', () => { + it('should separate exact and pattern budgets correctly', () => { + const config = { + budgets: { + // This is a pattern (glob file) + '**/*': { + '*': { + '*': { relative: { maxRegression: '15%' } }, + }, + }, + // This file has both pattern (wildcard suite) and exact match entries + 'api.bench.js': { + '*': { + '*': { absolute: { maxTime: '50ms' } }, + }, + 'User Routes': { + getUser: { absolute: { maxTime: '5ms' } }, + }, + }, + }, + }; + + const result = safeParsePartialConfig(config); + expect(result.success, 'to be true'); + + if (result.success && result.data.budgets) { + const resolved = result.data.budgets; + + // Should have one exact match (User Routes/getUser) + expect(Object.keys(resolved.exact).length, 'to equal', 1); + expect( + resolved.exact['api.bench.js/User Routes/getUser'], + 'to be defined', + ); + + // Should have two patterns: **/* and api.bench.js/*/*) + expect(resolved.patterns.length, 'to equal', 2); + } + }); + }); + + describe('real-world configuration example', () => { + it('should handle comprehensive wildcard configuration', () => { + const config = { + budgets: { + // Global default for all benchmarks + '**/*': { + '*': { + '*': { + relative: { maxRegression: '15%' }, + }, + }, + }, + // API benchmarks should be faster + '**/api/**/*.bench.js': { + '*': { + '*': { + absolute: { maxTime: '50ms' }, + }, + }, + }, + // Specific file overrides + 'benchmarks/api/users.bench.js': { + 'User Routes': { + '*': { absolute: { maxTime: '20ms' } }, + getUser: { absolute: { maxTime: '5ms' } }, + }, + }, + }, + }; + + const result = safeParsePartialConfig(config); + expect(result.success, 'to be true'); + + if (result.success && result.data.budgets) { + const resolved = result.data.budgets; + + // Should have one exact match (getUser) + expect(Object.keys(resolved.exact).length, 'to equal', 1); + expect( + resolved.exact['benchmarks/api/users.bench.js/User Routes/getUser'] + ?.absolute?.maxTime, + 'to equal', + 5_000_000, + ); + + // Should have patterns sorted by specificity + expect(resolved.patterns.length, 'to equal', 3); + + // Verify patterns are sorted (most specific first) + // Note: exact file with exact suite and wildcard task has specificity 3 + // Glob with specific parts has specificity 1 + // Generic glob has specificity 0 + const specificities = resolved.patterns.map((p) => p.specificity); + expect( + specificities[0], + 'to be greater than or equal to', + specificities[1], + ); + expect( + specificities[1], + 'to be greater than or equal to', + specificities[2], + ); + } + }); + }); +}); diff --git a/test/unit/services/budget-evaluator.test.ts b/test/unit/services/budget-evaluator.test.ts index 1757ef7..a9bada4 100644 --- a/test/unit/services/budget-evaluator.test.ts +++ b/test/unit/services/budget-evaluator.test.ts @@ -4,6 +4,7 @@ import { describe, it } from 'node:test'; import type { BaselineSummaryData, Budget, + ResolvedBudgets, TaskId, TaskResult, } from '../../../src/types/core.js'; @@ -11,12 +12,22 @@ import type { import { BudgetEvaluator } from '../../../src/services/budget-evaluator.js'; import { createTaskId } from '../../../src/utils/identifiers.js'; +/** + * Helper to wrap flat budgets into ResolvedBudgets format for testing + */ +const toResolvedBudgets = ( + flatBudgets: Record, +): ResolvedBudgets => ({ + exact: flatBudgets, + patterns: [], +}); + describe('BudgetEvaluator', () => { const evaluator = new BudgetEvaluator(); describe('evaluateRun', () => { it('should return empty summary when no budgets configured', () => { - const budgets: Record = {}; + const budgets = toResolvedBudgets({}); const taskResults = new Map(); const summary = evaluator.evaluateRun(budgets, taskResults); @@ -30,13 +41,13 @@ describe('BudgetEvaluator', () => { }); it('should evaluate absolute maxTime budget', () => { - const budgets: Record = { + const budgets = toResolvedBudgets({ 'test.bench.js/default/fastTask': { absolute: { maxTime: 10_000_000, // 10ms }, }, - }; + }); const taskResults = new Map([ [ @@ -64,13 +75,13 @@ describe('BudgetEvaluator', () => { }); it('should detect maxTime budget violation', () => { - const budgets: Record = { + const budgets = toResolvedBudgets({ 'test.bench.js/default/slowTask': { absolute: { maxTime: 10_000_000, // 10ms }, }, - }; + }); const taskResults = new Map([ [ @@ -103,13 +114,13 @@ describe('BudgetEvaluator', () => { }); it('should evaluate absolute minOpsPerSec budget', () => { - const budgets: Record = { + const budgets = toResolvedBudgets({ 'test.bench.js/default/task': { absolute: { minOpsPerSec: 100000, }, }, - }; + }); const taskResults = new Map([ [ @@ -131,13 +142,13 @@ describe('BudgetEvaluator', () => { }); it('should detect minOpsPerSec budget violation', () => { - const budgets: Record = { + const budgets = toResolvedBudgets({ 'test.bench.js/default/task': { absolute: { minOpsPerSec: 100000, }, }, - }; + }); const taskResults = new Map([ [ @@ -163,13 +174,13 @@ describe('BudgetEvaluator', () => { }); it('should evaluate absolute maxP99 budget', () => { - const budgets: Record = { + const budgets = toResolvedBudgets({ 'test.bench.js/default/task': { absolute: { maxP99: 20_000_000, // 20ms }, }, - }; + }); const taskResults = new Map([ [ @@ -191,13 +202,13 @@ describe('BudgetEvaluator', () => { }); it('should detect maxP99 budget violation', () => { - const budgets: Record = { + const budgets = toResolvedBudgets({ 'test.bench.js/default/task': { absolute: { maxP99: 20_000_000, }, }, - }; + }); const taskResults = new Map([ [ @@ -220,13 +231,13 @@ describe('BudgetEvaluator', () => { }); it('should evaluate relative maxRegression budget', () => { - const budgets: Record = { + const budgets = toResolvedBudgets({ 'test.bench.js/default/task': { relative: { maxRegression: 0.1, // 10% }, }, - }; + }); const taskResults = new Map([ [ @@ -259,13 +270,13 @@ describe('BudgetEvaluator', () => { }); it('should detect maxRegression budget violation', () => { - const budgets: Record = { + const budgets = toResolvedBudgets({ 'test.bench.js/default/task': { relative: { maxRegression: 0.1, // 10% }, }, - }; + }); const taskResults = new Map([ [ @@ -301,7 +312,7 @@ describe('BudgetEvaluator', () => { }); it('should evaluate combined absolute and relative budgets', () => { - const budgets: Record = { + const budgets = toResolvedBudgets({ 'test.bench.js/default/task': { absolute: { maxTime: 15_000_000, @@ -310,7 +321,7 @@ describe('BudgetEvaluator', () => { maxRegression: 0.2, }, }, - }; + }); const taskResults = new Map([ [ @@ -343,14 +354,14 @@ describe('BudgetEvaluator', () => { }); it('should fail if any budget threshold is exceeded', () => { - const budgets: Record = { + const budgets = toResolvedBudgets({ 'test.bench.js/default/task': { absolute: { maxTime: 15_000_000, // Pass minOpsPerSec: 100000, // Fail }, }, - }; + }); const taskResults = new Map([ [ @@ -371,15 +382,16 @@ describe('BudgetEvaluator', () => { expect(summary.results[0]!.violations.length, 'to equal', 1); }); - it('should skip tasks without results', () => { - const budgets: Record = { + it('should skip tasks without matching budgets', () => { + const budgets = toResolvedBudgets({ 'test.bench.js/default/task1': { absolute: { maxTime: 10_000_000 }, }, + // task2 has a budget but no result 'test.bench.js/default/task2': { absolute: { maxTime: 10_000_000 }, }, - }; + }); const taskResults = new Map([ [ @@ -390,23 +402,37 @@ describe('BudgetEvaluator', () => { p99: 6_000_000, } as TaskResult, ], - // task2 missing + // task3 has a result but no budget + [ + createTaskId('test.bench.js/default/task3'), + { + mean: 5_000_000, + opsPerSecond: 200000, + p99: 6_000_000, + } as TaskResult, + ], ]); const summary = evaluator.evaluateRun(budgets, taskResults); + // Only task1 has both a budget and result expect(summary.total, 'to equal', 1); expect(summary.results.length, 'to equal', 1); + expect( + summary.results[0]!.taskId, + 'to equal', + 'test.bench.js/default/task1', + ); }); it('should skip relative budgets when baseline data is missing', () => { - const budgets: Record = { + const budgets = toResolvedBudgets({ 'test.bench.js/default/task': { relative: { maxRegression: 0.1, }, }, - }; + }); const taskResults = new Map([ [ @@ -427,13 +453,13 @@ describe('BudgetEvaluator', () => { }); it('should include baseline values in result when evaluating relative budgets', () => { - const budgets: Record = { + const budgets = toResolvedBudgets({ 'test.bench.js/default/task': { relative: { maxRegression: 0.1, }, }, - }; + }); const taskResults = new Map([ [ @@ -467,13 +493,13 @@ describe('BudgetEvaluator', () => { }); it('should generate descriptive violation messages', () => { - const budgets: Record = { + const budgets = toResolvedBudgets({ 'test.bench.js/default/task': { absolute: { maxTime: 10_000_000, }, }, - }; + }); const taskResults = new Map([ [ diff --git a/test/unit/services/budget-resolver.test.ts b/test/unit/services/budget-resolver.test.ts new file mode 100644 index 0000000..909fb62 --- /dev/null +++ b/test/unit/services/budget-resolver.test.ts @@ -0,0 +1,430 @@ +import { expect } from 'bupkis'; +import { describe, it } from 'node:test'; + +import type { Budget, ResolvedBudgets } from '../../../src/types/core.js'; + +import { + calculateSpecificity, + createBudgetPattern, + isGlobPattern, + matchesFile, + matchesSuiteOrTask, + mergeBudgets, + parseTaskId, + resolveBudget, +} from '../../../src/services/budget-resolver.js'; +import { createTaskId } from '../../../src/utils/identifiers.js'; + +describe('BudgetResolver', () => { + describe('isGlobPattern', () => { + it('should detect asterisk glob', () => { + expect(isGlobPattern('*.bench.js'), 'to be true'); + expect(isGlobPattern('**/*.bench.js'), 'to be true'); + }); + + it('should detect question mark glob', () => { + expect(isGlobPattern('test?.bench.js'), 'to be true'); + }); + + it('should detect bracket glob', () => { + expect(isGlobPattern('[abc].bench.js'), 'to be true'); + }); + + it('should not detect plain strings', () => { + expect(isGlobPattern('test.bench.js'), 'to be false'); + expect(isGlobPattern('path/to/file.js'), 'to be false'); + }); + }); + + describe('matchesFile', () => { + it('should match exact file paths', () => { + expect(matchesFile('test.bench.js', 'test.bench.js'), 'to be true'); + expect( + matchesFile('api/users.bench.js', 'api/users.bench.js'), + 'to be true', + ); + }); + + it('should not match different exact paths', () => { + expect(matchesFile('test.bench.js', 'other.bench.js'), 'to be false'); + }); + + it('should match with simple glob', () => { + expect(matchesFile('*.bench.js', 'test.bench.js'), 'to be true'); + expect(matchesFile('*.bench.js', 'api.bench.js'), 'to be true'); + }); + + it('should match with double star glob', () => { + expect(matchesFile('**/*.bench.js', 'test.bench.js'), 'to be true'); + expect( + matchesFile('**/*.bench.js', 'nested/deep/test.bench.js'), + 'to be true', + ); + }); + + it('should match with path prefix glob', () => { + expect( + matchesFile('api/**/*.bench.js', 'api/users.bench.js'), + 'to be true', + ); + expect( + matchesFile('api/**/*.bench.js', 'api/v2/users.bench.js'), + 'to be true', + ); + expect( + matchesFile('api/**/*.bench.js', 'db/users.bench.js'), + 'to be false', + ); + }); + + it('should match with generic glob', () => { + expect(matchesFile('**/*', 'anything.js'), 'to be true'); + expect(matchesFile('**/*', 'deep/nested/file.ts'), 'to be true'); + }); + }); + + describe('matchesSuiteOrTask', () => { + it('should match exact names', () => { + expect(matchesSuiteOrTask('User Routes', 'User Routes'), 'to be true'); + expect(matchesSuiteOrTask('getUser', 'getUser'), 'to be true'); + }); + + it('should not match different names', () => { + expect(matchesSuiteOrTask('User Routes', 'API Routes'), 'to be false'); + }); + + it('should match wildcard to any value', () => { + expect(matchesSuiteOrTask('*', 'User Routes'), 'to be true'); + expect(matchesSuiteOrTask('*', 'anything'), 'to be true'); + }); + }); + + describe('calculateSpecificity', () => { + it('should score 0 for fully generic pattern', () => { + const score = calculateSpecificity({ + filePattern: '**/*', + suitePattern: '*', + taskPattern: '*', + }); + expect(score, 'to equal', 0); + }); + + it('should score 1 for glob file with wildcard suite/task', () => { + const score = calculateSpecificity({ + filePattern: '**/*.bench.js', + suitePattern: '*', + taskPattern: '*', + }); + expect(score, 'to equal', 1); + }); + + it('should score 2 for exact file with wildcard suite/task', () => { + const score = calculateSpecificity({ + filePattern: 'test.bench.js', + suitePattern: '*', + taskPattern: '*', + }); + expect(score, 'to equal', 2); + }); + + it('should score 2 for glob file with exact suite', () => { + const score = calculateSpecificity({ + filePattern: '**/*.bench.js', + suitePattern: 'User Routes', + taskPattern: '*', + }); + expect(score, 'to equal', 2); + }); + + it('should score 3 for exact file with exact suite', () => { + const score = calculateSpecificity({ + filePattern: 'test.bench.js', + suitePattern: 'User Routes', + taskPattern: '*', + }); + expect(score, 'to equal', 3); + }); + + it('should score 4 for fully exact pattern', () => { + const score = calculateSpecificity({ + filePattern: 'test.bench.js', + suitePattern: 'User Routes', + taskPattern: 'getUser', + }); + expect(score, 'to equal', 4); + }); + }); + + describe('parseTaskId', () => { + it('should parse simple TaskId', () => { + const taskId = createTaskId('test.bench.js/default/myTask'); + const { file, suite, task } = parseTaskId(taskId); + expect(file, 'to equal', 'test.bench.js'); + expect(suite, 'to equal', 'default'); + expect(task, 'to equal', 'myTask'); + }); + + it('should parse TaskId with nested file path', () => { + const taskId = createTaskId('api/users.bench.js/User Routes/getUser'); + const { file, suite, task } = parseTaskId(taskId); + expect(file, 'to equal', 'api/users.bench.js'); + expect(suite, 'to equal', 'User Routes'); + expect(task, 'to equal', 'getUser'); + }); + + it('should parse TaskId with deeply nested file path', () => { + const taskId = createTaskId( + 'benchmarks/api/v2/users.bench.js/HTTP Routes/createUser', + ); + const { file, suite, task } = parseTaskId(taskId); + expect(file, 'to equal', 'benchmarks/api/v2/users.bench.js'); + expect(suite, 'to equal', 'HTTP Routes'); + expect(task, 'to equal', 'createUser'); + }); + }); + + describe('mergeBudgets', () => { + it('should merge absolute budgets', () => { + const base: Budget = { + absolute: { maxTime: 100 }, + }; + const override: Budget = { + absolute: { minOpsPerSec: 1000 }, + }; + + const merged = mergeBudgets(base, override); + + expect(merged.absolute?.maxTime, 'to equal', 100); + expect(merged.absolute?.minOpsPerSec, 'to equal', 1000); + }); + + it('should override conflicting absolute values', () => { + const base: Budget = { + absolute: { maxTime: 100 }, + }; + const override: Budget = { + absolute: { maxTime: 50 }, + }; + + const merged = mergeBudgets(base, override); + + expect(merged.absolute?.maxTime, 'to equal', 50); + }); + + it('should merge relative budgets', () => { + const base: Budget = { + relative: { maxRegression: 0.1 }, + }; + const override: Budget = { + relative: { baseline: 'main' }, + }; + + const merged = mergeBudgets(base, override); + + expect(merged.relative?.maxRegression, 'to equal', 0.1); + expect(merged.relative?.baseline, 'to equal', 'main'); + }); + + it('should merge both absolute and relative', () => { + const base: Budget = { + absolute: { maxTime: 100 }, + }; + const override: Budget = { + relative: { maxRegression: 0.1 }, + }; + + const merged = mergeBudgets(base, override); + + expect(merged.absolute?.maxTime, 'to equal', 100); + expect(merged.relative?.maxRegression, 'to equal', 0.1); + }); + }); + + describe('createBudgetPattern', () => { + it('should create pattern with calculated specificity', () => { + const pattern = createBudgetPattern('**/*.bench.js', '*', '*', { + absolute: { maxTime: 100 }, + }); + + expect(pattern.filePattern, 'to equal', '**/*.bench.js'); + expect(pattern.suitePattern, 'to equal', '*'); + expect(pattern.taskPattern, 'to equal', '*'); + expect(pattern.specificity, 'to equal', 1); + }); + }); + + describe('resolveBudget', () => { + it('should return undefined when no budgets match', () => { + const budgets: ResolvedBudgets = { + exact: {}, + patterns: [], + }; + + const taskId = createTaskId('test.bench.js/default/myTask'); + const result = resolveBudget(taskId, budgets); + + expect(result, 'to be undefined'); + }); + + it('should return exact match when available', () => { + const budgets: ResolvedBudgets = { + exact: { + 'test.bench.js/default/myTask': { + absolute: { maxTime: 100 }, + }, + }, + patterns: [], + }; + + const taskId = createTaskId('test.bench.js/default/myTask'); + const result = resolveBudget(taskId, budgets); + + expect(result?.absolute?.maxTime, 'to equal', 100); + }); + + it('should match pattern when no exact match', () => { + const budgets: ResolvedBudgets = { + exact: {}, + patterns: [ + createBudgetPattern('**/*', '*', '*', { + absolute: { maxTime: 1000 }, + }), + ], + }; + + const taskId = createTaskId('test.bench.js/default/myTask'); + const result = resolveBudget(taskId, budgets); + + expect(result?.absolute?.maxTime, 'to equal', 1000); + }); + + it('should prefer more specific patterns', () => { + const budgets: ResolvedBudgets = { + exact: {}, + patterns: [ + createBudgetPattern('**/*', '*', '*', { + absolute: { maxTime: 1000 }, + }), + createBudgetPattern('**/*.bench.js', '*', '*', { + absolute: { maxTime: 500 }, + }), + createBudgetPattern('test.bench.js', '*', '*', { + absolute: { maxTime: 100 }, + }), + ], + }; + + const taskId = createTaskId('test.bench.js/default/myTask'); + const result = resolveBudget(taskId, budgets); + + // Most specific pattern (exact file) should win + expect(result?.absolute?.maxTime, 'to equal', 100); + }); + + it('should merge budgets from multiple matching patterns', () => { + const budgets: ResolvedBudgets = { + exact: {}, + patterns: [ + createBudgetPattern('**/*', '*', '*', { + relative: { maxRegression: 0.15 }, + }), + createBudgetPattern('test.bench.js', '*', '*', { + absolute: { maxTime: 100 }, + }), + ], + }; + + const taskId = createTaskId('test.bench.js/default/myTask'); + const result = resolveBudget(taskId, budgets); + + // Should have both from merge + expect(result?.absolute?.maxTime, 'to equal', 100); + expect(result?.relative?.maxRegression, 'to equal', 0.15); + }); + + it('should merge exact match with patterns', () => { + const budgets: ResolvedBudgets = { + exact: { + 'test.bench.js/default/myTask': { + absolute: { maxTime: 50 }, + }, + }, + patterns: [ + createBudgetPattern('**/*', '*', '*', { + relative: { maxRegression: 0.1 }, + }), + ], + }; + + const taskId = createTaskId('test.bench.js/default/myTask'); + const result = resolveBudget(taskId, budgets); + + // Should have exact match's maxTime and pattern's maxRegression + expect(result?.absolute?.maxTime, 'to equal', 50); + expect(result?.relative?.maxRegression, 'to equal', 0.1); + }); + + it('should let exact match override pattern values', () => { + const budgets: ResolvedBudgets = { + exact: { + 'test.bench.js/default/myTask': { + absolute: { maxTime: 50 }, + }, + }, + patterns: [ + createBudgetPattern('**/*', '*', '*', { + absolute: { maxTime: 1000 }, + }), + ], + }; + + const taskId = createTaskId('test.bench.js/default/myTask'); + const result = resolveBudget(taskId, budgets); + + // Exact match should win + expect(result?.absolute?.maxTime, 'to equal', 50); + }); + + it('should match suite-specific patterns', () => { + const budgets: ResolvedBudgets = { + exact: {}, + patterns: [ + createBudgetPattern('**/*', 'User Routes', '*', { + absolute: { maxTime: 100 }, + }), + ], + }; + + const taskId = createTaskId('test.bench.js/User Routes/getUser'); + const result = resolveBudget(taskId, budgets); + + expect(result?.absolute?.maxTime, 'to equal', 100); + + // Different suite should not match + const otherTaskId = createTaskId('test.bench.js/API Routes/getUser'); + const otherResult = resolveBudget(otherTaskId, budgets); + expect(otherResult, 'to be undefined'); + }); + + it('should match task-specific patterns', () => { + const budgets: ResolvedBudgets = { + exact: {}, + patterns: [ + createBudgetPattern('**/*', '*', 'getUser', { + absolute: { maxTime: 50 }, + }), + ], + }; + + const taskId = createTaskId('test.bench.js/User Routes/getUser'); + const result = resolveBudget(taskId, budgets); + + expect(result?.absolute?.maxTime, 'to equal', 50); + + // Different task should not match + const otherTaskId = createTaskId('test.bench.js/User Routes/createUser'); + const otherResult = resolveBudget(otherTaskId, budgets); + expect(otherResult, 'to be undefined'); + }); + }); +});