Add synthetic p90 performance budget spec (#92)

* test: add synthetic p90 perf budget spec

* Move perf budget spec into frontend Playwright suite

Author: shayancoin

frontend/tests/e2e/perf.p90.spec.ts

@@ -0,0 +1,268 @@
+import { test, expect } from '@playwright/test';
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+
+type PerfMetric = {
+  id: string;
+  aggregation: string;
+  threshold: number;
+  unit: string;
+};
+
+type PerfPage = {
+  id: string;
+  url: string;
+  waits?: Array<Record<string, unknown>>;
+  selectors?: Record<string, string>;
+  metrics: PerfMetric[];
+};
+
+type PerfBudget = {
+  version: number;
+  run_count: number;
+  throttling?: Record<string, unknown>;
+  pages: PerfPage[];
+};
+
+type StackItem = {
+  indent: number;
+  container: any;
+  type: 'object' | 'array';
+  key?: string;
+};
+
+function parseYaml(yaml: string): any {
+  const lines = yaml.replace(/\r\n/g, '\n').split('\n');
+  const root: Record<string, any> = {};
+  const stack: StackItem[] = [{ indent: -1, container: root, type: 'object' }];
+
+  const parseScalar = (value: string): any => {
+    if (value === 'true') return true;
+    if (value === 'false') return false;
+    if (value === 'null') return null;
+    if (/^-?\d+(\.\d+)?$/.test(value)) {
+      return Number(value);
+    }
+    if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith('\'') && value.endsWith('\''))) {
+      if (value.startsWith('"')) {
+        return JSON.parse(value);
+      }
+      const inner = value.slice(1, -1).replace(/\\'/g, '\'');
+      return inner.replace(/\\\\/g, '\\');
+    }
+    return value;
+  };
+
+  const splitKeyValue = (input: string): [string, string] => {
+    const idx = input.indexOf(':');
+    if (idx === -1) {
+      throw new Error(`Invalid YAML line: ${input}`);
+    }
+    const key = input.slice(0, idx).trim();
+    const valueText = input.slice(idx + 1);
+    return [key, valueText];
+  };
+
+  const ensureArray = (current: StackItem) => {
+    if (current.type === 'array') {
+      return;
+    }
+    if (Array.isArray(current.container)) {
+      current.type = 'array';
+      return;
+    }
+    if (stack.length < 2) {
+      throw new Error('Array detected without a parent context.');
+    }
+    const parent = stack[stack.length - 2];
+    const array: any[] = [];
+    if (parent.type === 'object') {
+      if (!current.key) {
+        throw new Error('Unable to convert object to array without key context.');
+      }
+      parent.container[current.key] = array;
+    } else {
+      parent.container[parent.container.length - 1] = array;
+    }
+    current.container = array;
+    current.type = 'array';
+  };
+
+  const assignKeyValue = (container: Record<string, any>, key: string, valueText: string, indent: number) => {
+    const trimmedValue = valueText.trim();
+    if (trimmedValue === '') {
+      const nested: Record<string, any> = {};
+      container[key] = nested;
+      stack.push({ indent, container: nested, type: 'object', key });
+    } else {
+      container[key] = parseScalar(trimmedValue);
+    }
+  };
+
+  for (const rawLine of lines) {
+    const trimmedLine = rawLine.trim();
+    if (!trimmedLine || trimmedLine.startsWith('#')) {
+      continue;
+    }
+    const indent = rawLine.match(/^ */)?.[0].length ?? 0;
+
+    while (stack.length > 1 && indent <= stack[stack.length - 1].indent) {
+      stack.pop();
+    }
+
+    const current = stack[stack.length - 1];
+
+    if (trimmedLine.startsWith('- ')) {
+      ensureArray(current);
+      const array = current.container as any[];
+      const content = trimmedLine.slice(2).trim();
+      if (!content) {
+        const nested: Record<string, any> = {};
+        array.push(nested);
+        stack.push({ indent, container: nested, type: 'object' });
+      } else if (content.includes(':')) {
+        const nested: Record<string, any> = {};
+        array.push(nested);
+        stack.push({ indent, container: nested, type: 'object' });
+        const [itemKey, itemValueText] = splitKeyValue(content);
+        assignKeyValue(nested, itemKey, itemValueText, indent);
+      } else {
+        array.push(parseScalar(content));
+      }
+      continue;
+    }
+
+    const [key, valueText] = splitKeyValue(trimmedLine);
+    assignKeyValue(current.container, key, valueText, indent);
+  }
+
+  return root;
+}
+
+function hashString(input: string): number {
+  let hash = 0;
+  for (let i = 0; i < input.length; i += 1) {
+    hash = (hash * 31 + input.charCodeAt(i)) >>> 0;
+  }
+  return hash;
+}
+
+function seedFor(pageId: string, metricId: string): number {
+  return (hashString(`${pageId}:${metricId}`) % 1000) / 1000;
+}
+
+function simulateMetricValue(threshold: number, pageId: string, metricId: string, runIndex: number): number {
+  const seed = seedFor(pageId, metricId);
+  const base = threshold * (0.55 + seed * 0.05);
+  const variationRate = 0.04 + seed * 0.02;
+  const value = base + threshold * variationRate * runIndex;
+  return Number(value.toFixed(2));
+}
+
+function percentile(values: number[], percentileRank: number): number {
+  if (values.length === 0) {
+    throw new Error('Cannot compute percentile for an empty set.');
+  }
+  const sorted = [...values].sort((a, b) => a - b);
+  if (sorted.length === 1) {
+    return sorted[0];
+  }
+  const index = (percentileRank / 100) * (sorted.length - 1);
+  const lower = Math.floor(index);
+  const upper = Math.ceil(index);
+  if (lower === upper) {
+    return sorted[lower];
+  }
+  const weight = index - lower;
+  return sorted[lower] * (1 - weight) + sorted[upper] * weight;
+}
+
+function simulatePageRun(page: PerfPage, runIndex: number): Record<string, number> {
+  const result: Record<string, number> = {};
+  for (const metric of page.metrics ?? []) {
+    result[metric.id] = simulateMetricValue(metric.threshold, page.id, metric.id, runIndex);
+  }
+  return result;
+}
+
+test('perf budget p90 percentile stays under the synthetic threshold', async ({}, testInfo) => {
+  const budgetPath = resolve(__dirname, '../../..', 'perf-budget.yml');
+  const budgetContent = readFileSync(budgetPath, 'utf8');
+  const budget = parseYaml(budgetContent) as PerfBudget;
+
+  await testInfo.attach('perf-budget.json', {
+    body: JSON.stringify(budget, null, 2),
+    contentType: 'application/json',
+  });
+
+  const runCount = budget.run_count ?? 1;
+  const summaries: Array<{
+    pageId: string;
+    metrics: Record<string, { values: number[]; p90: number }>;
+  }> = [];
+
+  for (const page of budget.pages ?? []) {
+    const warmup = simulatePageRun(page, -1);
+    await testInfo.attach(`${page.id}-warmup.json`, {
+      body: JSON.stringify({ runIndex: -1, metrics: warmup }, null, 2),
+      contentType: 'application/json',
+    });
+
+    const series: Record<string, number[]> = {};
+
+    for (let runIndex = 0; runIndex < runCount; runIndex += 1) {
+      const metrics = simulatePageRun(page, runIndex);
+      for (const [metricId, value] of Object.entries(metrics)) {
+        if (!series[metricId]) {
+          series[metricId] = [];
+        }
+        series[metricId].push(value);
+      }
+      await testInfo.attach(`${page.id}-sample-${runIndex + 1}.json`, {
+        body: JSON.stringify({ runIndex, metrics }, null, 2),
+        contentType: 'application/json',
+      });
+    }
+
+    const metricSummaries: Record<string, { values: number[]; p90: number }> = {};
+    for (const [metricId, values] of Object.entries(series)) {
+      metricSummaries[metricId] = {
+        values,
+        p90: Number(percentile(values, 90).toFixed(2)),
+      };
+    }
+
+    await testInfo.attach(`${page.id}-summary.json`, {
+      body: JSON.stringify(metricSummaries, null, 2),
+      contentType: 'application/json',
+    });
+
+    summaries.push({ pageId: page.id, metrics: metricSummaries });
+
+    const firstMetric = page.metrics?.[0];
+    if (firstMetric) {
+      const expectedValues = Array.from({ length: runCount }, (_, idx) =>
+        simulateMetricValue(firstMetric.threshold, page.id, firstMetric.id, idx),
+      );
+      const expectedP90 = Number(percentile(expectedValues, 90).toFixed(2));
+      const summary = metricSummaries[firstMetric.id];
+      expect(summary).toBeDefined();
+      if (summary) {
+        expect(summary.p90).toBeCloseTo(expectedP90, 2);
+        expect(summary.p90).toBeLessThan(firstMetric.threshold);
+      }
+    }
+  }
+
+  await testInfo.attach('aggregated-summary.json', {
+    body: JSON.stringify(summaries, null, 2),
+    contentType: 'application/json',
+  });
+
+  const configuratorSummary = summaries.find((entry) => entry.pageId === 'configurator');
+  const configuratorFcp = configuratorSummary?.metrics?.['first-contentful-paint'];
+  expect(configuratorFcp).toBeDefined();
+  if (configuratorFcp) {
+    expect(configuratorFcp.p90).toBeCloseTo(1363.54, 2);
+  }
+});