perf-lab.e2e-spec.ts

import type { INestApplication } from "@nestjs/common";
import { performance } from "node:perf_hooks";
import otelSDK from "../../src/tracing";
import { initApp } from "../utils/init-app";
import { getPerfCase, resolvePerfCaseIdsWithExclusions } from "./registry";
import { runPerfCase } from "./framework/run-perf-case";
import { seedPerfCase } from "./framework/run-perf-seed";
import { updatePerfArtifactTraceSummary } from "./framework/artifacts";
import {
  getComputeSinkDiagnostics,
  installComputeSpanSink,
} from "./framework/compute-span-sink";
import { applyCaseRuntimeEnv } from "./framework/perf-runtime-env";
import type { PerfCase } from "./framework/types";
import {
  installPerfTraceCollector,
  finalizePerfTraceJobTailLifecycle,
  resetPerfTraceJobTail,
  setPerfTraceFlush,
  uninstallPerfTraceCollector,
} from "./framework/trace-collector";
import {
  installWatchdogActivityProbe,
  uninstallWatchdogActivityProbe,
} from "./framework/watchdog";
import { axios } from "@teable/openapi";

const specStarted = performance.now();

const logPhase = (
  phase: string,
  details: Record<string, string | number | boolean | undefined> = {},
) => {
  if (process.env.PERF_LAB_PHASE_LOG === "false") {
    return;
  }

  const detailText = Object.entries(details)
    .filter(([, value]) => value !== undefined)
    .map(([key, value]) => `${key}=${String(value)}`)
    .join(" ");

  console.log(
    `[perf-lab] ${phase} at=${new Date().toISOString()} elapsedMs=${Math.round(
      performance.now() - specStarted,
    )}${detailText ? ` ${detailText}` : ""}`,
  );
};

type Engine = "v1" | "v2";
type Mode = "execute" | "seed";
const SEED_BOOTSTRAP_ENGINE: Engine = "v1";

const parseEngineList = (engineList = "v1,v2"): Engine[] => {
  const engines = engineList
    .split(",")
    .map((engine) => engine.trim())
    .filter(Boolean);

  if (engines.length === 0) {
    throw new Error("PERF_LAB_ENGINE_LIST must include at least one engine");
  }

  const invalidEngines = engines.filter(
    (engine) => engine !== "v1" && engine !== "v2",
  );
  if (invalidEngines.length > 0) {
    throw new Error(
      `Unsupported PERF_LAB_ENGINE_LIST: ${invalidEngines.join(
        ", ",
      )}. Available engines: v1, v2.`,
    );
  }

  return [...new Set(engines)] as Engine[];
};

const getForceV2All = (engine: Engine) => (engine === "v2" ? "true" : "false");

const getMode = (): Mode =>
  process.env.PERF_LAB_MODE === "seed" ? "seed" : "execute";

const resetAxiosInterceptors = () => {
  uninstallPerfTraceCollector();
  uninstallWatchdogActivityProbe();
  axios.interceptors.request.clear?.();
  axios.interceptors.response.clear?.();
  installPerfTraceCollector();
  installWatchdogActivityProbe();
};

const getOtelForceFlush = () => {
  const sdk = otelSDK as unknown as {
    forceFlush?: () => Promise<void>;
    _tracerProvider?: { forceFlush?: () => Promise<void> };
    tracerProvider?: { forceFlush?: () => Promise<void> };
  };
  const provider = sdk._tracerProvider ?? sdk.tracerProvider;

  if (typeof sdk.forceFlush === "function") {
    return () => sdk.forceFlush?.call(sdk);
  }

  if (typeof provider?.forceFlush === "function") {
    return () => provider.forceFlush?.call(provider);
  }

  return undefined;
};

const withEngineEnv = async <T>(engine: Engine, fn: () => Promise<T>) => {
  const previousForceV2All = process.env.FORCE_V2_ALL;
  const previousEngine = process.env.PERF_LAB_ENGINE;
  const previousOtelServiceName = process.env.OTEL_SERVICE_NAME;
  // The teable-ee e2e setup file unconditionally forces
  // V2_COMPUTED_UPDATE_MODE=sync for deterministic tests. When the workflow asks
  // for a specific mode (e.g. hybrid, the production outbox + polling-worker
  // path), override it here — this runs after the setup file and before
  // initApp() builds the V2 container, which reads the env once at creation.
  // The V2 container only reads it; V1 ignores it. Leave it untouched (sync)
  // when the override is not set.
  const previousComputedUpdateMode = process.env.V2_COMPUTED_UPDATE_MODE;
  const requestedComputedUpdateMode = process.env.PERF_LAB_COMPUTED_UPDATE_MODE;

  process.env.FORCE_V2_ALL = getForceV2All(engine);
  process.env.PERF_LAB_ENGINE = engine;
  if (requestedComputedUpdateMode) {
    process.env.V2_COMPUTED_UPDATE_MODE = requestedComputedUpdateMode;
  }
  process.env.OTEL_SERVICE_NAME =
    process.env.PERF_LAB_OTEL_SERVICE_PREFIX != null
      ? `${process.env.PERF_LAB_OTEL_SERVICE_PREFIX}-${engine}`
      : `teable-perf-serial-${engine}`;

  try {
    return await fn();
  } finally {
    if (previousForceV2All == null) {
      delete process.env.FORCE_V2_ALL;
    } else {
      process.env.FORCE_V2_ALL = previousForceV2All;
    }

    if (previousEngine == null) {
      delete process.env.PERF_LAB_ENGINE;
    } else {
      process.env.PERF_LAB_ENGINE = previousEngine;
    }

    if (previousOtelServiceName == null) {
      delete process.env.OTEL_SERVICE_NAME;
    } else {
      process.env.OTEL_SERVICE_NAME = previousOtelServiceName;
    }

    if (requestedComputedUpdateMode) {
      if (previousComputedUpdateMode == null) {
        delete process.env.V2_COMPUTED_UPDATE_MODE;
      } else {
        process.env.V2_COMPUTED_UPDATE_MODE = previousComputedUpdateMode;
      }
    }
  }
};

const withRunEngineEnv = async <T>(
  engine: Engine | "seed",
  fn: () => Promise<T>,
) =>
  withEngineEnv(
    engine === "seed" ? SEED_BOOTSTRAP_ENGINE : engine,
    async () => {
      if (engine === "seed") {
        process.env.PERF_LAB_ENGINE = "seed";
        process.env.OTEL_SERVICE_NAME =
          process.env.PERF_LAB_OTEL_SERVICE_PREFIX != null
            ? `${process.env.PERF_LAB_OTEL_SERVICE_PREFIX}-seed`
            : "teable-perf-serial-seed";
      }

      return fn();
    },
  );

describe("perf-lab serial case runner (e2e)", () => {
  const perfCaseIds = resolvePerfCaseIdsWithExclusions(
    process.env.PERF_LAB_CASE_FILTER ??
      process.env.PERF_LAB_CASE_ID ??
      "smoke/auth-user",
    process.env.PERF_LAB_EXCLUDE_CASE_FILTER,
  );
  const perfCases = perfCaseIds.map(getPerfCase);
  applyCaseRuntimeEnv(perfCases);
  const engines = parseEngineList(process.env.PERF_LAB_ENGINE_LIST);
  const mode = getMode();

  logPhase("module-loaded", {
    mode,
    cases: perfCases.map((perfCase) => perfCase.id).join(","),
    excludedCases: process.env.PERF_LAB_EXCLUDE_CASE_FILTER || "(none)",
    engines: engines.join(","),
    computedUpdateMode:
      process.env.PERF_LAB_COMPUTED_UPDATE_MODE || "(default)",
    maxPasteCells: process.env.MAX_PASTE_CELLS,
    maxSelectChoices: process.env.TABLE_LIMIT_SELECT_CHOICES_MAX,
  });

  beforeAll(() => {
    setPerfTraceFlush(getOtelForceFlush());
    installPerfTraceCollector();
    // Installed once per process, and deliberately not part of
    // resetAxiosInterceptors(): the span processor is attached to the OTel
    // provider, which outlives each engine's Nest app.
    logPhase("compute-span-sink", {
      attached: installComputeSpanSink(otelSDK),
    });
  });

  afterAll(() => {
    // The sink attaches through an OpenTelemetry private field, so an OTel
    // upgrade can detach it silently — and a detached sink looks exactly like
    // a suite that did no computing. Seeing zero spans of ANY name is the one
    // signal that tells those apart. Warn rather than fail: a perf run must
    // not go red because observability moved.
    const diagnostics = getComputeSinkDiagnostics();
    logPhase("compute-span-sink:done", diagnostics);
    if (diagnostics.attached && diagnostics.spansSeen === 0) {
      console.warn(
        "[perf-lab] compute span sink attached but observed no spans at all. " +
          "This is an attachment break, not a quiet suite — check whether the " +
          "OpenTelemetry SDK layout changed.",
      );
    }
  });

  const runEngines: Array<Engine | "seed"> =
    mode === "seed" ? ["seed"] : engines;

  for (const engine of runEngines) {
    describe(`engine ${engine}`, () => {
      let app: INestApplication;
      let appUrl: string;
      let cookie: string | undefined;

      beforeAll(async () => {
        logPhase("engine:beforeAll:start", { engine });
        resetPerfTraceJobTail();
        await withRunEngineEnv(engine, async () => {
          resetAxiosInterceptors();
          const initStarted = performance.now();
          const appCtx = await initApp();
          app = appCtx.app;
          appUrl = appCtx.appUrl;
          cookie = appCtx.cookie;
          logPhase("engine:beforeAll:ready", {
            engine,
            initAppMs: Math.round(performance.now() - initStarted),
            appUrl,
          });
        });
      });

      afterAll(async () => {
        try {
          const traceTail = await withRunEngineEnv(engine, () =>
            finalizePerfTraceJobTailLifecycle({
              reconcileArtifact: (traceResult) =>
                updatePerfArtifactTraceSummary({
                  artifactDir: traceResult.artifactDir,
                  perfCase: traceResult.perfCase,
                  engine: traceResult.engine,
                  traceSummary: traceResult.summary,
                }),
            }),
          );
          for (const artifactError of traceTail.artifactErrors) {
            console.warn(
              `[perf-lab] trace-tail artifact update failed caseId=${artifactError.caseId} engine=${artifactError.engine}: ${artifactError.message}`,
            );
          }
          logPhase("engine:trace-tail:done", {
            engine,
            cases: traceTail.results.length,
            tailErrors: traceTail.results.filter(
              ({ tailError, summary }) =>
                tailError != null ||
                summary.traceFetchBreakerState === "tail-error",
            ).length,
            traceTailMs: traceTail.elapsedMs,
            traceTailBudgetExceeded: traceTail.exceededBudget,
          });
        } catch (error) {
          console.warn(
            `[perf-lab] trace-tail failed engine=${engine}; provisional manifests remain available: ${
              error instanceof Error ? error.message : String(error)
            }`,
          );
        }
        const closeStarted = performance.now();
        await app?.close();
        logPhase("engine:afterAll:closed", {
          engine,
          closeMs: Math.round(performance.now() - closeStarted),
        });
      });

      for (const perfCase of perfCases) {
        it(
          `runs ${perfCase.id} (${engine})`,
          { timeout: perfCase.timeoutMs },
          async () => {
            await withRunEngineEnv(engine, async () => {
              logPhase("case:start", { caseId: perfCase.id, engine });
              const caseStarted = performance.now();
              if (mode === "seed") {
                await seedPerfCase(perfCase, { app, appUrl, cookie });
              } else {
                await runPerfCase(perfCase, { app, appUrl, cookie });
              }
              logPhase("case:done", {
                caseId: perfCase.id,
                engine,
                caseMs: Math.round(performance.now() - caseStarted),
              });
            });
          },
        );
      }
    });
  }
});