feat(ng-dev): create workflow performance testing tooling

.github/workflows/ci.yml

@@ -71,3 +71,19 @@ jobs:
       - uses: ./github-actions/bazel/setup
       - run: yarn install --immutable
       - run: yarn bazel test --sandbox_writable_path="$HOME/Library/Application Support" --test_tag_filters=macos --build_tests_only -- //...
+
+  workflow-perf:
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    steps:
+      # Because the checkout and setup node action is contained in the dev-infra repo, we must
+      # checkout the repo to be able to run the action we have created.  Other repos will skip
+      # this step.
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: ./github-actions/npm/checkout-and-setup-node
+      - uses: ./github-actions/bazel/setup
+      - run: yarn install --immutable
+      - run: yarn ng-dev perf workflows --json
+      # Always run this step to ensure that the job always is successful
+      - if: ${{ always() }}
+        run: exit 0

.ng-dev/perf-tests/test-rerun.diff

@@ -0,0 +1,17 @@
+diff --git a/ng-dev/utils/test/g3.spec.ts b/ng-dev/utils/test/g3.spec.ts
+index a82c1b7a..8e0b24f8 100644
+--- a/ng-dev/utils/test/g3.spec.ts
++++ b/ng-dev/utils/test/g3.spec.ts
+@@ -29,9 +29,9 @@ describe('G3Stats', () => {
+   });
+
+   function setupFakeSyncConfig(config: GoogleSyncConfig): string {
+-    const configFileName = 'sync-test-conf.json';
+-    fs.writeFileSync(path.join(git.baseDir, configFileName), JSON.stringify(config));
+-    return configFileName;
++    const somethingelse = 'sync-test-conf.json';
++    fs.writeFileSync(path.join(git.baseDir, somethingelse), JSON.stringify(config));
++    return somethingelse;
+   }
+
+   describe('gathering stats', () => {

.ng-dev/workflows.yml

@@ -0,0 +1,17 @@
+workflows:
+  - name: Rerun a test
+    prepare: |
+      bazel clean;
+      bazel build //ng-dev/utils/test;
+    workflow: |
+      bazel test //ng-dev/utils/test;
+      git apply .ng-dev/perf-tests/test-rerun.diff;
+      bazel test //ng-dev/utils/test;
+    cleanup: |
+      git apply -R .ng-dev/perf-tests/test-rerun.diff;
+
+  - name: Build Everything
+    prepare: |
+      bazel clean;
+    workflow: |
+      bazel build //...;

ng-dev/BUILD.bazel

@@ -27,6 +27,7 @@ ts_library(
         "//ng-dev/format",
         "//ng-dev/misc",
         "//ng-dev/ngbot",
+        "//ng-dev/perf",
         "//ng-dev/pr",
         "//ng-dev/pr/common/labels",
         "//ng-dev/pr/config",

ng-dev/cli.ts

@@ -20,6 +20,7 @@ import {buildReleaseParser} from './release/cli.js';
 import {tsCircularDependenciesBuilder} from './ts-circular-dependencies/index.js';
 import {captureLogOutputForCommand} from './utils/logging.js';
 import {buildAuthParser} from './auth/cli.js';
+import {buildPerfParser} from './perf/cli.js';
 import {Argv} from 'yargs';
 
 runParserWithCompletedFunctions((yargs: Argv) => {
@@ -38,6 +39,7 @@ runParserWithCompletedFunctions((yargs: Argv) => {
     .command('caretaker <command>', '', buildCaretakerParser)
     .command('misc <command>', '', buildMiscParser)
     .command('ngbot <command>', false, buildNgbotParser)
+    .command('perf <command>', '', buildPerfParser)
     .wrap(120)
     .strict();
 });

ng-dev/perf/BUILD.bazel

@@ -0,0 +1,11 @@
+load("//tools:defaults.bzl", "ts_library")
+
+ts_library(
+    name = "perf",
+    srcs = ["cli.ts"],
+    visibility = ["//ng-dev:__subpackages__"],
+    deps = [
+        "//ng-dev/perf/workflow",
+        "@npm//@types/yargs",
+    ],
+)

ng-dev/perf/cli.ts

@@ -0,0 +1,16 @@
+/**
+ * @license
+ * Copyright Google LLC
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {Argv} from 'yargs';
+
+import {WorkflowsModule} from './workflow/cli.js';
+
+/** Build the parser for pull request commands. */
+export function buildPerfParser(localYargs: Argv) {
+  return localYargs.help().strict().demandCommand().command(WorkflowsModule);
+}

ng-dev/perf/workflow/BUILD.bazel

@@ -0,0 +1,13 @@
+load("//tools:defaults.bzl", "ts_library")
+
+ts_library(
+    name = "workflow",
+    srcs = glob(["*.ts"]),
+    visibility = ["//ng-dev:__subpackages__"],
+    deps = [
+        "//ng-dev/utils",
+        "@npm//@types/node",
+        "@npm//@types/yargs",
+        "@npm//yaml",
+    ],
+)

ng-dev/perf/workflow/cli.ts

@@ -0,0 +1,55 @@
+/**
+ * @license
+ * Copyright Google LLC
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {Argv, CommandModule} from 'yargs';
+import {measureWorkflow} from './workflow.js';
+import {loadWorkflows} from './loader.js';
+import {join} from 'path';
+import {determineRepoBaseDirFromCwd} from '../../utils/repo-directory.js';
+
+interface WorkflowsParams {
+  configFile: string;
+  json: boolean;
+}
+
+/** Builds the checkout pull request command. */
+function builder(yargs: Argv) {
+  return yargs
+    .option('config-file' as 'configFile', {
+      default: '.ng-dev/workflows.yml',
+      type: 'string',
+      description: 'The path to the workflow definitions in a yml file',
+    })
+    .option('json', {
+      default: false,
+      type: 'boolean',
+      description: 'Whether to ouput the results as a json object',
+    });
+}
+
+/** Handles the checkout pull request command. */
+async function handler({configFile, json}: WorkflowsParams) {
+  const workflows = await loadWorkflows(join(determineRepoBaseDirFromCwd(), configFile));
+  const results: {[key: string]: number} = {};
+  for (const workflow of workflows) {
+    const {name, duration} = await measureWorkflow(workflow);
+    results[name] = duration;
+  }
+
+  if (json) {
+    process.stdout.write(JSON.stringify(results));
+  }
+}
+
+/** yargs command module for checking out a PR  */
+export const WorkflowsModule: CommandModule<{}, WorkflowsParams> = {
+  handler,
+  builder,
+  command: 'workflows',
+  describe: 'Evaluate the performance of the provided workflows',
+};

ng-dev/perf/workflow/loader.ts

@@ -0,0 +1,14 @@
+import {readFile} from 'fs/promises';
+import {parse} from 'yaml';
+
+export interface Workflow {
+  name: string;
+  workflow: string;
+  prepare?: string;
+  cleanup?: string;
+}
+
+export async function loadWorkflows(src: string) {
+  const rawWorkflows = await readFile(src, {encoding: 'utf-8'});
+  return parse(rawWorkflows).workflows as Workflow[];
+}

ng-dev/perf/workflow/workflow.ts

@@ -0,0 +1,60 @@
+import {ChildProcess} from '../../utils/child-process.js';
+import {green} from '../../utils/logging.js';
+import {Spinner} from '../../utils/spinner.js';
+import {Workflow} from './loader.js';
+
+export async function measureWorkflow({name, workflow, prepare, cleanup}: Workflow) {
+  const spinner = new Spinner('');
+  try {
+    if (prepare) {
+      spinner.update('Preparing environment for workflow execution');
+      // Run the `prepare` commands to establish the environment, caching, etc prior to running the
+      // workflow.
+      await runCommands(prepare);
+      spinner.update('Environment preperation completed');
+    }
+
+    spinner.update(`Executing workflow (${name})`);
+    // Mark the start time of the workflow, execute all of the commands provided in the workflow and
+    // then mark the ending time.
+    performance.mark('start');
+    await runCommands(workflow);
+    performance.mark('end');
+
+    spinner.update('Workflow completed');
+
+    if (cleanup) {
+      spinner.update('Cleaning up environment after workflow');
+      // Run the clean up commands to reset the environment and undo changes made during the workflow.
+      await runCommands(cleanup);
+      spinner.update('Environment cleanup complete');
+    }
+
+    const results = performance.measure(name, 'start', 'end');
+
+    spinner.complete(` ${green('✓')} ${name}: ${results.duration.toFixed(2)}ms`);
+
+    return results.toJSON();
+  } finally {
+    spinner.complete();
+  }
+}
+
+/**
+ * Run a set of commands provided as a multiline text block. Commands are assumed to always be
+ * provided on a single line.
+ */
+async function runCommands(cmds?: string) {
+  cmds = cmds?.trim();
+  if (!cmds) {
+    return;
+  }
+  let commands = cmds
+    .split('\n')
+    .filter((_) => !!_)
+    .map((cmdStr: string) => cmdStr.trim().split(' '));
+
+  for (let [cmd, ...args] of commands) {
+    await ChildProcess.spawn(cmd, args, {mode: 'silent'});
+  }
+}

ng-dev/utils/spinner.ts

@@ -14,15 +14,20 @@ const hideCursor = '\x1b[?25l';
 const showCursor = '\x1b[?25h';
 
 export class Spinner {
+  /** Whether the spinner is currently running. */
+  private isRunning = true;
   /** The id of the interval being used to trigger frame printing. */
   private intervalId = setInterval(() => this.printFrame(), 125);
   /** The characters to iterate through to create the appearance of spinning in the spinner. */
   private spinnerCharacters = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
   /** The index of the spinner character used in the frame. */
   private currentSpinnerCharacterIndex = 0;
+  /** The current text of the spinner. */
+  private text: string = '';
 
-  constructor(private text: string) {
+  constructor(text: string) {
     process.stdout.write(hideCursor);
+    this.update(text);
   }
 
   /** Get the next spinner character. */
@@ -44,12 +49,24 @@ export class Spinner {
   /** Updates the spinner text with the provided text. */
   update(text: string) {
     this.text = text;
+    this.printFrame(this.spinnerCharacters[this.currentSpinnerCharacterIndex]);
   }
 
   /** Completes the spinner. */
-  complete() {
+  complete(): void;
+  complete(text: string): void;
+  complete(text?: string) {
+    if (!this.isRunning) {
+      return;
+    }
     clearInterval(this.intervalId);
-    process.stdout.write('\n');
+    clearLine(process.stdout, 1);
+    cursorTo(process.stdout, 0);
+    if (text) {
+      process.stdout.write(text);
+      process.stdout.write('\n');
+    }
     process.stdout.write(showCursor);
+    this.isRunning = false;
   }
 }