fixup! feat(ng-dev): create workflow performance testing tooling

Author: josephperrott

.ng-dev/dx-perf-workflows.yml

@@ -0,0 +1,17 @@
+workflows:
+  - name: Rerun a test
+    prepare:
+      - bazel clean;
+      - bazel build //ng-dev/utils/test;
+    workflow:
+      - bazel test //ng-dev/utils/test;
+      - git apply .ng-dev/perf-tests/test-rerun.diff;
+      - bazel test //ng-dev/utils/test;
+    cleanup:
+      - git apply -R .ng-dev/perf-tests/test-rerun.diff;
+
+  - name: Build Everything
+    prepare:
+      - bazel clean;
+    workflow:
+      - bazel build //...;

.ng-dev/workflows.yml

@@ -21,14 +21,14 @@ interface WorkflowsParams {
 function builder(yargs: Argv) {
   return yargs
     .option('config-file' as 'configFile', {
-      default: '.ng-dev/workflows.yml',
+      default: '.ng-dev/dx-perf-workflows.yml',
       type: 'string',
       description: 'The path to the workflow definitions in a yml file',
     })
     .option('json', {
       default: false,
       type: 'boolean',
-      description: 'Whether to ouput the results as a json object',
+      description: 'Whether to output the results as a json object',
     });
 }
 
@@ -46,7 +46,7 @@ async function handler({configFile, json}: WorkflowsParams) {
   }
 }
 
-/** yargs command module for checking out a PR  */
+/** yargs command module for checking out a PR. */
 export const WorkflowsModule: CommandModule<{}, WorkflowsParams> = {
   handler,
   builder,

ng-dev/perf/workflow/cli.ts

@@ -3,9 +3,9 @@ import {parse} from 'yaml';
 
 export interface Workflow {
   name: string;
-  workflow: string;
-  prepare?: string;
-  cleanup?: string;
+  workflow: string[];
+  prepare?: string[];
+  cleanup?: string[];
 }
 
 export async function loadWorkflows(src: string) {

ng-dev/perf/workflow/loader.ts

@@ -44,17 +44,11 @@ export async function measureWorkflow({name, workflow, prepare, cleanup}: Workfl
  * Run a set of commands provided as a multiline text block. Commands are assumed to always be
  * provided on a single line.
  */
-async function runCommands(cmds?: string) {
-  cmds = cmds?.trim();
-  if (!cmds) {
+async function runCommands(commands?: string[]) {
+  if (!commands || commands.length === 0) {
     return;
   }
-  let commands = cmds
-    .split('\n')
-    .filter((_) => !!_)
-    .map((cmdStr: string) => cmdStr.trim().split(' '));
-
-  for (let [cmd, ...args] of commands) {
-    await ChildProcess.spawn(cmd, args, {mode: 'silent'});
+  for (let cmd of commands) {
+    await ChildProcess.exec(cmd, {mode: 'silent'});
   }
 }

ng-dev/perf/workflow/workflow.ts

@@ -12,6 +12,8 @@ import {
   SpawnOptions as _SpawnOptions,
   spawnSync as _spawnSync,
   SpawnSyncOptions as _SpawnSyncOptions,
+  ExecOptions as _ExecOptions,
+  exec as _exec,
 } from 'child_process';
 import {Log} from './logging.js';
 
@@ -32,6 +34,14 @@ export interface SpawnOptions extends Omit<_SpawnOptions, 'shell' | 'stdio'> {
   input?: string;
 }
 
+/** Interface describing the options for exec-ing a process. */
+export interface ExecOptions extends Omit<_ExecOptions, 'shell' | 'stdio'> {
+  /** Console output mode. Defaults to "enabled". */
+  mode?: 'enabled' | 'silent' | 'on-error';
+  /** Whether to prevent exit codes being treated as failures. */
+  suppressErrorOnFailingExitCode?: boolean;
+}
+
 /** Interface describing the options for spawning an interactive process. */
 export type SpawnInteractiveCommandOptions = Omit<_SpawnOptions, 'shell' | 'stdio'>;
 
@@ -81,7 +91,7 @@ export abstract class ChildProcess {
     return new Promise((resolve, reject) => {
       const commandText = `${command} ${args.join(' ')}`;
       const outputMode = options.mode;
-      const env = getEnvironmentForNonInteractiveSpawn(options.env);
+      const env = getEnvironmentForNonInteractiveCommand(options.env);
 
       Log.debug(`Executing command: ${commandText}`);
 
@@ -148,7 +158,7 @@ export abstract class ChildProcess {
    */
   static spawnSync(command: string, args: string[], options: SpawnSyncOptions = {}): SpawnResult {
     const commandText = `${command} ${args.join(' ')}`;
-    const env = getEnvironmentForNonInteractiveSpawn(options.env);
+    const env = getEnvironmentForNonInteractiveCommand(options.env);
 
     Log.debug(`Executing command: ${commandText}`);
 
@@ -168,6 +178,63 @@ export abstract class ChildProcess {
 
     throw new Error(stderr);
   }
+
+  static exec(command: string, options: ExecOptions = {}) {
+    return new Promise((resolve, reject) => {
+      const outputMode = options.mode;
+      const env = getEnvironmentForNonInteractiveCommand(options.env);
+
+      Log.debug(`Executing command: ${command}`);
+
+      const childProcess = _exec(command, {...options, env});
+      let logOutput = '';
+      let stdout = '';
+      let stderr = '';
+
+      // Capture the stdout separately so that it can be passed as resolve value.
+      // This is useful if commands return parsable stdout.
+      childProcess.stderr?.on('data', (message) => {
+        stderr += message;
+        logOutput += message;
+        // If console output is enabled, print the message directly to the stderr. Note that
+        // we intentionally print all output to stderr as stdout should not be polluted.
+        if (outputMode === undefined || outputMode === 'enabled') {
+          process.stderr.write(message);
+        }
+      });
+
+      childProcess.stdout?.on('data', (message) => {
+        stdout += message;
+        logOutput += message;
+        // If console output is enabled, print the message directly to the stderr. Note that
+        // we intentionally print all output to stderr as stdout should not be polluted.
+        if (outputMode === undefined || outputMode === 'enabled') {
+          process.stderr.write(message);
+        }
+      });
+
+      // The `close` event is used because the process is guaranteed to have completed writing to
+      // stdout and stderr, using the `exit` event can cause inconsistent information in stdout and
+      // stderr due to a race condition around exiting.
+      childProcess.on('close', (exitCode, signal) => {
+        const exitDescription =
+          exitCode !== null ? `exit code "${exitCode}"` : `signal "${signal}"`;
+        const printFn = outputMode === 'on-error' ? Log.error : Log.debug;
+        const status = statusFromExitCodeAndSignal(exitCode, signal);
+
+        printFn(`Command "${command}" completed with ${exitDescription}.`);
+        printFn(`Process output: \n${logOutput}`);
+
+        // On success, resolve the promise. Otherwise reject with the captured stderr
+        // and stdout log output if the output mode was set to `silent`.
+        if (status === 0 || options.suppressErrorOnFailingExitCode) {
+          resolve({stdout, stderr, status});
+        } else {
+          reject(outputMode === 'silent' ? logOutput : undefined);
+        }
+      });
+    });
+  }
 }
 /**
  * Convert the provided exitCode and signal to a single status code.
@@ -188,7 +255,7 @@ function statusFromExitCodeAndSignal(exitCode: number | null, signal: NodeJS.Sig
  * Currently we enable `FORCE_COLOR` since non-interactive spawn's with
  * non-inherited `stdio` will not have colors enabled due to a missing TTY.
  */
-function getEnvironmentForNonInteractiveSpawn(
+function getEnvironmentForNonInteractiveCommand(
   userProvidedEnv?: NodeJS.ProcessEnv,
 ): NodeJS.ProcessEnv {
   // Pass through the color level from the TTY/process performing the `spawn` call.