fix(cli): write notices to stderr or don't write them at all (#221)

(Replaces #188).

On CI systems, the CDK CLI tries to avoid writing to `stderr` because
there are a couple of CI systems that are commonly configured to fail if
any output is written to `stderr`. That means all output, like notices,
must go to `stdout`.

Some commands (like `cdk synth` or `cdk bootstrap --show-template`)
produce usable output on `stdout`, and these are commonly scripted, like
piping their output to a file.

However, because notices must go to `stdout`, these now interfere with
the output of these commands.

This needs a more thorough reworking of the CLI output streams, but
there is a risk of affecting users who are currently relying on the fact
that all output goes to `stdout`.

In this PR, we are doing the first steps to solving this situation:

- Notices will always go to `stderr`, so that they will never interfere
with `stdout` anymore.
- We try to detect what CI system we are running on, and we will
completely suppress notices *unless* we determine that we are running on
a CI system where it is "safe" to write to `sterr` (fail closed).

"Safe" in this case means that the CI system doesn't come with an easy
to toggle checkbox that makes commands fail based on what they print,
instead of their exit codes. The only systems I'm aware of that have
this checkbox are "Azure DevOps", and "TeamCity running PowerShell
scripts".

Even though we know the systems that are "unsafe", we will only show
notices on systems known to be "safe".

Fixes aws/aws-cdk#33589.

Also in this PR, because this grew.

* Introduce `IoDefaultMessages` in the CLI package, which helps migrate
"legacy" logging code to just emit default warning/info/etc messages to
the IoHost.
* Removed the ability to log with a `{ message: 'asdf' }` object to the
global logger functions. This wasn't being used anywhere other than
tests, and it's sort of pointless: if you know the code you should be
using the `MessageMaker` to make a message object; if you don't know the
code you can emit a string. There is no need to look up the right code
given a level and a message object.
* Make it possible for result types to be any type, not just object
types. This is necessary to cover the "result" from legacy logging,
where the result is just a string.
* Updated many tests in a test file (`cli-io-host.test.ts`) that failed
type checking, but succeeded running, and therefore didn't fail the
build of #220.
* Centralized `TestIoHost` into the helper package, and renamed it to
`MockIoHost`.
* Introducing a `FakeIoHost` in the CLI package to assert on messages
emitted to an `IoHost`.

---
By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache-2.0 license

---------

Signed-off-by: github-actions <[email protected]>
Co-authored-by: github-actions <[email protected]>

Author: rix0rrr

packages/@aws-cdk/tmp-toolkit-helpers/src/api/index.ts

@@ -1,3 +1,4 @@
 export * from './cloud-assembly';
 export * from './io';
 export * from './toolkit-error';
+export * from './require-approval';

packages/@aws-cdk/tmp-toolkit-helpers/src/api/io/private/index.ts

@@ -4,3 +4,5 @@ export * from './span';
 export * from './message-maker';
 export * from './messages';
 export * from './types';
+export * from './io-default-messages';
+export * from './testing';

packages/@aws-cdk/tmp-toolkit-helpers/src/api/io/private/io-default-messages.ts

@@ -0,0 +1,65 @@
+import * as util from 'util';
+import type { ActionLessMessage, ActionLessRequest, IoHelper } from './io-helper';
+import type { IoMessageMaker } from './message-maker';
+import { IO } from './messages';
+
+/**
+ * Helper class to emit standard log messages to an IoHost
+ *
+ * It wraps an `IoHelper`, and adds convenience methods to emit default messages
+ * for the various log levels.
+ */
+export class IoDefaultMessages {
+  constructor(private readonly ioHelper: IoHelper) {
+  }
+
+  public notify(msg: ActionLessMessage<unknown>): Promise<void> {
+    return this.ioHelper.notify(msg);
+  }
+
+  public requestResponse<T, U>(msg: ActionLessRequest<T, U>): Promise<U> {
+    return this.ioHelper.requestResponse(msg);
+  }
+
+  public error(input: string, ...args: unknown[]) {
+    this.emitMessage(IO.DEFAULT_TOOLKIT_ERROR, input, ...args);
+  }
+
+  public warn(input: string, ...args: unknown[]) {
+    this.emitMessage(IO.DEFAULT_TOOLKIT_WARN, input, ...args);
+  }
+
+  public warning(input: string, ...args: unknown[]) {
+    this.emitMessage(IO.DEFAULT_TOOLKIT_WARN, input, ...args);
+  }
+
+  public info(input: string, ...args: unknown[]) {
+    this.emitMessage(IO.DEFAULT_TOOLKIT_INFO, input, ...args);
+  }
+
+  public debug(input: string, ...args: unknown[]) {
+    this.emitMessage(IO.DEFAULT_TOOLKIT_DEBUG, input, ...args);
+  }
+
+  public trace(input: string, ...args: unknown[]) {
+    this.emitMessage(IO.DEFAULT_TOOLKIT_TRACE, input, ...args);
+  }
+
+  public result(input: string, ...args: unknown[]) {
+    const message = args.length > 0 ? util.format(input, ...args) : input;
+    // This is just the default "info" message but with a level of "result"
+    void this.ioHelper.notify({
+      time: new Date(),
+      code: IO.DEFAULT_TOOLKIT_INFO.code,
+      level: 'result',
+      message,
+      data: undefined,
+    });
+  }
+
+  private emitMessage(maker: IoMessageMaker<void>, input: string, ...args: unknown[]) {
+    // Format message if args are provided
+    const message = args.length > 0 ? util.format(input, ...args) : input;
+    void this.ioHelper.notify(maker.msg(message));
+  }
+}

packages/@aws-cdk/tmp-toolkit-helpers/src/api/io/private/message-maker.ts

@@ -100,7 +100,7 @@ export const debug = <T = AbsentData>(details: CodeInfoMaybeInterface<T>) => mes
 export const info = <T = AbsentData>(details: CodeInfoMaybeInterface<T>) => message<T>('info', details);
 export const warn = <T = AbsentData>(details: CodeInfoMaybeInterface<T>) => message<T>('warn', details);
 export const error = <T = AbsentData>(details: CodeInfoMaybeInterface<T>) => message<T>('error', details);
-export const result = <T extends object = ImpossibleType>(details: Required<CodeInfo>) => message<T extends object ? T : undefined>('result', details);
+export const result = <T extends object = ImpossibleType>(details: Required<CodeInfo>) => message<T>('result', details);
 
 interface RequestInfo<U> extends CodeInfo {
   readonly defaultResponse: U;

packages/@aws-cdk/tmp-toolkit-helpers/src/api/io/private/messages.ts

@@ -35,6 +35,14 @@ export const IO = {
     code: 'CDK_TOOLKIT_W0000',
     description: 'Default warning messages emitted from the Toolkit',
   }),
+  DEFAULT_TOOLKIT_ERROR: make.error({
+    code: 'CDK_TOOLKIT_E0000',
+    description: 'Default error messages emitted from the Toolkit',
+  }),
+  DEFAULT_TOOLKIT_TRACE: make.trace({
+    code: 'CDK_TOOLKIT_I0000',
+    description: 'Default trace messages emitted from the Toolkit',
+  }),
 
   // 1: Synth (1xxx)
   CDK_TOOLKIT_I1000: make.info<Duration>({
@@ -331,6 +339,24 @@ export const IO = {
     interface: 'ErrorPayload',
   }),
 
+  // Notices
+  CDK_TOOLKIT_I0100: make.info({
+    code: 'CDK_TOOLKIT_I0100',
+    description: 'Notices decoration (the header or footer of a list of notices)',
+  }),
+  CDK_TOOLKIT_W0101: make.warn({
+    code: 'CDK_TOOLKIT_W0101',
+    description: 'A notice that is marked as a warning',
+  }),
+  CDK_TOOLKIT_E0101: make.error({
+    code: 'CDK_TOOLKIT_E0101',
+    description: 'A notice that is marked as an error',
+  }),
+  CDK_TOOLKIT_I0101: make.info({
+    code: 'CDK_TOOLKIT_I0101',
+    description: 'A notice that is marked as informational',
+  }),
+
   // Assembly codes
   CDK_ASSEMBLY_I0010: make.debug({
     code: 'CDK_ASSEMBLY_I0010',

packages/@aws-cdk/tmp-toolkit-helpers/src/api/io/private/testing/fake-io-host.ts

@@ -0,0 +1,43 @@
+import type { IIoHost } from '../../io-host';
+import type { IoMessage, IoMessageLevel, IoRequest } from '../../io-message';
+
+/**
+ * An implementation of `IIoHost` that records messages and lets you assert on what was logged
+ *
+ * It's like `TestIoHost`, but comes with a predefined implementation for `notify`
+ * that appends all messages to an in-memory array, and comes with a helper function
+ * `expectMessage()` to test for the existence of a function in that array.
+ *
+ * Has a public mock for `requestResponse` that you configure like any
+ * other mock function.
+ *
+ * # How to use
+ *
+ * Either create a new instance of this class for every test, or call `clear()`
+ * on it between runs.
+ */
+export class FakeIoHost implements IIoHost {
+  public messages: Array<IoMessage<unknown>> = [];
+  public requestResponse!: <T, U>(msg: IoRequest<T, U>) => Promise<U>;
+
+  constructor() {
+    this.clear();
+  }
+
+  public clear() {
+    this.messages.splice(0, this.messages.length);
+    this.requestResponse = jest.fn().mockRejectedValue(new Error('requestResponse not mocked'));
+  }
+
+  public async notify(msg: IoMessage<unknown>): Promise<void> {
+    this.messages.push(msg);
+  }
+
+  public expectMessage(m: { containing: string; level?: IoMessageLevel }) {
+    expect(this.messages).toContainEqual(expect.objectContaining({
+      ...m.level ? { level: m.level } : undefined,
+      // Can be a partial string as well
+      message: expect.stringContaining(m.containing),
+    }));
+  }
+}

packages/@aws-cdk/tmp-toolkit-helpers/src/api/io/private/testing/index.ts

@@ -0,0 +1,2 @@
+export * from './test-io-host';
+export * from './fake-io-host';

…oolkit-lib/test/_helpers/test-io-host.ts …c/api/io/private/testing/test-io-host.tspackages/@aws-cdk/toolkit-lib/test/_helpers/test-io-host.ts renamed to packages/@aws-cdk/tmp-toolkit-helpers/src/api/io/private/testing/test-io-host.ts packages/@aws-cdk/toolkit-lib/test/_helpers/test-io-host.ts renamed to packages/@aws-cdk/tmp-toolkit-helpers/src/api/io/private/testing/test-io-host.ts

@@ -1,11 +1,19 @@
-import type { IIoHost, IoMessage, IoMessageLevel, IoRequest } from '../../lib';
-import { RequireApproval } from '../../lib';
-import { isMessageRelevantForLevel } from '../../lib/api/shared-private';
+import { RequireApproval } from '../../../require-approval';
+import type { IIoHost } from '../../io-host';
+import type { IoMessage, IoMessageLevel, IoRequest } from '../../io-message';
+import { isMessageRelevantForLevel } from '../level-priority';
 
 /**
- * A test implementation of IIoHost that does nothing but can by spied on.
- * Optionally set a level to filter out all irrelevant messages.
- * Optionally set a approval level.
+ * A test implementation of IIoHost that does nothing but can be spied on.
+ *
+ * Includes a level to filter out irrelevant messages, defaults to `info`.
+ *
+ * Optionally set an approval level for code `CDK_TOOLKIT_I5060`.
+ *
+ * # How to use
+ *
+ * Configure and reset the `notifySpy` and `requestSpy` members as you would any
+ * mock function.
  */
 export class TestIoHost implements IIoHost {
   public readonly notifySpy: jest.Mock<any, any, any>;

packages/@aws-cdk/tmp-toolkit-helpers/src/api/require-approval.ts

@@ -0,0 +1,17 @@
+/**
+ * @deprecated
+ */
+export enum RequireApproval {
+  /**
+   * Never require any security approvals
+   */
+  NEVER = 'never',
+  /**
+   * Any security changes require an approval
+   */
+  ANY_CHANGE = 'any-change',
+  /**
+   * Require approval only for changes that are access broadening
+   */
+  BROADENING = 'broadening',
+}

packages/@aws-cdk/toolkit-lib/docs/message-registry.md

@@ -9,6 +9,8 @@ group: Documents
 | `CDK_TOOLKIT_I0000` | Default info messages emitted from the Toolkit | `info` | n/a |
 | `CDK_TOOLKIT_I0000` | Default debug messages emitted from the Toolkit | `debug` | n/a |
 | `CDK_TOOLKIT_W0000` | Default warning messages emitted from the Toolkit | `warn` | n/a |
+| `CDK_TOOLKIT_E0000` | Default error messages emitted from the Toolkit | `error` | n/a |
+| `CDK_TOOLKIT_I0000` | Default trace messages emitted from the Toolkit | `trace` | n/a |
 | `CDK_TOOLKIT_I1000` | Provides synthesis times. | `info` | {@link Duration} |
 | `CDK_TOOLKIT_I1001` | Cloud Assembly synthesis is starting | `trace` | {@link StackSelectionDetails} |
 | `CDK_TOOLKIT_I1901` | Provides stack data | `result` | {@link StackAndAssemblyData} |
@@ -64,6 +66,10 @@ group: Documents
 | `CDK_TOOLKIT_I9100` | Bootstrap progress | `info` | {@link BootstrapEnvironmentProgress} |
 | `CDK_TOOLKIT_I9900` | Bootstrap results on success | `result` | [cxapi.Environment](https://docs.aws.amazon.com/cdk/api/v2/docs/@aws-cdk_cx-api.Environment.html) |
 | `CDK_TOOLKIT_E9900` | Bootstrap failed | `error` | {@link ErrorPayload} |
+| `CDK_TOOLKIT_I0100` | Notices decoration (the header or footer of a list of notices) | `info` | n/a |
+| `CDK_TOOLKIT_W0101` | A notice that is marked as a warning | `warn` | n/a |
+| `CDK_TOOLKIT_E0101` | A notice that is marked as an error | `error` | n/a |
+| `CDK_TOOLKIT_I0101` | A notice that is marked as informational | `info` | n/a |
 | `CDK_ASSEMBLY_I0010` | Generic environment preparation debug messages | `debug` | n/a |
 | `CDK_ASSEMBLY_W0010` | Emitted if the found framework version does not support context overflow | `warn` | n/a |
 | `CDK_ASSEMBLY_I0042` | Writing updated context | `debug` | {@link UpdatedContext} |