fix(telemetry): Crash monitoring fixes (aws#5741)

## Problem

Crash monitoring is reporting incorrect crash metrics.
This seems to be due to various filesystem errors such as eperm (even
though we were doing an operation on a file we created), enospc (the
user ran out of space on their machine, and other errors.

Because of this we ran in to situations where our state did not reflect
reality, and due to this certain extension
instances were seen as crashed.

## Solution

- Determine if a filesystem is reliable on a machine (try a bunch of
different filesystem flows and ensure nothing throws), if it is THEN we
start the crash monitoring process. Otherwise we do not run it since we
cannot rely it will be accurate.
- We added a `function_call` metric to allow us to determine the ratio
of successes to failures
- Add retries to critical filesystem operations such as the heartbeats
and deleting a crashed extension instance from the state.
- Other various fixes


---

<!--- REMINDER: Ensure that your PR meets the guidelines in
CONTRIBUTING.md -->

License: I confirm that my contribution is made under the terms of the
Apache 2.0 license.

---------

Signed-off-by: nkomonen-amazon <[email protected]>

Author: nkomonen-amazon

packages/amazonq/src/extensionNode.ts

@@ -32,7 +32,8 @@ export async function activate(context: vscode.ExtensionContext) {
  * the code compatible with web and move it to {@link activateAmazonQCommon}.
  */
 async function activateAmazonQNode(context: vscode.ExtensionContext) {
-    await (await CrashMonitoring.instance()).start()
+    // Intentionally do not await since this is slow and non-critical
+    void (await CrashMonitoring.instance())?.start()
 
     const extContext = {
         extensionContext: context,
@@ -96,5 +97,5 @@ async function setupDevMode(context: vscode.ExtensionContext) {
 
 export async function deactivate() {
     // Run concurrently to speed up execution. stop() does not throw so it is safe
-    await Promise.all([(await CrashMonitoring.instance()).stop(), deactivateCommon()])
+    await Promise.all([(await CrashMonitoring.instance())?.shutdown(), deactivateCommon()])
 }

packages/core/src/extensionNode.ts

@@ -78,7 +78,8 @@ export async function activate(context: vscode.ExtensionContext) {
         // IMPORTANT: If you are doing setup that should also work in web mode (browser), it should be done in the function below
         const extContext = await activateCommon(context, contextPrefix, false)
 
-        await (await CrashMonitoring.instance()).start()
+        // Intentionally do not await since this can be slow and non-critical
+        void (await CrashMonitoring.instance())?.start()
 
         initializeCredentialsProviderManager()
 
@@ -254,7 +255,7 @@ export async function activate(context: vscode.ExtensionContext) {
 
 export async function deactivate() {
     // Run concurrently to speed up execution. stop() does not throw so it is safe
-    await Promise.all([await (await CrashMonitoring.instance()).stop(), deactivateCommon()])
+    await Promise.all([await (await CrashMonitoring.instance())?.shutdown(), deactivateCommon()])
     await globals.resourceManager.dispose()
 }