performance: Create a starter set of tools for measuring the CPU/memory impact of codei (#5446)

Problem:
- We are sometimes running into problems where functions are using a ton of memory/cpu but we have no way to figure that out ahead of time without manually debugging/customer reports

Solution:
- Create a starter set of tools that allows us to measure the impact of different parts of code. These tools should be used as needed and surround heavy computational parts of the code i.e. collecting all files in the workspace, reading them in and putting them in a zip

Author: jpinkney-aws

packages/core/src/shared/performance/performance.ts

@@ -0,0 +1,118 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import assert from 'assert'
+import { getLogger } from '../logger'
+import { isWeb } from '../extensionGlobals'
+
+interface PerformanceMetrics {
+    cpuUsage: number
+    heapTotal: number
+    duration: number
+}
+
+interface TestOptions {
+    darwin?: Partial<PerformanceMetrics>
+    win32?: Partial<PerformanceMetrics>
+    linux?: Partial<PerformanceMetrics>
+}
+
+export interface PerformanceSpan<T> {
+    value: T
+    performance: PerformanceMetrics
+}
+
+export class PerformanceTracker {
+    #startPerformance:
+        | {
+              cpuUsage: NodeJS.CpuUsage
+              memory: number
+              duration: [number, number]
+          }
+        | undefined
+
+    constructor(private readonly name: string) {}
+
+    static enabled(name: string, trackPerformance?: boolean): boolean {
+        return name === 'function_call' && (trackPerformance ?? false) && !isWeb()
+    }
+
+    start() {
+        this.#startPerformance = {
+            cpuUsage: process.cpuUsage(),
+            memory: process.memoryUsage().heapTotal,
+            duration: process.hrtime(),
+        }
+    }
+
+    stop(): PerformanceMetrics | undefined {
+        if (this.#startPerformance) {
+            const endCpuUsage = process.cpuUsage(this.#startPerformance?.cpuUsage)
+            const userCpuUsage = endCpuUsage.user / 1000000
+            const systemCpuUsage = endCpuUsage.system / 1000000
+
+            const elapsedTime = process.hrtime(this.#startPerformance.duration)
+            const duration = elapsedTime[0] + elapsedTime[1] / 1e9 // convert microseconds to seconds
+            const usage = ((userCpuUsage + systemCpuUsage) / duration) * 100 // convert to percentage
+
+            const endMemoryUsage = process.memoryUsage().heapTotal - this.#startPerformance?.memory
+            const endMemoryUsageInMB = endMemoryUsage / (1024 * 1024) // converting bytes to MB
+
+            return {
+                cpuUsage: usage,
+                heapTotal: endMemoryUsageInMB,
+                duration,
+            }
+        } else {
+            getLogger().debug(`PerformanceTracker: start performance not defined for ${this.name}`)
+        }
+    }
+}
+
+export function performanceTest(options: TestOptions, name: string, fn: () => Promise<void>): Mocha.Test
+export function performanceTest(options: TestOptions, name: string, fn: () => void): Mocha.Test
+export function performanceTest(options: TestOptions, name: string, fn: () => void | Promise<void>) {
+    const testOption = options[process.platform as 'linux' | 'darwin' | 'win32']
+
+    const performanceTracker = new PerformanceTracker(name)
+
+    return it(name, async () => {
+        performanceTracker.start()
+        await fn()
+        const metrics = performanceTracker.stop()
+        if (!metrics) {
+            assert.fail('Performance metrics not found')
+        }
+        assertPerformanceMetrics(metrics, name, testOption)
+    })
+}
+
+function assertPerformanceMetrics(
+    performanceMetrics: PerformanceMetrics,
+    name: string,
+    testOption?: Partial<PerformanceMetrics>
+) {
+    const expectedCPUUsage = testOption?.cpuUsage ?? 50
+    const foundCPUUsage = performanceMetrics.cpuUsage
+
+    assert(
+        foundCPUUsage < expectedCPUUsage,
+        `Expected total CPU usage for ${name} to be less than ${expectedCPUUsage}. Actual CPU usage was ${foundCPUUsage}`
+    )
+
+    const expectedMemoryUsage = testOption?.heapTotal ?? 400
+    const foundMemoryUsage = performanceMetrics.heapTotal
+    assert(
+        foundMemoryUsage < expectedMemoryUsage,
+        `Expected total memory usage for ${name} to be less than ${expectedMemoryUsage}. Actual memory usage was ${foundMemoryUsage}`
+    )
+
+    const expectedDuration = testOption?.duration ?? 5
+    const foundDuration = performanceMetrics.duration
+    assert(
+        foundDuration < expectedDuration,
+        `Expected total duration for ${name} to be less than ${expectedDuration}. Actual duration was ${foundDuration}`
+    )
+}

packages/core/src/shared/telemetry/spans.ts

@@ -23,6 +23,7 @@ import {
     getTelemetryResult,
 } from '../errors'
 import { entries, NumericKeys } from '../utilities/tsUtils'
+import { PerformanceTracker } from '../performance/performance'
 
 const AsyncLocalStorage: typeof AsyncLocalStorageClass =
     require('async_hooks').AsyncLocalStorage ??
@@ -97,6 +98,9 @@ export type SpanOptions = {
     /** True if this span should emit its telemetry events. Defaults to true if undefined. */
     emit?: boolean
 
+    /** True if this span should emit performance metrics acquired when running the function. Defaults to false if undefined */
+    trackPerformance?: boolean
+
     /**
      * Adds a function entry to the span stack.
      *
@@ -134,6 +138,7 @@ export type SpanOptions = {
 export class TelemetrySpan<T extends MetricBase = MetricBase> {
     #startTime?: Date
     #options: SpanOptions
+    #performance?: PerformanceTracker
 
     private readonly state: Partial<T> = {}
     private readonly definition = definitions[this.name] ?? {
@@ -157,6 +162,7 @@ export class TelemetrySpan<T extends MetricBase = MetricBase> {
             // do emit by default
             emit: options?.emit === undefined ? true : options.emit,
             functionId: options?.functionId,
+            trackPerformance: PerformanceTracker.enabled(this.name, options?.trackPerformance),
         }
     }
 
@@ -195,6 +201,10 @@ export class TelemetrySpan<T extends MetricBase = MetricBase> {
      */
     public start(): this {
         this.#startTime = new globals.clock.Date()
+        if (this.#options.trackPerformance) {
+            ;(this.#performance ??= new PerformanceTracker(this.name)).start()
+        }
+
         return this
     }
 
@@ -206,6 +216,19 @@ export class TelemetrySpan<T extends MetricBase = MetricBase> {
     public stop(err?: unknown): void {
         const duration = this.startTime !== undefined ? globals.clock.Date.now() - this.startTime.getTime() : undefined
 
+        if (this.#options.trackPerformance) {
+            // TODO add these to the global metrics, right now it just forces them in the telemetry and ignores the type
+            // if someone enables this action
+            const performanceMetrics = this.#performance?.stop()
+            if (performanceMetrics) {
+                this.record({
+                    cpuUsage: performanceMetrics.cpuUsage,
+                    heapTotal: performanceMetrics.heapTotal,
+                    functionName: this.#options.functionId?.name ?? this.name,
+                } as any)
+            }
+        }
+
         if (this.#options.emit) {
             this.emit({
                 duration,
@@ -307,11 +330,7 @@ export class TelemetryTracer extends TelemetryBase {
      * All changes made to {@link attributes} (via {@link record}) during the execution are
      * reverted after the execution completes.
      */
-    public run<T, U extends MetricName>(
-        name: U,
-        fn: (span: Metric<MetricShapes[U]>) => T,
-        options?: SpanOptions | undefined
-    ): T {
+    public run<T, U extends MetricName>(name: U, fn: (span: Metric<MetricShapes[U]>) => T, options?: SpanOptions): T {
         const span = this.createSpan(name, options).start()
         const frame = this.switchContext(span)