implement telemetry for exceeding thresholds

Author: Hweinstock

packages/core/src/shared/telemetry/vscodeTelemetry.json

@@ -1,5 +1,11 @@
 {
     "types": [
+        {
+            "name": "systemResource",
+            "type": "string",
+            "allowedValues": ["cpu", "memory"],
+            "description": "The type of system resource being measured"
+        },
         {
             "name": "amazonGenerateApproachLatency",
             "type": "double",
@@ -372,6 +378,16 @@
                 }
             ]
         },
+        {
+            "name": "ide_childProcessWarning",
+            "description": "Child Process warning due to high system usage",
+            "metadata": [
+                {
+                    "type": "systemResource",
+                    "required": true
+                }
+            ]
+        },
         {
             "name": "vscode_executeCommand",
             "description": "Emitted whenever a registered Toolkit command is executed",

packages/core/src/shared/utilities/processUtils.ts

@@ -9,6 +9,7 @@ import * as logger from '../logger'
 import { Timeout, CancellationError, waitUntil } from './timeoutUtils'
 import { PollingSet } from './pollingSet'
 import { getLogger } from '../logger/logger'
+import { telemetry } from '../telemetry'
 
 export interface RunParameterContext {
     /** Reports an error parsed from the stdin/stdout streams. */
@@ -100,18 +101,29 @@ export class ChildProcessTracker {
     }
 
     private async checkProcessUsage(pid: number): Promise<void> {
+        const doesExceedThreshold = (resource: keyof ProcessStats, value: number) => {
+            const threshold = ChildProcessTracker.thresholds[resource]
+            return value > threshold
+        }
+        const warn = (resource: keyof ProcessStats, value: number) => {
+            telemetry.ide_childProcessWarning.run((span) => {
+                ChildProcessTracker.logger.warn(`Process ${pid} exceeded ${resource} threshold: ${value}`)
+                span.record({ systemResource: resource })
+            })
+        }
+
         if (!this.#pids.has(pid)) {
             ChildProcessTracker.logger.warn(`Missing process with id ${pid}`)
             return
         }
         const stats = await this.getUsage(pid)
-        if (stats) {
-            ChildProcessTracker.logger.debug(`Process ${pid} usage: %O`, stats)
-            if (stats.memory > ChildProcessTracker.thresholds.memory) {
-                ChildProcessTracker.logger.warn(`Process ${pid} exceeded memory threshold: ${stats.memory}`)
-            }
-            if (stats.cpu > ChildProcessTracker.thresholds.cpu) {
-                ChildProcessTracker.logger.warn(`Process ${pid} exceeded cpu threshold: ${stats.cpu}`)
+        if (!stats) {
+            ChildProcessTracker.logger.warn(`Failed to get process stats for ${pid}`)
+            return
+        }
+        for (const resource of Object.keys(ChildProcessTracker.thresholds) as (keyof ProcessStats)[]) {
+            if (doesExceedThreshold(resource, stats[resource])) {
+                warn(resource as keyof ProcessStats, stats[resource])
             }
         }
     }

packages/core/src/test/shared/utilities/processUtils.test.ts

@@ -19,7 +19,7 @@ import { sleep } from '../../../shared/utilities/timeoutUtils'
 import { Timeout, waitUntil } from '../../../shared/utilities/timeoutUtils'
 import { fs } from '../../../shared'
 import * as FakeTimers from '@sinonjs/fake-timers'
-import { installFakeClock } from '../../testUtil'
+import { assertTelemetry, installFakeClock } from '../../testUtil'
 import { isWin } from '../../../shared/vscode/env'
 import { assertLogsContain } from '../../globalSetup.test'
 
@@ -445,7 +445,7 @@ describe('ChildProcessTracker', function () {
         assert.strictEqual(tracker.size, 0, 'expected tracker to be empty')
     })
 
-    it('logs a warning message when system usage exceeds threshold', async function () {
+    it('logs a warning message and emits metric when system usage exceeds threshold', async function () {
         const runningProcess = startSleepProcess()
         tracker.add(runningProcess.childProcess)
 
@@ -462,10 +462,12 @@ describe('ChildProcessTracker', function () {
 
         await clock.tickAsync(ChildProcessTracker.pollingInterval)
         assertLogsContain('exceeded cpu threshold', false, 'warn')
+        assertTelemetry('ide_childProcessWarning', { systemResource: 'cpu' })
 
         usageMock.resolves(highMemory)
         await clock.tickAsync(ChildProcessTracker.pollingInterval)
         assertLogsContain('exceeded memory threshold', false, 'warn')
+        assertTelemetry('ide_childProcessWarning', { systemResource: 'memory' })
 
         await stopAndWait(runningProcess)
     })
@@ -485,7 +487,7 @@ describe('ChildProcessTracker', function () {
         await stopAndWait(runningProcess)
     })
 
-    it('does not log for processes within threshold', async function () {
+    it('does not log or emit telemetry for processes within threshold', async function () {
         const runningProcess = startSleepProcess()
 
         usageMock.resolves({
@@ -496,6 +498,7 @@ describe('ChildProcessTracker', function () {
         await clock.tickAsync(ChildProcessTracker.pollingInterval)
 
         assert.throws(() => assertLogsContain(runningProcess.childProcess.pid().toString(), false, 'warn'))
+        assert.throws(() => assertTelemetry('ide_childProcessWarning', {}))
 
         await stopAndWait(runningProcess)
     })