From d23214db7b504c832ba2518b5c7ede460718f578 Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 29 Apr 2025 21:40:52 -0400 Subject: [PATCH 1/6] feat: allow child process to configure custom thresholds --- packages/amazonq/src/lsp/client.ts | 2 + packages/core/src/amazonq/lsp/lspClient.ts | 2 + .../core/src/shared/lsp/utils/platform.ts | 4 +- .../core/src/shared/utilities/processUtils.ts | 39 ++++++++++---- .../shared/utilities/processUtils.test.ts | 52 ++++++++++++++++--- 5 files changed, 82 insertions(+), 17 deletions(-) diff --git a/packages/amazonq/src/lsp/client.ts b/packages/amazonq/src/lsp/client.ts index d813370cb0b..39fcb6f010d 100644 --- a/packages/amazonq/src/lsp/client.ts +++ b/packages/amazonq/src/lsp/client.ts @@ -55,11 +55,13 @@ export async function startLanguageServer( '--pre-init-encryption', '--set-credentials-encryption-key', ] + const memoryWarnThreshold = 200 * 1024 * 1024 // 200 MB const serverOptions = createServerOptions({ encryptionKey, executable: resourcePaths.node, serverModule, execArgv: argv, + warnThresholds: { memory: memoryWarnThreshold }, }) const documentSelector = [{ scheme: 'file', language: '*' }] diff --git a/packages/core/src/amazonq/lsp/lspClient.ts b/packages/core/src/amazonq/lsp/lspClient.ts index b992fef2fe3..0b1eb608d95 100644 --- a/packages/core/src/amazonq/lsp/lspClient.ts +++ b/packages/core/src/amazonq/lsp/lspClient.ts @@ -252,6 +252,7 @@ export async function activate(extensionContext: ExtensionContext, resourcePaths } const serverModule = resourcePaths.lsp + const memoryWarnThreshold = 600 * 1024 * 1024 // 600 MB const serverOptions = createServerOptions({ encryptionKey: key, @@ -259,6 +260,7 @@ export async function activate(extensionContext: ExtensionContext, resourcePaths serverModule, // TODO(jmkeyes): we always use the debug options...? execArgv: debugOptions.execArgv, + warnThresholds: { memory: memoryWarnThreshold }, }) const documentSelector = [{ scheme: 'file', language: '*' }] diff --git a/packages/core/src/shared/lsp/utils/platform.ts b/packages/core/src/shared/lsp/utils/platform.ts index 5d96ef496f4..bab49d9b2ec 100644 --- a/packages/core/src/shared/lsp/utils/platform.ts +++ b/packages/core/src/shared/lsp/utils/platform.ts @@ -78,18 +78,20 @@ export function createServerOptions({ executable, serverModule, execArgv, + warnThresholds, }: { encryptionKey: Buffer executable: string serverModule: string execArgv: string[] + warnThresholds?: { cpu?: number; memory?: number } }) { return async () => { const args = [serverModule, ...execArgv] if (isDebugInstance()) { args.unshift('--inspect=6080') } - const lspProcess = new ChildProcess(executable, args) + const lspProcess = new ChildProcess(executable, args, { warnThresholds }) // this is a long running process, awaiting it will never resolve void lspProcess.run() diff --git a/packages/core/src/shared/utilities/processUtils.ts b/packages/core/src/shared/utilities/processUtils.ts index 0204736f500..fa61de70fe8 100644 --- a/packages/core/src/shared/utilities/processUtils.ts +++ b/packages/core/src/shared/utilities/processUtils.ts @@ -44,6 +44,13 @@ export interface ChildProcessOptions { onStdout?: (text: string, context: RunParameterContext) => void /** Callback for intercepting text from the stderr stream. */ onStderr?: (text: string, context: RunParameterContext) => void + /** Thresholds to configure warning logs */ + warnThresholds?: { + /** Threshold for memory usage in bytes */ + memory?: number + /** Threshold for CPU usage in percent */ + cpu?: number + } } export interface ChildProcessRunOptions extends Omit { @@ -60,8 +67,12 @@ export interface ChildProcessResult { stderr: string signal?: string } - +export const oneMB = 1024 * 1024 export const eof = Symbol('EOF') +export const defaultProcessWarnThresholds = { + memory: 100 * oneMB, // 100 MB + cpu: 50, +} export interface ProcessStats { memory: number @@ -69,10 +80,6 @@ export interface ProcessStats { } export class ChildProcessTracker { static readonly pollingInterval: number = 10000 // Check usage every 10 seconds - static readonly thresholds: ProcessStats = { - memory: 100 * 1024 * 1024, // 100 MB - cpu: 50, - } static readonly logger = logger.getLogger('childProcess') static readonly loggedPids = new CircularBuffer(1000) #processByPid: Map = new Map() @@ -82,6 +89,15 @@ export class ChildProcessTracker { this.#pids = new PollingSet(ChildProcessTracker.pollingInterval, () => this.monitor()) } + private getThreshold(pid: number) { + if (!this.#processByPid.has(pid)) { + ChildProcessTracker.logOnce(pid, `Missing process with id ${pid}, returning default threshold`) + return defaultProcessWarnThresholds + } + // Safe to assert since it exists from check above. + return this.#processByPid.get(pid)!.getWarnThresholds() + } + private cleanUp() { const terminatedProcesses = Array.from(this.#pids.values()).filter( (pid: number) => this.#processByPid.get(pid)?.stopped @@ -106,13 +122,14 @@ export class ChildProcessTracker { return } const stats = this.getUsage(pid) + const threshold = this.getThreshold(pid) if (stats) { ChildProcessTracker.logger.debug(`Process ${pid} usage: %O`, stats) - if (stats.memory > ChildProcessTracker.thresholds.memory) { - ChildProcessTracker.logOnce(pid, `Process ${pid} exceeded memory threshold: ${stats.memory}`) + if (stats.memory > threshold.memory) { + ChildProcessTracker.logOnce(pid, `Process ${pid} exceeded memory threshold: ${stats.memory / oneMB} MB`) } - if (stats.cpu > ChildProcessTracker.thresholds.cpu) { - ChildProcessTracker.logOnce(pid, `Process ${pid} exceeded cpu threshold: ${stats.cpu}`) + if (stats.cpu > threshold.cpu) { + ChildProcessTracker.logOnce(pid, `Process ${pid} exceeded cpu threshold: %${stats.cpu}`) } } } @@ -248,6 +265,10 @@ export class ChildProcess { return await new ChildProcess(command, args, options).run() } + public getWarnThresholds(): ProcessStats { + return { ...defaultProcessWarnThresholds, ...this.#baseOptions.warnThresholds } + } + // Inspired by 'got' /** * Creates a one-off {@link ChildProcess} class that always uses the specified options. diff --git a/packages/core/src/test/shared/utilities/processUtils.test.ts b/packages/core/src/test/shared/utilities/processUtils.test.ts index 436ac48ecc4..ddcc2cf9819 100644 --- a/packages/core/src/test/shared/utilities/processUtils.test.ts +++ b/packages/core/src/test/shared/utilities/processUtils.test.ts @@ -10,8 +10,10 @@ import * as sinon from 'sinon' import { makeTemporaryToolkitFolder, tryRemoveFolder } from '../../../shared/filesystemUtilities' import { ChildProcess, + ChildProcessOptions, ChildProcessResult, ChildProcessTracker, + defaultProcessWarnThresholds, eof, ProcessStats, } from '../../../shared/utilities/processUtils' @@ -376,8 +378,8 @@ async function stopAndWait(runningProcess: RunningProcess): Promise { await runningProcess.result } -function startSleepProcess(timeout: number = 90): RunningProcess { - const childProcess = new ChildProcess(getSleepCmd(), [timeout.toString()]) +function startSleepProcess(options?: ChildProcessOptions, timeout: number = 90): RunningProcess { + const childProcess = new ChildProcess(getSleepCmd(), [timeout.toString()], options) const result = childProcess.run().catch(() => assert.fail('sleep command threw an error')) return { childProcess, result } } @@ -454,12 +456,12 @@ describe('ChildProcessTracker', function () { tracker.add(runningProcess.childProcess) const highCpu: ProcessStats = { - cpu: ChildProcessTracker.thresholds.cpu + 1, + cpu: defaultProcessWarnThresholds.cpu + 1, memory: 0, } const highMemory: ProcessStats = { cpu: 0, - memory: ChildProcessTracker.thresholds.memory + 1, + memory: defaultProcessWarnThresholds.memory + 1, } usageMock.returns(highCpu) @@ -480,7 +482,7 @@ describe('ChildProcessTracker', function () { tracker.add(runningProcess.childProcess) usageMock.returns({ - cpu: ChildProcessTracker.thresholds.cpu + 1, + cpu: defaultProcessWarnThresholds.cpu + 1, memory: 0, }) @@ -494,8 +496,8 @@ describe('ChildProcessTracker', function () { const runningProcess = startSleepProcess() usageMock.returns({ - cpu: ChildProcessTracker.thresholds.cpu - 1, - memory: ChildProcessTracker.thresholds.memory - 1, + cpu: defaultProcessWarnThresholds.cpu - 1, + memory: defaultProcessWarnThresholds.memory - 1, }) await clock.tickAsync(ChildProcessTracker.pollingInterval) @@ -504,4 +506,40 @@ describe('ChildProcessTracker', function () { await stopAndWait(runningProcess) }) + + it('respects custom thresholds', async function () { + const runningProcess = startSleepProcess({ + warnThresholds: { + cpu: defaultProcessWarnThresholds.cpu + 10, + memory: defaultProcessWarnThresholds.memory + 10, + }, + }) + + usageMock.returns({ + cpu: defaultProcessWarnThresholds.cpu + 5, + memory: defaultProcessWarnThresholds.memory + 5, + }) + + await clock.tickAsync(ChildProcessTracker.pollingInterval) + assert.throws(() => assertLogsContain(runningProcess.childProcess.pid().toString(), false, 'warn')) + + await stopAndWait(runningProcess) + }) + + it('fills custom thresholds with default', async function () { + const runningProcess = startSleepProcess({ + warnThresholds: { + cpu: defaultProcessWarnThresholds.cpu + 10, + }, + }) + + usageMock.returns({ + memory: defaultProcessWarnThresholds.memory + 1, + }) + + await clock.tickAsync(ChildProcessTracker.pollingInterval) + assertLogsContain(runningProcess.childProcess.pid().toString(), false, 'warn') + + await stopAndWait(runningProcess) + }) }) From 6046dcb71603ffefbfde96866c7a35e95caf9bdd Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 29 Apr 2025 21:54:50 -0400 Subject: [PATCH 2/6] refactor: reuse constant from processUtils --- packages/amazonq/src/lsp/client.ts | 3 ++- packages/core/src/amazonq/lsp/lspClient.ts | 3 ++- packages/core/src/shared/utilities/processUtils.ts | 9 ++++++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/amazonq/src/lsp/client.ts b/packages/amazonq/src/lsp/client.ts index 39fcb6f010d..8af43c40d58 100644 --- a/packages/amazonq/src/lsp/client.ts +++ b/packages/amazonq/src/lsp/client.ts @@ -33,6 +33,7 @@ import { undefinedIfEmpty, getOptOutPreference, } from 'aws-core-vscode/shared' +import { processUtils } from 'aws-core-vscode/shared' import { activate } from './chat/activation' import { AmazonQResourcePaths } from './lspInstaller' import { ConfigSection, isValidConfigSection, toAmazonQLSPLogLevel } from './config' @@ -55,7 +56,7 @@ export async function startLanguageServer( '--pre-init-encryption', '--set-credentials-encryption-key', ] - const memoryWarnThreshold = 200 * 1024 * 1024 // 200 MB + const memoryWarnThreshold = 200 * processUtils.oneMB // 200 MB const serverOptions = createServerOptions({ encryptionKey, executable: resourcePaths.node, diff --git a/packages/core/src/amazonq/lsp/lspClient.ts b/packages/core/src/amazonq/lsp/lspClient.ts index 0b1eb608d95..f3d1e9c2cd7 100644 --- a/packages/core/src/amazonq/lsp/lspClient.ts +++ b/packages/core/src/amazonq/lsp/lspClient.ts @@ -8,6 +8,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. */ import * as vscode from 'vscode' +import { oneMB } from '../../shared/utilities/processUtils' import * as path from 'path' import * as nls from 'vscode-nls' import * as crypto from 'crypto' @@ -252,7 +253,7 @@ export async function activate(extensionContext: ExtensionContext, resourcePaths } const serverModule = resourcePaths.lsp - const memoryWarnThreshold = 600 * 1024 * 1024 // 600 MB + const memoryWarnThreshold = 600 * oneMB // 600 MB const serverOptions = createServerOptions({ encryptionKey: key, diff --git a/packages/core/src/shared/utilities/processUtils.ts b/packages/core/src/shared/utilities/processUtils.ts index fa61de70fe8..51a193e700c 100644 --- a/packages/core/src/shared/utilities/processUtils.ts +++ b/packages/core/src/shared/utilities/processUtils.ts @@ -89,7 +89,7 @@ export class ChildProcessTracker { this.#pids = new PollingSet(ChildProcessTracker.pollingInterval, () => this.monitor()) } - private getThreshold(pid: number) { + private getThreshold(pid: number): ProcessStats { if (!this.#processByPid.has(pid)) { ChildProcessTracker.logOnce(pid, `Missing process with id ${pid}, returning default threshold`) return defaultProcessWarnThresholds @@ -126,10 +126,13 @@ export class ChildProcessTracker { if (stats) { ChildProcessTracker.logger.debug(`Process ${pid} usage: %O`, stats) if (stats.memory > threshold.memory) { - ChildProcessTracker.logOnce(pid, `Process ${pid} exceeded memory threshold: ${stats.memory / oneMB} MB`) + ChildProcessTracker.logOnce( + pid, + `Process ${pid} exceeded memory threshold: ${(stats.memory / oneMB).toFixed(2)} MB` + ) } if (stats.cpu > threshold.cpu) { - ChildProcessTracker.logOnce(pid, `Process ${pid} exceeded cpu threshold: %${stats.cpu}`) + ChildProcessTracker.logOnce(pid, `Process ${pid} exceeded cpu threshold: ${stats.cpu}%`) } } } From e64071bcc7188cc10ef9f5464c7bc076adb98e26 Mon Sep 17 00:00:00 2001 From: hkobew Date: Wed, 30 Apr 2025 10:27:48 -0400 Subject: [PATCH 3/6] fix: tests --- .../test/shared/utilities/processUtils.test.ts | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/core/src/test/shared/utilities/processUtils.test.ts b/packages/core/src/test/shared/utilities/processUtils.test.ts index ddcc2cf9819..65817d80a56 100644 --- a/packages/core/src/test/shared/utilities/processUtils.test.ts +++ b/packages/core/src/test/shared/utilities/processUtils.test.ts @@ -494,6 +494,7 @@ describe('ChildProcessTracker', function () { it('does not log for processes within threshold', async function () { const runningProcess = startSleepProcess() + tracker.add(runningProcess.childProcess) usageMock.returns({ cpu: defaultProcessWarnThresholds.cpu - 1, @@ -508,12 +509,20 @@ describe('ChildProcessTracker', function () { }) it('respects custom thresholds', async function () { - const runningProcess = startSleepProcess({ + const largeRunningProcess = startSleepProcess({ warnThresholds: { cpu: defaultProcessWarnThresholds.cpu + 10, memory: defaultProcessWarnThresholds.memory + 10, }, }) + tracker.add(largeRunningProcess.childProcess) + const smallRunningProcess = startSleepProcess({ + warnThresholds: { + cpu: defaultProcessWarnThresholds.cpu - 10, + memory: defaultProcessWarnThresholds.memory - 10, + }, + }) + tracker.add(smallRunningProcess.childProcess) usageMock.returns({ cpu: defaultProcessWarnThresholds.cpu + 5, @@ -521,9 +530,11 @@ describe('ChildProcessTracker', function () { }) await clock.tickAsync(ChildProcessTracker.pollingInterval) - assert.throws(() => assertLogsContain(runningProcess.childProcess.pid().toString(), false, 'warn')) + assert.throws(() => assertLogsContain(largeRunningProcess.childProcess.pid().toString(), false, 'warn')) + assertLogsContain(smallRunningProcess.childProcess.pid().toString(), false, 'warn') - await stopAndWait(runningProcess) + await stopAndWait(largeRunningProcess) + await stopAndWait(smallRunningProcess) }) it('fills custom thresholds with default', async function () { @@ -532,6 +543,7 @@ describe('ChildProcessTracker', function () { cpu: defaultProcessWarnThresholds.cpu + 10, }, }) + tracker.add(runningProcess.childProcess) usageMock.returns({ memory: defaultProcessWarnThresholds.memory + 1, From cc5c8c8f48532a0510a4501ff1c99515b34840db Mon Sep 17 00:00:00 2001 From: hkobew Date: Wed, 30 Apr 2025 14:11:41 -0400 Subject: [PATCH 4/6] refactor: adjust workspace indexing threshold --- packages/core/src/amazonq/lsp/lspClient.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/amazonq/lsp/lspClient.ts b/packages/core/src/amazonq/lsp/lspClient.ts index f3d1e9c2cd7..665acbdf3a7 100644 --- a/packages/core/src/amazonq/lsp/lspClient.ts +++ b/packages/core/src/amazonq/lsp/lspClient.ts @@ -253,7 +253,7 @@ export async function activate(extensionContext: ExtensionContext, resourcePaths } const serverModule = resourcePaths.lsp - const memoryWarnThreshold = 600 * oneMB // 600 MB + const memoryWarnThreshold = 800 * oneMB // 800 MB const serverOptions = createServerOptions({ encryptionKey: key, From 362785d293f358d6c7783fa8039ced7922831ed2 Mon Sep 17 00:00:00 2001 From: hkobew Date: Fri, 2 May 2025 06:48:53 -0400 Subject: [PATCH 5/6] docs: cleanup comments --- packages/amazonq/src/lsp/client.ts | 2 +- packages/core/src/amazonq/lsp/lspClient.ts | 2 +- packages/core/src/shared/utilities/processUtils.ts | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/amazonq/src/lsp/client.ts b/packages/amazonq/src/lsp/client.ts index e86acc0809c..2ad4905bc83 100644 --- a/packages/amazonq/src/lsp/client.ts +++ b/packages/amazonq/src/lsp/client.ts @@ -81,7 +81,7 @@ export async function startLanguageServer( executable = [resourcePaths.node] } - const memoryWarnThreshold = 200 * processUtils.oneMB // 200 MB + const memoryWarnThreshold = 200 * processUtils.oneMB const serverOptions = createServerOptions({ encryptionKey, executable: executable, diff --git a/packages/core/src/amazonq/lsp/lspClient.ts b/packages/core/src/amazonq/lsp/lspClient.ts index 20fe91418ad..eba89c961c4 100644 --- a/packages/core/src/amazonq/lsp/lspClient.ts +++ b/packages/core/src/amazonq/lsp/lspClient.ts @@ -253,7 +253,7 @@ export async function activate(extensionContext: ExtensionContext, resourcePaths } const serverModule = resourcePaths.lsp - const memoryWarnThreshold = 800 * oneMB // 800 MB + const memoryWarnThreshold = 800 * oneMB const serverOptions = createServerOptions({ encryptionKey: key, diff --git a/packages/core/src/shared/utilities/processUtils.ts b/packages/core/src/shared/utilities/processUtils.ts index 51a193e700c..be44ba89bd2 100644 --- a/packages/core/src/shared/utilities/processUtils.ts +++ b/packages/core/src/shared/utilities/processUtils.ts @@ -48,7 +48,7 @@ export interface ChildProcessOptions { warnThresholds?: { /** Threshold for memory usage in bytes */ memory?: number - /** Threshold for CPU usage in percent */ + /** Threshold for CPU usage by percentage */ cpu?: number } } @@ -70,7 +70,7 @@ export interface ChildProcessResult { export const oneMB = 1024 * 1024 export const eof = Symbol('EOF') export const defaultProcessWarnThresholds = { - memory: 100 * oneMB, // 100 MB + memory: 100 * oneMB, cpu: 50, } From c7e7a27d5657f2f424f09ac086aa9eceb778a4e7 Mon Sep 17 00:00:00 2001 From: hkobew Date: Fri, 2 May 2025 10:48:37 -0400 Subject: [PATCH 6/6] refactor: bump threshold to 1GB --- packages/amazonq/src/lsp/client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/amazonq/src/lsp/client.ts b/packages/amazonq/src/lsp/client.ts index 2ad4905bc83..a8cb8d76a40 100644 --- a/packages/amazonq/src/lsp/client.ts +++ b/packages/amazonq/src/lsp/client.ts @@ -81,7 +81,7 @@ export async function startLanguageServer( executable = [resourcePaths.node] } - const memoryWarnThreshold = 200 * processUtils.oneMB + const memoryWarnThreshold = 1024 * processUtils.oneMB const serverOptions = createServerOptions({ encryptionKey, executable: executable,