diff --git a/packages/core/src/amazonqFeatureDev/util/files.ts b/packages/core/src/amazonqFeatureDev/util/files.ts index cffa74af867..1b83bdbe2b5 100644 --- a/packages/core/src/amazonqFeatureDev/util/files.ts +++ b/packages/core/src/amazonqFeatureDev/util/files.ts @@ -18,6 +18,7 @@ import { AmazonqCreateUpload, Span, telemetry as amznTelemetry } from '../../sha import { TelemetryHelper } from './telemetryHelper' import { maxRepoSizeBytes } from '../constants' import { isCodeFile } from '../../shared/filetypes' +import { fs } from '../../shared' const getSha256 = (file: Buffer) => createHash('sha256').update(file).digest('base64') @@ -28,17 +29,17 @@ export async function prepareRepoData( repoRootPaths: string[], workspaceFolders: CurrentWsFolders, telemetry: TelemetryHelper, - span: Span + span: Span, + zip: AdmZip = new AdmZip() ) { try { const files = await collectFiles(repoRootPaths, workspaceFolders, true, maxRepoSizeBytes) - const zip = new AdmZip() let totalBytes = 0 const ignoredExtensionMap = new Map() for (const file of files) { - const fileSize = (await vscode.workspace.fs.stat(file.fileUri)).size + const fileSize = (await fs.stat(file.fileUri)).size const isCodeFile_ = isCodeFile(file.relativeFilePath) if (fileSize >= maxFileSizeBytes || !isCodeFile_) { diff --git a/packages/core/src/codewhisperer/commands/startSecurityScan.ts b/packages/core/src/codewhisperer/commands/startSecurityScan.ts index 8dee806f9df..58eaee2bf40 100644 --- a/packages/core/src/codewhisperer/commands/startSecurityScan.ts +++ b/packages/core/src/codewhisperer/commands/startSecurityScan.ts @@ -93,7 +93,8 @@ export async function startSecurityScan( editor: vscode.TextEditor | undefined, client: DefaultCodeWhispererClient, context: vscode.ExtensionContext, - scope: CodeWhispererConstants.CodeAnalysisScope + scope: CodeWhispererConstants.CodeAnalysisScope, + zipUtil: ZipUtil = new ZipUtil() ) { const logger = getLoggerForScope(scope) /** @@ -130,7 +131,6 @@ export async function startSecurityScan( * Step 1: Generate zip */ throwIfCancelled(scope, codeScanStartTime) - const zipUtil = new ZipUtil() const zipMetadata = await zipUtil.generateZip(editor?.document.uri, scope) const projectPaths = zipUtil.getProjectPaths() diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts index 4d8a3468d32..5556ae34f48 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts @@ -287,13 +287,15 @@ interface ZipCodeResult { fileSize: number } -export async function zipCode({ dependenciesFolder, humanInTheLoopFlag, modulePath, zipManifest }: IZipCodeParams) { +export async function zipCode( + { dependenciesFolder, humanInTheLoopFlag, modulePath, zipManifest }: IZipCodeParams, + zip: AdmZip = new AdmZip() +) { let tempFilePath = undefined let logFilePath = undefined let dependenciesCopied = false try { throwIfCancelled() - const zip = new AdmZip() // If no modulePath is passed in, we are not uploaded the source folder // NOTE: We only upload dependencies for human in the loop work diff --git a/packages/core/src/shared/performance/performance.ts b/packages/core/src/shared/performance/performance.ts index 54d108ccea4..f5d65b05ab4 100644 --- a/packages/core/src/shared/performance/performance.ts +++ b/packages/core/src/shared/performance/performance.ts @@ -206,3 +206,11 @@ function assertPerformanceMetrics( `Expected total duration for ${name} to be less than ${expectedDuration}. Actual duration was ${foundDuration}` ) } + +export function getEqualOSTestOptions(testOptions: Partial): Partial { + return { + linux: testOptions, + darwin: testOptions, + win32: testOptions, + } +} diff --git a/packages/core/src/shared/utilities/workspaceUtils.ts b/packages/core/src/shared/utilities/workspaceUtils.ts index f7bea42beaa..d8bd9177592 100644 --- a/packages/core/src/shared/utilities/workspaceUtils.ts +++ b/packages/core/src/shared/utilities/workspaceUtils.ts @@ -582,7 +582,7 @@ export async function collectFilesForIndex( continue } - const fileStat = await vscode.workspace.fs.stat(file) + const fileStat = await fs.stat(file) // ignore single file over 10 MB if (fileStat.size > 10 * 1024 * 1024) { continue diff --git a/packages/core/src/testInteg/perf/buildIndex.test.ts b/packages/core/src/testInteg/perf/buildIndex.test.ts index 29f07e28587..3b13746ab68 100644 --- a/packages/core/src/testInteg/perf/buildIndex.test.ts +++ b/packages/core/src/testInteg/perf/buildIndex.test.ts @@ -11,26 +11,36 @@ import { LspClient, LspController } from '../../amazonq' import { LanguageClient, ServerOptions } from 'vscode-languageclient' import { createTestWorkspace } from '../../test/testUtil' import { GetUsageRequestType, IndexRequestType } from '../../amazonq/lsp/types' -import { getRandomString } from '../../shared' +import { fs, getRandomString } from '../../shared' +import { FileSystem } from '../../shared/fs/fs' +import { getFsCallsUpperBound } from './utilities' interface SetupResult { clientReqStub: sinon.SinonStub + fsSpy: sinon.SinonSpiedInstance + findFilesSpy: sinon.SinonSpy } async function verifyResult(setup: SetupResult) { assert.ok(setup.clientReqStub.calledTwice) assert.ok(setup.clientReqStub.firstCall.calledWith(IndexRequestType)) assert.ok(setup.clientReqStub.secondCall.calledWith(GetUsageRequestType)) + + assert.strictEqual(getFsCallsUpperBound(setup.fsSpy), 0, 'should not make any fs calls') + assert.ok(setup.findFilesSpy.callCount <= 2, 'findFiles should not be called more than twice') } async function setupWithWorkspace(numFiles: number, options: { fileContent: string }): Promise { // Force VSCode to find my test workspace only to keep test contained and controlled. const testWorksapce = await createTestWorkspace(numFiles, options) sinon.stub(vscode.workspace, 'workspaceFolders').value([testWorksapce]) + // Avoid sending real request to lsp. const clientReqStub = sinon.stub(LanguageClient.prototype, 'sendRequest').resolves(true) + const fsSpy = sinon.spy(fs) + const findFilesSpy = sinon.spy(vscode.workspace, 'findFiles') LspClient.instance.client = new LanguageClient('amazonq', 'test-client', {} as ServerOptions, {}) - return { clientReqStub } + return { clientReqStub, fsSpy, findFilesSpy } } describe('buildIndex', function () { diff --git a/packages/core/src/testInteg/perf/collectFiles.test.ts b/packages/core/src/testInteg/perf/collectFiles.test.ts index 3e9914898f5..d2bc705f29a 100644 --- a/packages/core/src/testInteg/perf/collectFiles.test.ts +++ b/packages/core/src/testInteg/perf/collectFiles.test.ts @@ -5,58 +5,80 @@ import assert from 'assert' import * as vscode from 'vscode' import * as sinon from 'sinon' -import { performanceTest } from '../../shared/performance/performance' +import { getEqualOSTestOptions, performanceTest } from '../../shared/performance/performance' import { createTestWorkspaceFolder, toFile } from '../../test/testUtil' import path from 'path' -import { randomUUID } from '../../shared' +import { fs, randomUUID } from '../../shared' import { collectFiles } from '../../shared/utilities/workspaceUtils' +import { getFsCallsUpperBound } from './utilities' +import { FileSystem } from '../../shared/fs/fs' -performanceTest( - // collecting all files in the workspace and zipping them is pretty resource intensive - { - linux: { - userCpuUsage: 85, +function performanceTestWrapper(totalFiles: number) { + return performanceTest( + getEqualOSTestOptions({ + userCpuUsage: 100, + systemCpuUsage: 35, heapTotal: 2, - duration: 0.8, - }, - }, - 'calculate cpu and memory usage', - function () { - const totalFiles = 100 - return { - setup: async () => { - const workspace = await createTestWorkspaceFolder() + }), + 'calculate cpu and memory usage', + function () { + return { + setup: async () => { + const workspace = await createTestWorkspaceFolder() - sinon.stub(vscode.workspace, 'workspaceFolders').value([workspace]) + sinon.stub(vscode.workspace, 'workspaceFolders').value([workspace]) + const fsSpy = sinon.spy(fs) + const findFilesSpy = sinon.spy(vscode.workspace, 'findFiles') + const fileContent = randomUUID() + for (let x = 0; x < totalFiles; x++) { + await toFile(fileContent, path.join(workspace.uri.fsPath, `file.${x}`)) + } - const fileContent = randomUUID() - for (let x = 0; x < totalFiles; x++) { - await toFile(fileContent, path.join(workspace.uri.fsPath, `file.${x}`)) - } + return { + workspace, + fsSpy, + findFilesSpy, + } + }, + execute: async ({ workspace }: { workspace: vscode.WorkspaceFolder }) => { + return { + result: await collectFiles([workspace.uri.fsPath], [workspace], true), + } + }, + verify: ( + setup: { + workspace: vscode.WorkspaceFolder + fsSpy: sinon.SinonSpiedInstance + findFilesSpy: sinon.SinonSpy + }, + { result }: { result: Awaited> } + ) => { + assert.deepStrictEqual(result.length, totalFiles) + const sortedFiles = [...result].sort((a, b) => { + const numA = parseInt(a.relativeFilePath.split('.')[1]) + const numB = parseInt(b.relativeFilePath.split('.')[1]) + return numA - numB + }) + for (let x = 0; x < totalFiles; x++) { + assert.deepStrictEqual(sortedFiles[x].relativeFilePath, `file.${x}`) + } - return { - workspace, - } - }, - execute: async ({ workspace }: { workspace: vscode.WorkspaceFolder }) => { - return { - result: await collectFiles([workspace.uri.fsPath], [workspace], true), - } - }, - verify: ( - _: { workspace: vscode.WorkspaceFolder }, - { result }: { result: Awaited> } - ) => { - assert.deepStrictEqual(result.length, totalFiles) - const sortedFiles = [...result].sort((a, b) => { - const numA = parseInt(a.relativeFilePath.split('.')[1]) - const numB = parseInt(b.relativeFilePath.split('.')[1]) - return numA - numB - }) - for (let x = 0; x < totalFiles; x++) { - assert.deepStrictEqual(sortedFiles[x].relativeFilePath, `file.${x}`) - } - }, + assert.ok( + getFsCallsUpperBound(setup.fsSpy) <= totalFiles * 5, + 'total system calls below 5 per file' + ) + assert.ok(setup.findFilesSpy.callCount <= 2, 'findFiles not called more than twice') + }, + } } - } -) + ) +} + +describe('collectFiles', function () { + afterEach(function () { + sinon.restore() + }) + performanceTestWrapper(10) + performanceTestWrapper(100) + performanceTestWrapper(250) +}) diff --git a/packages/core/src/testInteg/perf/downloadExportResultArchive.test.ts b/packages/core/src/testInteg/perf/downloadExportResultArchive.test.ts index 4f78fdc4b0e..feff513498a 100644 --- a/packages/core/src/testInteg/perf/downloadExportResultArchive.test.ts +++ b/packages/core/src/testInteg/perf/downloadExportResultArchive.test.ts @@ -9,7 +9,7 @@ import * as sinon from 'sinon' import path from 'path' import { fs, getRandomString } from '../../shared' import { createTestWorkspace } from '../../test/testUtil' -import { performanceTest } from '../../shared/performance/performance' +import { getEqualOSTestOptions, performanceTest } from '../../shared/performance/performance' import { downloadExportResultArchive } from '../../shared/utilities/download' interface SetupResult { @@ -47,24 +47,11 @@ async function setup(pieces: number, pieceSize: number): Promise { function perfTest(pieces: number, pieceSize: number, label: string) { return performanceTest( - { - testRuns: 10, - linux: { - userCpuUsage: 200, - systemCpuUsage: 35, - heapTotal: 4, - }, - darwin: { - userCpuUsage: 200, - systemCpuUsage: 35, - heapTotal: 4, - }, - win32: { - userCpuUsage: 200, - systemCpuUsage: 35, - heapTotal: 4, - }, - }, + getEqualOSTestOptions({ + userCpuUsage: 200, + systemCpuUsage: 35, + heapTotal: 4, + }), label, function () { return { diff --git a/packages/core/src/testInteg/perf/getFileSha384.test.ts b/packages/core/src/testInteg/perf/getFileSha384.test.ts index 5072f448244..c7768a3cdd1 100644 --- a/packages/core/src/testInteg/perf/getFileSha384.test.ts +++ b/packages/core/src/testInteg/perf/getFileSha384.test.ts @@ -4,31 +4,26 @@ */ import assert from 'assert' import path from 'path' +import sinon from 'sinon' import { getTestWorkspaceFolder } from '../integrationTestsUtilities' import { fs, getRandomString } from '../../shared' import { LspController } from '../../amazonq' -import { performanceTest } from '../../shared/performance/performance' +import { getEqualOSTestOptions, performanceTest } from '../../shared/performance/performance' +import { FileSystem } from '../../shared/fs/fs' +import { getFsCallsUpperBound } from './utilities' + +interface SetupResult { + testFile: string + fsSpy: sinon.SinonSpiedInstance +} function performanceTestWrapper(label: string, fileSize: number) { return performanceTest( - { - testRuns: 1, - linux: { - userCpuUsage: 400, - systemCpuUsage: 35, - heapTotal: 4, - }, - darwin: { - userCpuUsage: 400, - systemCpuUsage: 35, - heapTotal: 4, - }, - win32: { - userCpuUsage: 400, - systemCpuUsage: 35, - heapTotal: 4, - }, - }, + getEqualOSTestOptions({ + userCpuUsage: 400, + systemCpuUsage: 35, + heapTotal: 4, + }), label, function () { return { @@ -37,14 +32,15 @@ function performanceTestWrapper(label: string, fileSize: number) { const fileContent = getRandomString(fileSize) const testFile = path.join(workspace, 'test-file') await fs.writeFile(testFile, fileContent) - - return testFile + const fsSpy = sinon.spy(fs) + return { testFile, fsSpy } }, - execute: async (testFile: string) => { - return await LspController.instance.getFileSha384(testFile) + execute: async (setup: SetupResult) => { + return await LspController.instance.getFileSha384(setup.testFile) }, - verify: async (_testFile: string, result: string) => { + verify: async (setup: SetupResult, result: string) => { assert.strictEqual(result.length, 96) + assert.ok(getFsCallsUpperBound(setup.fsSpy) <= 1, 'makes a single call to fs') }, } } @@ -53,6 +49,9 @@ function performanceTestWrapper(label: string, fileSize: number) { describe('getFileSha384', function () { describe('performance tests', function () { + afterEach(function () { + sinon.restore() + }) performanceTestWrapper('1MB', 1000) performanceTestWrapper('2MB', 2000) performanceTestWrapper('4MB', 4000) diff --git a/packages/core/src/testInteg/perf/prepareRepoData.test.ts b/packages/core/src/testInteg/perf/prepareRepoData.test.ts index 3e10a1bec42..cf5d2bcbe37 100644 --- a/packages/core/src/testInteg/perf/prepareRepoData.test.ts +++ b/packages/core/src/testInteg/perf/prepareRepoData.test.ts @@ -3,72 +3,78 @@ * SPDX-License-Identifier: Apache-2.0 */ import assert from 'assert' +import * as sinon from 'sinon' import { WorkspaceFolder } from 'vscode' -import { performanceTest } from '../../shared/performance/performance' +import { getEqualOSTestOptions, performanceTest } from '../../shared/performance/performance' import { createTestWorkspace } from '../../test/testUtil' import { prepareRepoData, TelemetryHelper } from '../../amazonqFeatureDev' -import { AmazonqCreateUpload, getRandomString } from '../../shared' +import { AmazonqCreateUpload, fs, getRandomString } from '../../shared' import { Span } from '../../shared/telemetry' +import { FileSystem } from '../../shared/fs/fs' +import { getFsCallsUpperBound } from './utilities' type resultType = { zipFileBuffer: Buffer zipFileChecksum: string } +type setupResult = { + workspace: WorkspaceFolder + fsSpy: sinon.SinonSpiedInstance + numFiles: number + fileSize: number +} + function performanceTestWrapper(numFiles: number, fileSize: number) { return performanceTest( - { - testRuns: 10, - linux: { - userCpuUsage: 100, - systemCpuUsage: 35, - heapTotal: 4, - }, - darwin: { - userCpuUsage: 100, - systemCpuUsage: 35, - heapTotal: 4, - }, - win32: { - userCpuUsage: 100, - systemCpuUsage: 35, - heapTotal: 4, - }, - }, + getEqualOSTestOptions({ + userCpuUsage: 150, + systemCpuUsage: 35, + heapTotal: 4, + }), `handles ${numFiles} files of size ${fileSize} bytes`, function () { const telemetry = new TelemetryHelper() return { setup: async () => { - return await createTestWorkspace(numFiles, { + const fsSpy = sinon.spy(fs) + const workspace = await createTestWorkspace(numFiles, { fileNamePrefix: 'file', fileContent: getRandomString(fileSize), fileNameSuffix: '.md', }) + return { workspace, fsSpy, numFiles, fileSize } }, - execute: async (workspace: WorkspaceFolder) => { - return await prepareRepoData([workspace.uri.fsPath], [workspace], telemetry, { + execute: async (setup: setupResult) => { + return await prepareRepoData([setup.workspace.uri.fsPath], [setup.workspace], telemetry, { record: () => {}, } as unknown as Span) }, - verify: async (_w: WorkspaceFolder, result: resultType) => { - verifyResult(result, telemetry, numFiles * fileSize) + verify: async (setup: setupResult, result: resultType) => { + verifyResult(setup, result, telemetry, numFiles * fileSize) }, } } ) } -function verifyResult(result: resultType, telemetry: TelemetryHelper, expectedSize: number): void { +function verifyResult(setup: setupResult, result: resultType, telemetry: TelemetryHelper, expectedSize: number): void { assert.ok(result) assert.strictEqual(Buffer.isBuffer(result.zipFileBuffer), true) assert.strictEqual(telemetry.repositorySize, expectedSize) assert.strictEqual(result.zipFileChecksum.length, 44) + + assert.ok(getFsCallsUpperBound(setup.fsSpy) <= setup.numFiles * 4, 'total system calls should be under 4 per file') } describe('prepareRepoData', function () { describe('Performance Tests', function () { - performanceTestWrapper(250, 10) + afterEach(function () { + sinon.restore() + }) performanceTestWrapper(10, 1000) + performanceTestWrapper(50, 500) + performanceTestWrapper(100, 100) + performanceTestWrapper(250, 10) }) }) diff --git a/packages/core/src/testInteg/perf/registerNewFiles.test.ts b/packages/core/src/testInteg/perf/registerNewFiles.test.ts index d658bf39ded..c718bf46d26 100644 --- a/packages/core/src/testInteg/perf/registerNewFiles.test.ts +++ b/packages/core/src/testInteg/perf/registerNewFiles.test.ts @@ -3,15 +3,18 @@ * SPDX-License-Identifier: Apache-2.0 */ import assert from 'assert' +import sinon from 'sinon' import * as vscode from 'vscode' import { NewFileInfo, NewFileZipContents, registerNewFiles } from '../../amazonqFeatureDev' -import { performanceTest } from '../../shared/performance/performance' +import { getEqualOSTestOptions, performanceTest } from '../../shared/performance/performance' import { getTestWorkspaceFolder } from '../integrationTestsUtilities' import { VirtualFileSystem } from '../../shared' interface SetupResult { workspace: vscode.WorkspaceFolder fileContents: NewFileZipContents[] + vfsSpy: sinon.SinonSpiedInstance + vfs: VirtualFileSystem } function getFileContents(numFiles: number, fileSize: number): NewFileZipContents[] { @@ -26,30 +29,20 @@ function getFileContents(numFiles: number, fileSize: number): NewFileZipContents function performanceTestWrapper(label: string, numFiles: number, fileSize: number) { const conversationId = 'test-conversation' return performanceTest( - { - testRuns: 10, - linux: { - userCpuUsage: 200, - systemCpuUsage: 35, - heapTotal: 8, - }, - darwin: { - userCpuUsage: 200, - systemCpuUsage: 35, - heapTotal: 8, - }, - win32: { - userCpuUsage: 200, - systemCpuUsage: 35, - heapTotal: 8, - }, - }, + getEqualOSTestOptions({ + userCpuUsage: 200, + systemCpuUsage: 35, + heapTotal: 20, + }), label, function () { return { setup: async () => { const testWorkspaceUri = vscode.Uri.file(getTestWorkspaceFolder()) const fileContents = getFileContents(numFiles, fileSize) + const vfs = new VirtualFileSystem() + const vfsSpy = sinon.spy(vfs) + return { workspace: { uri: testWorkspaceUri, @@ -57,19 +50,25 @@ function performanceTestWrapper(label: string, numFiles: number, fileSize: numbe index: 0, }, fileContents: fileContents, + vfsSpy: vfsSpy, + vfs: vfs, } }, execute: async (setup: SetupResult) => { return registerNewFiles( - new VirtualFileSystem(), + setup.vfs, setup.fileContents, 'test-upload-id', [setup.workspace], conversationId ) }, - verify: async (_setup: SetupResult, result: NewFileInfo[]) => { + verify: async (setup: SetupResult, result: NewFileInfo[]) => { assert.strictEqual(result.length, numFiles) + assert.ok( + setup.vfsSpy.registerProvider.callCount <= numFiles, + 'only register each file once in vfs' + ) }, } } diff --git a/packages/core/src/testInteg/perf/startSecurityScan.test.ts b/packages/core/src/testInteg/perf/startSecurityScan.test.ts index 79209322d31..ff881bf7f61 100644 --- a/packages/core/src/testInteg/perf/startSecurityScan.test.ts +++ b/packages/core/src/testInteg/perf/startSecurityScan.test.ts @@ -21,9 +21,20 @@ import { } from '../../test/testUtil' import { getTestWindow } from '../../test/shared/vscode/window' import { SeverityLevel } from '../../test/shared/vscode/message' -import { CodeAnalysisScope } from '../../codewhisperer' -import { performanceTest } from '../../shared/performance/performance' +import { CodeAnalysisScope, ZipUtil } from '../../codewhisperer' +import { getEqualOSTestOptions, performanceTest } from '../../shared/performance/performance' import { createClient } from '../../test/codewhisperer/testUtil' +import { fs } from '../../shared' +import { FileSystem } from '../../shared/fs/fs' +import { getFsCallsUpperBound } from './utilities' + +interface SetupResult { + commandSpy: sinon.SinonSpy + securityScanRenderSpy: sinon.SinonSpy + zipUtil: ZipUtil + zipSpy: sinon.SinonSpiedInstance + fsSpy: sinon.SinonSpiedInstance +} describe('startSecurityScanPerformanceTest', function () { let extensionContext: FakeExtensionContext @@ -57,40 +68,53 @@ describe('startSecurityScanPerformanceTest', function () { }) } - performanceTest({}, 'Should calculate cpu and memory usage for file scans', function () { - return { - setup: async () => { - getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - const commandSpy = sinon.spy(vscode.commands, 'executeCommand') - const securityScanRenderSpy = sinon.spy(diagnosticsProvider, 'initSecurityScanRender') - await model.CodeScansState.instance.setScansEnabled(true) - return { commandSpy, securityScanRenderSpy } - }, - execute: async () => { - await startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - createClient(), - extensionContext, - CodeAnalysisScope.FILE - ) - }, - verify: ({ - commandSpy, - securityScanRenderSpy, - }: { - commandSpy: sinon.SinonSpy - securityScanRenderSpy: sinon.SinonSpy - }) => { - assert.ok(commandSpy.neverCalledWith('workbench.action.problems.focus')) - assert.ok(securityScanRenderSpy.calledOnce) - const warnings = getTestWindow().shownMessages.filter((m) => m.severity === SeverityLevel.Warning) - assert.strictEqual(warnings.length, 0) - assertTelemetry('codewhisperer_securityScan', { - codewhispererCodeScanScope: 'FILE', - passive: true, - }) - }, + performanceTest( + getEqualOSTestOptions({ + userCpuUsage: 150, + systemCpuUsage: 35, + heapTotal: 2, + }), + 'Should calculate cpu and memory usage for file scans', + function () { + return { + setup: async () => { + getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) + const commandSpy = sinon.spy(vscode.commands, 'executeCommand') + const securityScanRenderSpy = sinon.spy(diagnosticsProvider, 'initSecurityScanRender') + const zipUtil = new ZipUtil() + const zipSpy = sinon.spy(zipUtil) + const fsSpy = sinon.spy(fs) + await model.CodeScansState.instance.setScansEnabled(true) + return { commandSpy, securityScanRenderSpy, zipUtil, zipSpy, fsSpy } + }, + execute: async (setup: SetupResult) => { + await startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + createClient(), + extensionContext, + CodeAnalysisScope.FILE, + setup.zipUtil + ) + }, + verify: (setup: SetupResult) => { + assert.ok(setup.commandSpy.neverCalledWith('workbench.action.problems.focus')) + assert.ok(setup.securityScanRenderSpy.calledOnce) + + assert.ok(setup.zipSpy.generateZip.calledOnce) + assert.ok(setup.zipSpy.removeTmpFiles.calledOnce) + assert.ok( + getFsCallsUpperBound(setup.fsSpy) <= 5, + 'should make less than a small constant number of file system calls' + ) + const warnings = getTestWindow().shownMessages.filter((m) => m.severity === SeverityLevel.Warning) + assert.strictEqual(warnings.length, 0) + assertTelemetry('codewhisperer_securityScan', { + codewhispererCodeScanScope: 'FILE', + passive: true, + }) + }, + } } - }) + ) }) diff --git a/packages/core/src/testInteg/perf/tryInstallLsp.test.ts b/packages/core/src/testInteg/perf/tryInstallLsp.test.ts index 7cc9c18b2c4..d84da8626d3 100644 --- a/packages/core/src/testInteg/perf/tryInstallLsp.test.ts +++ b/packages/core/src/testInteg/perf/tryInstallLsp.test.ts @@ -10,7 +10,9 @@ import path from 'path' import { LspController } from '../../amazonq' import { fs, getRandomString, globals } from '../../shared' import { createTestWorkspace } from '../../test/testUtil' -import { performanceTest } from '../../shared/performance/performance' +import { getEqualOSTestOptions, performanceTest } from '../../shared/performance/performance' +import { getFsCallsUpperBound } from './utilities' +import { FileSystem } from '../../shared/fs/fs' // fakeFileContent is matched to fakeQServerContent based on hash. const fakeHash = '4eb2865c8f40a322aa04e17d8d83bdaa605d6f1cb363af615240a5442a010e0aef66e21bcf4c88f20fabff06efe8a214' @@ -31,7 +33,7 @@ const fakeNodeContent = { serverVersion: '1.1.1', } -function createStubs(numberOfFiles: number, fileSize: number) { +function createStubs(numberOfFiles: number, fileSize: number): sinon.SinonSpiedInstance { // Avoid making HTTP request or mocking giant manifest, stub what we need directly from request. sinon.stub(LspController.prototype, 'fetchManifest') // Directly feed the runtime specifications. @@ -41,8 +43,11 @@ function createStubs(numberOfFiles: number, fileSize: number) { sinon.stub(LspController.prototype, '_download').callsFake(getFakeDownload(numberOfFiles, fileSize)) // Hard code the hash since we are creating files on the spot, whose hashes can't be predicted. sinon.stub(LspController.prototype, 'getFileSha384').resolves(fakeHash) - // Don't allow tryInstallLsp to move runtimes out of temporary folder. - sinon.stub(fs, 'rename') + const fsSpy = sinon.spy(fs) + fsSpy.rename.restore() + // Don't allow tryInstallLsp to move runtimes out of temporary folder + sinon.stub(fsSpy, 'rename') + return fsSpy } /** @@ -66,40 +71,26 @@ const getFakeDownload = function (numberOfFiles: number, fileSize: number) { } } -function performanceTestWrapper(numFiles: number, fileSize: number) { +function performanceTestWrapper(numFiles: number, fileSize: number, message: string) { return performanceTest( - { - testRuns: 10, - linux: { - userCpuUsage: 100, - systemCpuUsage: 35, - heapTotal: 6, - duration: 15, - }, - darwin: { - userCpuUsage: 100, - systemCpuUsage: 35, - heapTotal: 6, - duration: 15, - }, - win32: { - userCpuUsage: 100, - systemCpuUsage: 35, - heapTotal: 6, - duration: 15, - }, - }, - 'many small files in zip', + getEqualOSTestOptions({ + userCpuUsage: 150, + systemCpuUsage: 35, + heapTotal: 6, + duration: 15, + }), + message, function () { return { setup: async () => { - createStubs(numFiles, fileSize) + return createStubs(numFiles, fileSize) }, execute: async () => { return await LspController.instance.tryInstallLsp(globals.context) }, - verify: async (_setup: any, result: boolean) => { + verify: async (fsSpy: sinon.SinonSpiedInstance, result: boolean) => { assert.ok(result) + assert.ok(getFsCallsUpperBound(fsSpy) <= 6 * numFiles) }, } } @@ -111,7 +102,7 @@ describe('tryInstallLsp', function () { sinon.restore() }) describe('performance tests', function () { - performanceTestWrapper(250, 10) - performanceTestWrapper(10, 1000) + performanceTestWrapper(250, 10, '250x10') + performanceTestWrapper(10, 1000, '10x1000') }) }) diff --git a/packages/core/src/testInteg/perf/utilities.ts b/packages/core/src/testInteg/perf/utilities.ts new file mode 100644 index 00000000000..e35fb148e95 --- /dev/null +++ b/packages/core/src/testInteg/perf/utilities.ts @@ -0,0 +1,49 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +import * as sinon from 'sinon' +import { FileSystem } from '../../shared/fs/fs' + +/** + * Provide an upper bound on total number of system calls done through our fs module. + * @param fsSpy filesystem spy (Ex. `sinon.spy(fs)`) + * @returns count of operations + */ +export function getFsCallsUpperBound(fsSpy: sinon.SinonSpiedInstance): number { + return getFsReadsUpperBound(fsSpy) + getFsWritesUpperBound(fsSpy) +} + +/** + * Provide an upper bound on the number of filesystem reads done through our fs module. + * This is an upper bound because some of the functions call eachother. + * @param fsSpy filesystem spy (Ex. `sinon.spy(fs)`) + * @returns value + */ +export function getFsReadsUpperBound(fsSpy: sinon.SinonSpiedInstance): number { + return ( + fsSpy.readFileBytes.callCount + + fsSpy.exists.callCount + + fsSpy.exists.callCount + + fsSpy.readdir.callCount + + fsSpy.copy.callCount + + fsSpy.checkPerms.callCount + + fsSpy.tryGetFilepathEnvVar.callCount + ) +} +/** + * Provide an upper bound on the number of filesystem writes done through our fs module. + * This is an upper bound because some of the functions call eachother. + * @param fsSpy filesystem spy (Ex. `sinon.spy(fs)`) + * @returns value + */ +export function getFsWritesUpperBound(fsSpy: sinon.SinonSpiedInstance): number { + return ( + fsSpy.writeFile.callCount + + fsSpy.mkdir.callCount + + fsSpy.rename.callCount + + fsSpy.chmod.callCount + + fsSpy.delete.callCount + + fsSpy.copy.callCount + ) +} diff --git a/packages/core/src/testInteg/perf/zipcode.test.ts b/packages/core/src/testInteg/perf/zipcode.test.ts index 1883861f222..316e53f446d 100644 --- a/packages/core/src/testInteg/perf/zipcode.test.ts +++ b/packages/core/src/testInteg/perf/zipcode.test.ts @@ -8,14 +8,16 @@ import { TransformByQState, ZipManifest } from '../../codewhisperer' import { fs, getRandomString, globals } from '../../shared' import { createTestWorkspace } from '../../test/testUtil' import * as CodeWhispererConstants from '../../codewhisperer/models/constants' -import { performanceTest } from '../../shared/performance/performance' +import { getEqualOSTestOptions, performanceTest } from '../../shared/performance/performance' import { zipCode } from '../../codewhisperer/indexNode' +import { FileSystem } from '../../shared/fs/fs' +import { getFsCallsUpperBound } from './utilities' interface SetupResult { tempDir: string tempFileName: string transformQManifest: ZipManifest - writeSpy: sinon.SinonSpy + fsSpy: sinon.SinonSpiedInstance } async function setup(numberOfFiles: number, fileSize: number): Promise { @@ -28,54 +30,43 @@ async function setup(numberOfFiles: number, fileSize: number): Promise await setup(numberOfFiles, fileSize), - execute: async ({ tempDir, tempFileName, transformQManifest, writeSpy }: SetupResult) => { + execute: async (setup: SetupResult) => { await zipCode({ dependenciesFolder: { - path: tempDir, - name: tempFileName, + path: setup.tempDir, + name: setup.tempFileName, }, humanInTheLoopFlag: false, - modulePath: tempDir, - zipManifest: transformQManifest, + modulePath: setup.tempDir, + zipManifest: setup.transformQManifest, }) }, verify: async (setup: SetupResult) => { assert.ok( - setup.writeSpy.args.find((arg) => { - return arg[0].endsWith('.zip') + setup.fsSpy.writeFile.args.find((arg) => { + return arg[0].toString().endsWith('.zip') }) ) + + assert.ok(getFsCallsUpperBound(setup.fsSpy) <= 15) }, } }