diff --git a/packages/core/src/codewhisperer/commands/startSecurityScan.ts b/packages/core/src/codewhisperer/commands/startSecurityScan.ts index cc9b733f56d..35f7649e16d 100644 --- a/packages/core/src/codewhisperer/commands/startSecurityScan.ts +++ b/packages/core/src/codewhisperer/commands/startSecurityScan.ts @@ -50,6 +50,7 @@ import { import { SecurityIssueTreeViewProvider } from '../service/securityIssueTreeViewProvider' import { ChatSessionManager } from '../../amazonqScan/chat/storages/chatSession' import { TelemetryHelper } from '../util/telemetryHelper' +import { getWorkspacePaths } from '../../shared/utilities/workspaceUtils' const localize = nls.loadMessageBundle() export const stopScanButton = localize('aws.codewhisperer.stopscan', 'Stop Scan') @@ -156,7 +157,7 @@ export async function startSecurityScan( }) } const zipMetadata = await zipUtil.generateZip(editor?.document.uri, scope) - const projectPaths = zipUtil.getProjectPaths() + const projectPaths = getWorkspacePaths() const contextTruncationStartTime = performance.now() codeScanTelemetryEntry.contextTruncationDuration = performance.now() - contextTruncationStartTime diff --git a/packages/core/src/codewhisperer/commands/startTestGeneration.ts b/packages/core/src/codewhisperer/commands/startTestGeneration.ts index 29e7148a17b..4d0443a1ba7 100644 --- a/packages/core/src/codewhisperer/commands/startTestGeneration.ts +++ b/packages/core/src/codewhisperer/commands/startTestGeneration.ts @@ -21,6 +21,7 @@ import { ChildProcess, spawn } from 'child_process' // eslint-disable-line no-re import { BuildStatus } from '../../amazonqTest/chat/session/session' import { fs } from '../../shared/fs/fs' import { Range } from '../client/codewhispereruserclient' +import { getWorkspaceForFile } from '../../shared/utilities/workspaceUtils' // eslint-disable-next-line unicorn/no-null let spawnResult: ChildProcess | null = null @@ -48,7 +49,7 @@ export async function startTestGenerationProcess( const zipUtil = new ZipUtil() if (initialExecution) { - const projectPath = zipUtil.getProjectPath(filePath) ?? '' + const projectPath = getWorkspaceForFile(filePath) ?? '' const relativeTargetPath = path.relative(projectPath, filePath) session.listOfTestGenerationJobId = [] session.shortAnswer = undefined diff --git a/packages/core/src/codewhisperer/models/constants.ts b/packages/core/src/codewhisperer/models/constants.ts index 5762f8609da..cd399546b77 100644 --- a/packages/core/src/codewhisperer/models/constants.ts +++ b/packages/core/src/codewhisperer/models/constants.ts @@ -926,3 +926,7 @@ export const testGenExcludePatterns = [ '**/*.deb', '**/*.model', ] + +export const isFileAnalysisScope = (scope: CodeAnalysisScope) => { + return scope === CodeAnalysisScope.FILE_AUTO || scope === CodeAnalysisScope.FILE_ON_DEMAND +} diff --git a/packages/core/src/codewhisperer/util/zipUtil.ts b/packages/core/src/codewhisperer/util/zipUtil.ts index 32687a6452c..ad927174b5d 100644 --- a/packages/core/src/codewhisperer/util/zipUtil.ts +++ b/packages/core/src/codewhisperer/util/zipUtil.ts @@ -12,7 +12,12 @@ import { fs } from '../../shared/fs/fs' import { getLoggerForScope } from '../service/securityScanHandler' import { runtimeLanguageContext } from './runtimeLanguageContext' import { CodewhispererLanguage } from '../../shared/telemetry/telemetry.gen' -import { CurrentWsFolders, collectFiles, defaultExcludePatterns } from '../../shared/utilities/workspaceUtils' +import { + CurrentWsFolders, + collectFiles, + defaultExcludePatterns, + getWorkspacePaths, +} from '../../shared/utilities/workspaceUtils' import { FileSizeExceededError, NoActiveFileError, @@ -20,11 +25,12 @@ import { ProjectSizeExceededError, } from '../models/errors' import { FeatureUseCase } from '../models/constants' -import { ChildProcess, ChildProcessOptions } from '../../shared/utilities/processUtils' import { ProjectZipError } from '../../amazonqTest/error' -import { removeAnsi } from '../../shared/utilities/textUtilities' import { normalize } from '../../shared/utilities/pathUtils' import { ZipStream } from '../../shared/utilities/zipStream' +import { getTextContent } from '../../shared/utilities/textDocumentUtilities' +import { ChildProcess, ChildProcessOptions } from '../../shared/utilities/processUtils' +import { removeAnsi } from '../../shared/utilities/textUtilities' export interface ZipMetadata { rootDir: string @@ -44,13 +50,6 @@ export const ZipConstants = { codeDiffFilePath: 'codeDiff/code.diff', } -interface GitDiffOptions { - projectPath: string - projectName: string - filePath?: string - scope?: CodeWhispererConstants.CodeAnalysisScope -} - export class ZipUtil { protected _pickedSourceFiles: Set = new Set() protected _pickedBuildFiles: Set = new Set() @@ -72,32 +71,10 @@ export class ZipUtil { return CodeWhispererConstants.projectScanPayloadSizeLimitBytes } - public getProjectPaths() { - const workspaceFolders = vscode.workspace.workspaceFolders - return workspaceFolders?.map((folder) => folder.uri.fsPath) ?? [] - } - - public getProjectPath(filePath: string) { - const fileUri = vscode.Uri.file(filePath) - const workspaceFolder = vscode.workspace.getWorkspaceFolder(fileUri) - return workspaceFolder?.uri.fsPath - } - - protected async getTextContent(uri: vscode.Uri) { - const document = await vscode.workspace.openTextDocument(uri) - const content = document.getText() - return content - } - public reachSizeLimit(size: number, scope: CodeWhispererConstants.CodeAnalysisScope): boolean { - if ( - scope === CodeWhispererConstants.CodeAnalysisScope.FILE_AUTO || - scope === CodeWhispererConstants.CodeAnalysisScope.FILE_ON_DEMAND - ) { - return size > this.getFileScanPayloadSizeLimitInBytes() - } else { - return size > this.getProjectScanPayloadSizeLimitInBytes() - } + return CodeWhispererConstants.isFileAnalysisScope(scope) + ? size > this.getFileScanPayloadSizeLimitInBytes() + : size > this.getProjectScanPayloadSizeLimitInBytes() } public willReachSizeLimit(current: number, adding: number): boolean { @@ -111,7 +88,7 @@ export class ZipUtil { } const zip = new ZipStream() - const content = await this.getTextContent(uri) + const content = await getTextContent(uri) const workspaceFolder = vscode.workspace.getWorkspaceFolder(uri) if (workspaceFolder) { @@ -211,7 +188,7 @@ export class ZipUtil { if (useCase === FeatureUseCase.TEST_GENERATION && projectPath) { projectPaths.push(projectPath) } else { - projectPaths = this.getProjectPaths() + projectPaths = getWorkspacePaths() } if (useCase === FeatureUseCase.CODE_SCAN) { await this.processCombinedGitDiff(zip, projectPaths, '', CodeWhispererConstants.CodeAnalysisScope.PROJECT) @@ -241,166 +218,12 @@ export class ZipUtil { filePath?: string, scope?: CodeWhispererConstants.CodeAnalysisScope ) { - let gitDiffContent = '' - for (const projectPath of projectPaths) { - const projectName = path.basename(projectPath) - // Get diff content - gitDiffContent += await this.executeGitDiff({ - projectPath, - projectName, - filePath, - scope, - }) - } + const gitDiffContent = await getGitDiffContentForProjects(projectPaths, filePath) if (gitDiffContent) { zip.writeString(gitDiffContent, ZipConstants.codeDiffFilePath) } } - private async getGitUntrackedFiles(projectPath: string): Promise { - const checkNewFileArgs = ['ls-files', '--others', '--exclude-standard'] - const checkProcess = new ChildProcess('git', checkNewFileArgs) - - try { - let output = '' - await checkProcess.run({ - rejectOnError: true, - rejectOnErrorCode: true, - onStdout: (text) => { - output += text - }, - spawnOptions: { - cwd: projectPath, - }, - }) - return output - } catch (err) { - getLogger().warn(`Failed to check if file is new: ${err}`) - return undefined - } - } - - private async generateNewFileDiff(projectPath: string, projectName: string, relativePath: string): Promise { - let diffContent = '' - - const gitArgs = [ - 'diff', - '--no-index', - `--src-prefix=a/${projectName}/`, - `--dst-prefix=b/${projectName}/`, - '/dev/null', // Use /dev/null as the old file - relativePath, - ] - - const childProcess = new ChildProcess('git', gitArgs) - const runOptions: ChildProcessOptions = { - rejectOnError: false, - rejectOnErrorCode: false, - onStdout: (text) => { - diffContent += text - getLogger().verbose(removeAnsi(text)) - }, - onStderr: (text) => { - getLogger().error(removeAnsi(text)) - }, - spawnOptions: { - cwd: projectPath, - }, - } - - try { - await childProcess.run(runOptions) - return diffContent - } catch (err) { - getLogger().warn(`Failed to run diff command: ${err}`) - return '' - } - } - - private async generateHeadDiff(projectPath: string, projectName: string, relativePath?: string): Promise { - let diffContent = '' - - const gitArgs = [ - 'diff', - 'HEAD', - `--src-prefix=a/${projectName}/`, - `--dst-prefix=b/${projectName}/`, - ...(relativePath ? [relativePath] : []), - ] - - const childProcess = new ChildProcess('git', gitArgs) - - const runOptions: ChildProcessOptions = { - rejectOnError: true, - rejectOnErrorCode: true, - onStdout: (text) => { - diffContent += text - getLogger().verbose(removeAnsi(text)) - }, - onStderr: (text) => { - getLogger().error(removeAnsi(text)) - }, - spawnOptions: { - cwd: projectPath, - }, - } - - try { - await childProcess.run(runOptions) - return diffContent - } catch (err) { - getLogger().warn(`Failed to run command \`${childProcess.toString()}\`: ${err}`) - return '' - } - } - - private async executeGitDiff(options: GitDiffOptions): Promise { - const { projectPath, projectName, filePath, scope } = options - const isProjectScope = scope === CodeWhispererConstants.CodeAnalysisScope.PROJECT - - const untrackedFilesString = await this.getGitUntrackedFiles(projectPath) - const untrackedFilesArray = untrackedFilesString?.trim()?.split('\n')?.filter(Boolean) - - if (isProjectScope && untrackedFilesArray && !untrackedFilesArray.length) { - return await this.generateHeadDiff(projectPath, projectName) - } - - let diffContent = '' - - if (isProjectScope) { - diffContent = await this.generateHeadDiff(projectPath, projectName) - - if (untrackedFilesArray) { - const untrackedDiffs = await Promise.all( - untrackedFilesArray.map((file) => this.generateNewFileDiff(projectPath, projectName, file)) - ) - diffContent += untrackedDiffs.join('') - } - } else if (!isProjectScope && filePath) { - const relativeFilePath = path.relative(projectPath, filePath) - - const newFileDiff = await this.generateNewFileDiff(projectPath, projectName, relativeFilePath) - diffContent = this.rewriteDiff(newFileDiff) - } - return diffContent - } - - private rewriteDiff(inputStr: string): string { - const lines = inputStr.split('\n') - const rewrittenLines = lines.slice(0, 5).map((line) => { - line = line.replace(/\\\\/g, '/') - line = line.replace(/("a\/[^"]*)/g, (match, p1) => p1) - line = line.replace(/("b\/[^"]*)/g, (match, p1) => p1) - line = line.replace(/"/g, '') - - return line - }) - const outputLines = [...rewrittenLines, ...lines.slice(5)] - const outputStr = outputLines.join('\n') - - return outputStr - } - protected async processSourceFiles( zip: ZipStream, languageCount: Map, @@ -437,7 +260,7 @@ export class ZipUtil { await this.processBinaryFile(zip, file.fileUri, zipEntryPath) } else { const isFileOpenAndDirty = this.isFileOpenAndDirty(file.fileUri) - const fileContent = isFileOpenAndDirty ? await this.getTextContent(file.fileUri) : file.fileContent + const fileContent = isFileOpenAndDirty ? await getTextContent(file.fileUri) : file.fileContent this.processTextFile(zip, file.fileUri, fileContent, languageCount, zipEntryPath) } } @@ -629,3 +452,173 @@ export class ZipUtil { logger.verbose(`Complete cleaning up temporary files.`) } } + +// TODO: port this to its own utility with tests. +interface GitDiffOptions { + projectPath: string + projectName: string + filepath?: string + scope?: CodeWhispererConstants.CodeAnalysisScope +} + +async function getGitDiffContentForProjects( + projectPaths: string[], + filepath?: string, + scope?: CodeWhispererConstants.CodeAnalysisScope +) { + let gitDiffContent = '' + for (const projectPath of projectPaths) { + const projectName = path.basename(projectPath) + gitDiffContent += await getGitDiffContent({ + projectPath, + projectName, + filepath, + scope, + }) + } + return gitDiffContent +} + +async function getGitDiffContent(options: GitDiffOptions): Promise { + const { projectPath, projectName, filepath: filePath, scope } = options + const isProjectScope = scope === CodeWhispererConstants.CodeAnalysisScope.PROJECT + + const untrackedFilesString = await getGitUntrackedFiles(projectPath) + const untrackedFilesArray = untrackedFilesString?.trim()?.split('\n')?.filter(Boolean) + + if (isProjectScope && untrackedFilesArray && !untrackedFilesArray.length) { + return await generateHeadDiff(projectPath, projectName) + } + + let diffContent = '' + + if (isProjectScope) { + diffContent = await generateHeadDiff(projectPath, projectName) + + if (untrackedFilesArray) { + const untrackedDiffs = await Promise.all( + untrackedFilesArray.map((file) => generateNewFileDiff(projectPath, projectName, file)) + ) + diffContent += untrackedDiffs.join('') + } + } else if (!isProjectScope && filePath) { + const relativeFilePath = path.relative(projectPath, filePath) + + const newFileDiff = await generateNewFileDiff(projectPath, projectName, relativeFilePath) + diffContent = rewriteDiff(newFileDiff) + } + return diffContent +} + +async function getGitUntrackedFiles(projectPath: string): Promise { + const checkNewFileArgs = ['ls-files', '--others', '--exclude-standard'] + const checkProcess = new ChildProcess('git', checkNewFileArgs) + + try { + let output = '' + await checkProcess.run({ + rejectOnError: true, + rejectOnErrorCode: true, + onStdout: (text) => { + output += text + }, + spawnOptions: { + cwd: projectPath, + }, + }) + return output + } catch (err) { + getLogger().warn(`Failed to check if file is new: ${err}`) + return undefined + } +} + +async function generateHeadDiff(projectPath: string, projectName: string, relativePath?: string): Promise { + let diffContent = '' + + const gitArgs = [ + 'diff', + 'HEAD', + `--src-prefix=a/${projectName}/`, + `--dst-prefix=b/${projectName}/`, + ...(relativePath ? [relativePath] : []), + ] + + const childProcess = new ChildProcess('git', gitArgs) + + const runOptions: ChildProcessOptions = { + rejectOnError: true, + rejectOnErrorCode: true, + onStdout: (text) => { + diffContent += text + getLogger().verbose(removeAnsi(text)) + }, + onStderr: (text) => { + getLogger().error(removeAnsi(text)) + }, + spawnOptions: { + cwd: projectPath, + }, + } + + try { + await childProcess.run(runOptions) + return diffContent + } catch (err) { + getLogger().warn(`Failed to run command \`${childProcess.toString()}\`: ${err}`) + return '' + } +} + +async function generateNewFileDiff(projectPath: string, projectName: string, relativePath: string): Promise { + let diffContent = '' + + const gitArgs = [ + 'diff', + '--no-index', + `--src-prefix=a/${projectName}/`, + `--dst-prefix=b/${projectName}/`, + '/dev/null', // Use /dev/null as the old file + relativePath, + ] + + const childProcess = new ChildProcess('git', gitArgs) + const runOptions: ChildProcessOptions = { + rejectOnError: false, + rejectOnErrorCode: false, + onStdout: (text) => { + diffContent += text + getLogger().verbose(removeAnsi(text)) + }, + onStderr: (text) => { + getLogger().error(removeAnsi(text)) + }, + spawnOptions: { + cwd: projectPath, + }, + } + + try { + await childProcess.run(runOptions) + return diffContent + } catch (err) { + getLogger().warn(`Failed to run diff command: ${err}`) + return '' + } +} + +function rewriteDiff(inputStr: string): string { + const lines = inputStr.split('\n') + const rewrittenLines = lines.slice(0, 5).map((line) => { + line = line.replace(/\\\\/g, '/') + line = line.replace(/("a\/[^"]*)/g, (match, p1) => p1) + line = line.replace(/("b\/[^"]*)/g, (match, p1) => p1) + line = line.replace(/"/g, '') + + return line + }) + const outputLines = [...rewrittenLines, ...lines.slice(5)] + const outputStr = outputLines.join('\n') + + return outputStr +} diff --git a/packages/core/src/shared/utilities/textDocumentUtilities.ts b/packages/core/src/shared/utilities/textDocumentUtilities.ts index 48a20a6c44b..9fa030e70b1 100644 --- a/packages/core/src/shared/utilities/textDocumentUtilities.ts +++ b/packages/core/src/shared/utilities/textDocumentUtilities.ts @@ -267,3 +267,9 @@ class ReadonlyDocument { } export const readonlyDocument = new ReadonlyDocument() + +export async function getTextContent(uri: vscode.Uri) { + const document = await vscode.workspace.openTextDocument(uri) + const content = document.getText() + return content +} diff --git a/packages/core/src/shared/utilities/workspaceUtils.ts b/packages/core/src/shared/utilities/workspaceUtils.ts index 12cce75b3ff..9d587c4c2d8 100644 --- a/packages/core/src/shared/utilities/workspaceUtils.ts +++ b/packages/core/src/shared/utilities/workspaceUtils.ts @@ -671,3 +671,14 @@ export async function findStringInDirectory(searchStr: string, dirPath: string) }) return spawnResult } + +export function getWorkspacePaths() { + const workspaceFolders = vscode.workspace.workspaceFolders + return workspaceFolders?.map((folder) => folder.uri.fsPath) ?? [] +} + +export function getWorkspaceForFile(filepath: string) { + const fileUri = vscode.Uri.file(filepath) + const workspaceFolder = vscode.workspace.getWorkspaceFolder(fileUri) + return workspaceFolder?.uri.fsPath +} diff --git a/packages/core/src/test/amazonq/util/workspaceUtils.test.ts b/packages/core/src/test/amazonq/util/workspaceUtils.test.ts new file mode 100644 index 00000000000..8a1393919ae --- /dev/null +++ b/packages/core/src/test/amazonq/util/workspaceUtils.test.ts @@ -0,0 +1,23 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'assert' +import path from 'path' +import { getWorkspaceForFile, getWorkspacePaths } from '../../../shared/utilities/workspaceUtils' +import { getTestWorkspaceFolder } from '../../../testInteg/integrationTestsUtilities' + +describe('getWorkspace utilities', function () { + const workspaceFolder = getTestWorkspaceFolder() + const appRoot = path.join(workspaceFolder, 'java11-plain-maven-sam-app') + const appCodePath = path.join(appRoot, 'HelloWorldFunction', 'src', 'main', 'java', 'helloworld', 'App.java') + + it('returns the correct workspace paths', function () { + assert.deepStrictEqual(getWorkspacePaths(), [workspaceFolder]) + }) + + it('returns the correct worspace for a filepath', function () { + assert.deepStrictEqual(getWorkspaceForFile(appCodePath), workspaceFolder) + }) +}) diff --git a/packages/core/src/test/codewhisperer/zipUtil.test.ts b/packages/core/src/test/codewhisperer/zipUtil.test.ts index a82db4a6840..f678e959027 100644 --- a/packages/core/src/test/codewhisperer/zipUtil.test.ts +++ b/packages/core/src/test/codewhisperer/zipUtil.test.ts @@ -24,18 +24,6 @@ describe('zipUtil', function () { const appCodePath = join(appRoot, 'HelloWorldFunction', 'src', 'main', 'java', 'helloworld', 'App.java') const appCodePathWithRepeatedProjectName = join(workspaceFolder, 'workspaceFolder', 'App.java') - describe('getProjectPaths', function () { - it('Should return the correct project paths', function () { - const zipUtil = new ZipUtil() - assert.deepStrictEqual(zipUtil.getProjectPaths(), [workspaceFolder]) - }) - - it('Should return the correct project path for unit test generation', function () { - const zipUtil = new ZipUtil() - assert.deepStrictEqual(zipUtil.getProjectPath(appCodePath), workspaceFolder) - }) - }) - describe('generateZip', function () { let zipUtil: ZipUtil beforeEach(function () { diff --git a/packages/core/src/testE2E/codewhisperer/securityScan.test.ts b/packages/core/src/testE2E/codewhisperer/securityScan.test.ts index 730b9628290..67200b46310 100644 --- a/packages/core/src/testE2E/codewhisperer/securityScan.test.ts +++ b/packages/core/src/testE2E/codewhisperer/securityScan.test.ts @@ -23,6 +23,7 @@ import { makeTemporaryToolkitFolder } from '../../shared/filesystemUtilities' import fs from '../../shared/fs/fs' import { ZipUtil } from '../../codewhisperer/util/zipUtil' import { randomUUID } from '../../shared/crypto' +import { getWorkspacePaths } from '../../shared/utilities/workspaceUtils' const filePromptWithSecurityIssues = `from flask import app @@ -92,7 +93,7 @@ describe('CodeWhisperer security scan', async function () { const zipUtil = new ZipUtil() const uri = editor.document.uri - const projectPaths = zipUtil.getProjectPaths() + const projectPaths = getWorkspacePaths() const scope = CodeWhispererConstants.CodeAnalysisScope.PROJECT const zipMetadata = await zipUtil.generateZip(uri, scope) const codeScanName = randomUUID()