From 776dffbbde6823487750dc0f4f3483dc70860316 Mon Sep 17 00:00:00 2001 From: hkobew Date: Fri, 21 Mar 2025 16:31:55 -0400 Subject: [PATCH 1/6] refactor: move functions dealing with workspace to workspaceUtils --- .../commands/startSecurityScan.ts | 3 ++- .../commands/startTestGeneration.ts | 3 ++- .../core/src/codewhisperer/util/zipUtil.ts | 20 ++++++---------- .../src/shared/utilities/workspaceUtils.ts | 11 +++++++++ .../test/amazonq/util/workspaceUtils.test.ts | 23 +++++++++++++++++++ .../src/test/codewhisperer/zipUtil.test.ts | 12 ---------- .../codewhisperer/securityScan.test.ts | 3 ++- 7 files changed, 47 insertions(+), 28 deletions(-) create mode 100644 packages/core/src/test/amazonq/util/workspaceUtils.test.ts diff --git a/packages/core/src/codewhisperer/commands/startSecurityScan.ts b/packages/core/src/codewhisperer/commands/startSecurityScan.ts index cc9b733f56d..35f7649e16d 100644 --- a/packages/core/src/codewhisperer/commands/startSecurityScan.ts +++ b/packages/core/src/codewhisperer/commands/startSecurityScan.ts @@ -50,6 +50,7 @@ import { import { SecurityIssueTreeViewProvider } from '../service/securityIssueTreeViewProvider' import { ChatSessionManager } from '../../amazonqScan/chat/storages/chatSession' import { TelemetryHelper } from '../util/telemetryHelper' +import { getWorkspacePaths } from '../../shared/utilities/workspaceUtils' const localize = nls.loadMessageBundle() export const stopScanButton = localize('aws.codewhisperer.stopscan', 'Stop Scan') @@ -156,7 +157,7 @@ export async function startSecurityScan( }) } const zipMetadata = await zipUtil.generateZip(editor?.document.uri, scope) - const projectPaths = zipUtil.getProjectPaths() + const projectPaths = getWorkspacePaths() const contextTruncationStartTime = performance.now() codeScanTelemetryEntry.contextTruncationDuration = performance.now() - contextTruncationStartTime diff --git a/packages/core/src/codewhisperer/commands/startTestGeneration.ts b/packages/core/src/codewhisperer/commands/startTestGeneration.ts index 29e7148a17b..4d0443a1ba7 100644 --- a/packages/core/src/codewhisperer/commands/startTestGeneration.ts +++ b/packages/core/src/codewhisperer/commands/startTestGeneration.ts @@ -21,6 +21,7 @@ import { ChildProcess, spawn } from 'child_process' // eslint-disable-line no-re import { BuildStatus } from '../../amazonqTest/chat/session/session' import { fs } from '../../shared/fs/fs' import { Range } from '../client/codewhispereruserclient' +import { getWorkspaceForFile } from '../../shared/utilities/workspaceUtils' // eslint-disable-next-line unicorn/no-null let spawnResult: ChildProcess | null = null @@ -48,7 +49,7 @@ export async function startTestGenerationProcess( const zipUtil = new ZipUtil() if (initialExecution) { - const projectPath = zipUtil.getProjectPath(filePath) ?? '' + const projectPath = getWorkspaceForFile(filePath) ?? '' const relativeTargetPath = path.relative(projectPath, filePath) session.listOfTestGenerationJobId = [] session.shortAnswer = undefined diff --git a/packages/core/src/codewhisperer/util/zipUtil.ts b/packages/core/src/codewhisperer/util/zipUtil.ts index 32687a6452c..c1361d57793 100644 --- a/packages/core/src/codewhisperer/util/zipUtil.ts +++ b/packages/core/src/codewhisperer/util/zipUtil.ts @@ -12,7 +12,12 @@ import { fs } from '../../shared/fs/fs' import { getLoggerForScope } from '../service/securityScanHandler' import { runtimeLanguageContext } from './runtimeLanguageContext' import { CodewhispererLanguage } from '../../shared/telemetry/telemetry.gen' -import { CurrentWsFolders, collectFiles, defaultExcludePatterns } from '../../shared/utilities/workspaceUtils' +import { + CurrentWsFolders, + collectFiles, + defaultExcludePatterns, + getWorkspacePaths, +} from '../../shared/utilities/workspaceUtils' import { FileSizeExceededError, NoActiveFileError, @@ -72,17 +77,6 @@ export class ZipUtil { return CodeWhispererConstants.projectScanPayloadSizeLimitBytes } - public getProjectPaths() { - const workspaceFolders = vscode.workspace.workspaceFolders - return workspaceFolders?.map((folder) => folder.uri.fsPath) ?? [] - } - - public getProjectPath(filePath: string) { - const fileUri = vscode.Uri.file(filePath) - const workspaceFolder = vscode.workspace.getWorkspaceFolder(fileUri) - return workspaceFolder?.uri.fsPath - } - protected async getTextContent(uri: vscode.Uri) { const document = await vscode.workspace.openTextDocument(uri) const content = document.getText() @@ -211,7 +205,7 @@ export class ZipUtil { if (useCase === FeatureUseCase.TEST_GENERATION && projectPath) { projectPaths.push(projectPath) } else { - projectPaths = this.getProjectPaths() + projectPaths = getWorkspacePaths() } if (useCase === FeatureUseCase.CODE_SCAN) { await this.processCombinedGitDiff(zip, projectPaths, '', CodeWhispererConstants.CodeAnalysisScope.PROJECT) diff --git a/packages/core/src/shared/utilities/workspaceUtils.ts b/packages/core/src/shared/utilities/workspaceUtils.ts index 12cce75b3ff..9d587c4c2d8 100644 --- a/packages/core/src/shared/utilities/workspaceUtils.ts +++ b/packages/core/src/shared/utilities/workspaceUtils.ts @@ -671,3 +671,14 @@ export async function findStringInDirectory(searchStr: string, dirPath: string) }) return spawnResult } + +export function getWorkspacePaths() { + const workspaceFolders = vscode.workspace.workspaceFolders + return workspaceFolders?.map((folder) => folder.uri.fsPath) ?? [] +} + +export function getWorkspaceForFile(filepath: string) { + const fileUri = vscode.Uri.file(filepath) + const workspaceFolder = vscode.workspace.getWorkspaceFolder(fileUri) + return workspaceFolder?.uri.fsPath +} diff --git a/packages/core/src/test/amazonq/util/workspaceUtils.test.ts b/packages/core/src/test/amazonq/util/workspaceUtils.test.ts new file mode 100644 index 00000000000..3584ac3bd9a --- /dev/null +++ b/packages/core/src/test/amazonq/util/workspaceUtils.test.ts @@ -0,0 +1,23 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'assert' +import path from 'path' +import { getWorkspaceForFile, getWorkspacePaths } from '../../../shared/utilities/workspaceUtils' +import { getTestWorkspaceFolder } from '../../../testInteg/integrationTestsUtilities' + +describe('getProjectPaths', function () { + const workspaceFolder = getTestWorkspaceFolder() + const appRoot = path.join(workspaceFolder, 'java11-plain-maven-sam-app') + const appCodePath = path.join(appRoot, 'HelloWorldFunction', 'src', 'main', 'java', 'helloworld', 'App.java') + + it('Should return the correct project paths', function () { + assert.deepStrictEqual(getWorkspacePaths(), [workspaceFolder]) + }) + + it('Should return the correct project path for unit test generation', function () { + assert.deepStrictEqual(getWorkspaceForFile(appCodePath), workspaceFolder) + }) +}) diff --git a/packages/core/src/test/codewhisperer/zipUtil.test.ts b/packages/core/src/test/codewhisperer/zipUtil.test.ts index a82db4a6840..f678e959027 100644 --- a/packages/core/src/test/codewhisperer/zipUtil.test.ts +++ b/packages/core/src/test/codewhisperer/zipUtil.test.ts @@ -24,18 +24,6 @@ describe('zipUtil', function () { const appCodePath = join(appRoot, 'HelloWorldFunction', 'src', 'main', 'java', 'helloworld', 'App.java') const appCodePathWithRepeatedProjectName = join(workspaceFolder, 'workspaceFolder', 'App.java') - describe('getProjectPaths', function () { - it('Should return the correct project paths', function () { - const zipUtil = new ZipUtil() - assert.deepStrictEqual(zipUtil.getProjectPaths(), [workspaceFolder]) - }) - - it('Should return the correct project path for unit test generation', function () { - const zipUtil = new ZipUtil() - assert.deepStrictEqual(zipUtil.getProjectPath(appCodePath), workspaceFolder) - }) - }) - describe('generateZip', function () { let zipUtil: ZipUtil beforeEach(function () { diff --git a/packages/core/src/testE2E/codewhisperer/securityScan.test.ts b/packages/core/src/testE2E/codewhisperer/securityScan.test.ts index 730b9628290..67200b46310 100644 --- a/packages/core/src/testE2E/codewhisperer/securityScan.test.ts +++ b/packages/core/src/testE2E/codewhisperer/securityScan.test.ts @@ -23,6 +23,7 @@ import { makeTemporaryToolkitFolder } from '../../shared/filesystemUtilities' import fs from '../../shared/fs/fs' import { ZipUtil } from '../../codewhisperer/util/zipUtil' import { randomUUID } from '../../shared/crypto' +import { getWorkspacePaths } from '../../shared/utilities/workspaceUtils' const filePromptWithSecurityIssues = `from flask import app @@ -92,7 +93,7 @@ describe('CodeWhisperer security scan', async function () { const zipUtil = new ZipUtil() const uri = editor.document.uri - const projectPaths = zipUtil.getProjectPaths() + const projectPaths = getWorkspacePaths() const scope = CodeWhispererConstants.CodeAnalysisScope.PROJECT const zipMetadata = await zipUtil.generateZip(uri, scope) const codeScanName = randomUUID() From 98c7b96f3df4f85939e5a0e75adbed3bdbaed1b6 Mon Sep 17 00:00:00 2001 From: hkobew Date: Fri, 21 Mar 2025 16:35:56 -0400 Subject: [PATCH 2/6] refactor: move file opening util outside zipUtil --- packages/core/src/codewhisperer/util/zipUtil.ts | 11 +++-------- packages/core/src/shared/utilities/vsCodeUtils.ts | 6 ++++++ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/core/src/codewhisperer/util/zipUtil.ts b/packages/core/src/codewhisperer/util/zipUtil.ts index c1361d57793..d14ae99da7a 100644 --- a/packages/core/src/codewhisperer/util/zipUtil.ts +++ b/packages/core/src/codewhisperer/util/zipUtil.ts @@ -30,6 +30,7 @@ import { ProjectZipError } from '../../amazonqTest/error' import { removeAnsi } from '../../shared/utilities/textUtilities' import { normalize } from '../../shared/utilities/pathUtils' import { ZipStream } from '../../shared/utilities/zipStream' +import { getTextContent } from '../../shared/utilities/vsCodeUtils' export interface ZipMetadata { rootDir: string @@ -77,12 +78,6 @@ export class ZipUtil { return CodeWhispererConstants.projectScanPayloadSizeLimitBytes } - protected async getTextContent(uri: vscode.Uri) { - const document = await vscode.workspace.openTextDocument(uri) - const content = document.getText() - return content - } - public reachSizeLimit(size: number, scope: CodeWhispererConstants.CodeAnalysisScope): boolean { if ( scope === CodeWhispererConstants.CodeAnalysisScope.FILE_AUTO || @@ -105,7 +100,7 @@ export class ZipUtil { } const zip = new ZipStream() - const content = await this.getTextContent(uri) + const content = await getTextContent(uri) const workspaceFolder = vscode.workspace.getWorkspaceFolder(uri) if (workspaceFolder) { @@ -431,7 +426,7 @@ export class ZipUtil { await this.processBinaryFile(zip, file.fileUri, zipEntryPath) } else { const isFileOpenAndDirty = this.isFileOpenAndDirty(file.fileUri) - const fileContent = isFileOpenAndDirty ? await this.getTextContent(file.fileUri) : file.fileContent + const fileContent = isFileOpenAndDirty ? await getTextContent(file.fileUri) : file.fileContent this.processTextFile(zip, file.fileUri, fileContent, languageCount, zipEntryPath) } } diff --git a/packages/core/src/shared/utilities/vsCodeUtils.ts b/packages/core/src/shared/utilities/vsCodeUtils.ts index 03229cf104a..c8c9c58cc37 100644 --- a/packages/core/src/shared/utilities/vsCodeUtils.ts +++ b/packages/core/src/shared/utilities/vsCodeUtils.ts @@ -254,3 +254,9 @@ export function subscribeOnce(event: vscode.Event): vscode.Event { return result } } + +export async function getTextContent(uri: vscode.Uri) { + const document = await vscode.workspace.openTextDocument(uri) + const content = document.getText() + return content +} From 21a9a486fba6d538c26e431c4c0fb41decc2940f Mon Sep 17 00:00:00 2001 From: hkobew Date: Fri, 21 Mar 2025 16:55:40 -0400 Subject: [PATCH 3/6] refactor: move gitDiff files functionality out of zipUtil --- .../core/src/amazonq/util/gitDiffUtils.ts | 179 ++++++++++++++++++ .../src/codewhisperer/models/constants.ts | 4 + .../core/src/codewhisperer/util/zipUtil.ts | 179 +----------------- 3 files changed, 189 insertions(+), 173 deletions(-) create mode 100644 packages/core/src/amazonq/util/gitDiffUtils.ts diff --git a/packages/core/src/amazonq/util/gitDiffUtils.ts b/packages/core/src/amazonq/util/gitDiffUtils.ts new file mode 100644 index 00000000000..25a02a54b12 --- /dev/null +++ b/packages/core/src/amazonq/util/gitDiffUtils.ts @@ -0,0 +1,179 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +import path from 'path' +import { CodeWhispererConstants } from '../../codewhisperer/indexNode' +import { ChildProcess, ChildProcessOptions } from '../../shared/utilities/processUtils' +import { getLogger } from '../../shared/logger/logger' +import { removeAnsi } from '../../shared/utilities/textUtilities' + +interface GitDiffOptions { + projectPath: string + projectName: string + filepath?: string + scope?: CodeWhispererConstants.CodeAnalysisScope +} + +export async function getGitDiffContent( + projectPaths: string[], + filepath?: string, + scope?: CodeWhispererConstants.CodeAnalysisScope +) { + let gitDiffContent = '' + for (const projectPath of projectPaths) { + const projectName = path.basename(projectPath) + // Get diff content + gitDiffContent += await executeGitDiff({ + projectPath, + projectName, + filepath, + scope, + }) + } + return gitDiffContent +} + +async function executeGitDiff(options: GitDiffOptions): Promise { + const { projectPath, projectName, filepath: filePath, scope } = options + const isProjectScope = scope === CodeWhispererConstants.CodeAnalysisScope.PROJECT + + const untrackedFilesString = await getGitUntrackedFiles(projectPath) + const untrackedFilesArray = untrackedFilesString?.trim()?.split('\n')?.filter(Boolean) + + if (isProjectScope && untrackedFilesArray && !untrackedFilesArray.length) { + return await generateHeadDiff(projectPath, projectName) + } + + let diffContent = '' + + if (isProjectScope) { + diffContent = await generateHeadDiff(projectPath, projectName) + + if (untrackedFilesArray) { + const untrackedDiffs = await Promise.all( + untrackedFilesArray.map((file) => generateNewFileDiff(projectPath, projectName, file)) + ) + diffContent += untrackedDiffs.join('') + } + } else if (!isProjectScope && filePath) { + const relativeFilePath = path.relative(projectPath, filePath) + + const newFileDiff = await generateNewFileDiff(projectPath, projectName, relativeFilePath) + diffContent = rewriteDiff(newFileDiff) + } + return diffContent +} + +async function getGitUntrackedFiles(projectPath: string): Promise { + const checkNewFileArgs = ['ls-files', '--others', '--exclude-standard'] + const checkProcess = new ChildProcess('git', checkNewFileArgs) + + try { + let output = '' + await checkProcess.run({ + rejectOnError: true, + rejectOnErrorCode: true, + onStdout: (text) => { + output += text + }, + spawnOptions: { + cwd: projectPath, + }, + }) + return output + } catch (err) { + getLogger().warn(`Failed to check if file is new: ${err}`) + return undefined + } +} + +async function generateHeadDiff(projectPath: string, projectName: string, relativePath?: string): Promise { + let diffContent = '' + + const gitArgs = [ + 'diff', + 'HEAD', + `--src-prefix=a/${projectName}/`, + `--dst-prefix=b/${projectName}/`, + ...(relativePath ? [relativePath] : []), + ] + + const childProcess = new ChildProcess('git', gitArgs) + + const runOptions: ChildProcessOptions = { + rejectOnError: true, + rejectOnErrorCode: true, + onStdout: (text) => { + diffContent += text + getLogger().verbose(removeAnsi(text)) + }, + onStderr: (text) => { + getLogger().error(removeAnsi(text)) + }, + spawnOptions: { + cwd: projectPath, + }, + } + + try { + await childProcess.run(runOptions) + return diffContent + } catch (err) { + getLogger().warn(`Failed to run command \`${childProcess.toString()}\`: ${err}`) + return '' + } +} + +async function generateNewFileDiff(projectPath: string, projectName: string, relativePath: string): Promise { + let diffContent = '' + + const gitArgs = [ + 'diff', + '--no-index', + `--src-prefix=a/${projectName}/`, + `--dst-prefix=b/${projectName}/`, + '/dev/null', // Use /dev/null as the old file + relativePath, + ] + + const childProcess = new ChildProcess('git', gitArgs) + const runOptions: ChildProcessOptions = { + rejectOnError: false, + rejectOnErrorCode: false, + onStdout: (text) => { + diffContent += text + getLogger().verbose(removeAnsi(text)) + }, + onStderr: (text) => { + getLogger().error(removeAnsi(text)) + }, + spawnOptions: { + cwd: projectPath, + }, + } + + try { + await childProcess.run(runOptions) + return diffContent + } catch (err) { + getLogger().warn(`Failed to run diff command: ${err}`) + return '' + } +} + +function rewriteDiff(inputStr: string): string { + const lines = inputStr.split('\n') + const rewrittenLines = lines.slice(0, 5).map((line) => { + line = line.replace(/\\\\/g, '/') + line = line.replace(/("a\/[^"]*)/g, (match, p1) => p1) + line = line.replace(/("b\/[^"]*)/g, (match, p1) => p1) + line = line.replace(/"/g, '') + + return line + }) + const outputLines = [...rewrittenLines, ...lines.slice(5)] + const outputStr = outputLines.join('\n') + + return outputStr +} diff --git a/packages/core/src/codewhisperer/models/constants.ts b/packages/core/src/codewhisperer/models/constants.ts index 5762f8609da..cd399546b77 100644 --- a/packages/core/src/codewhisperer/models/constants.ts +++ b/packages/core/src/codewhisperer/models/constants.ts @@ -926,3 +926,7 @@ export const testGenExcludePatterns = [ '**/*.deb', '**/*.model', ] + +export const isFileAnalysisScope = (scope: CodeAnalysisScope) => { + return scope === CodeAnalysisScope.FILE_AUTO || scope === CodeAnalysisScope.FILE_ON_DEMAND +} diff --git a/packages/core/src/codewhisperer/util/zipUtil.ts b/packages/core/src/codewhisperer/util/zipUtil.ts index d14ae99da7a..c6a81178ef7 100644 --- a/packages/core/src/codewhisperer/util/zipUtil.ts +++ b/packages/core/src/codewhisperer/util/zipUtil.ts @@ -25,12 +25,11 @@ import { ProjectSizeExceededError, } from '../models/errors' import { FeatureUseCase } from '../models/constants' -import { ChildProcess, ChildProcessOptions } from '../../shared/utilities/processUtils' import { ProjectZipError } from '../../amazonqTest/error' -import { removeAnsi } from '../../shared/utilities/textUtilities' import { normalize } from '../../shared/utilities/pathUtils' import { ZipStream } from '../../shared/utilities/zipStream' import { getTextContent } from '../../shared/utilities/vsCodeUtils' +import { getGitDiffContent } from '../../amazonq/util/gitDiffUtils' export interface ZipMetadata { rootDir: string @@ -50,13 +49,6 @@ export const ZipConstants = { codeDiffFilePath: 'codeDiff/code.diff', } -interface GitDiffOptions { - projectPath: string - projectName: string - filePath?: string - scope?: CodeWhispererConstants.CodeAnalysisScope -} - export class ZipUtil { protected _pickedSourceFiles: Set = new Set() protected _pickedBuildFiles: Set = new Set() @@ -78,15 +70,10 @@ export class ZipUtil { return CodeWhispererConstants.projectScanPayloadSizeLimitBytes } - public reachSizeLimit(size: number, scope: CodeWhispererConstants.CodeAnalysisScope): boolean { - if ( - scope === CodeWhispererConstants.CodeAnalysisScope.FILE_AUTO || - scope === CodeWhispererConstants.CodeAnalysisScope.FILE_ON_DEMAND - ) { - return size > this.getFileScanPayloadSizeLimitInBytes() - } else { - return size > this.getProjectScanPayloadSizeLimitInBytes() - } + protected reachSizeLimit(size: number, scope: CodeWhispererConstants.CodeAnalysisScope): boolean { + return CodeWhispererConstants.isFileAnalysisScope(scope) + ? size > this.getFileScanPayloadSizeLimitInBytes() + : size > this.getProjectScanPayloadSizeLimitInBytes() } public willReachSizeLimit(current: number, adding: number): boolean { @@ -230,166 +217,12 @@ export class ZipUtil { filePath?: string, scope?: CodeWhispererConstants.CodeAnalysisScope ) { - let gitDiffContent = '' - for (const projectPath of projectPaths) { - const projectName = path.basename(projectPath) - // Get diff content - gitDiffContent += await this.executeGitDiff({ - projectPath, - projectName, - filePath, - scope, - }) - } + const gitDiffContent = await getGitDiffContent(projectPaths, filePath) if (gitDiffContent) { zip.writeString(gitDiffContent, ZipConstants.codeDiffFilePath) } } - private async getGitUntrackedFiles(projectPath: string): Promise { - const checkNewFileArgs = ['ls-files', '--others', '--exclude-standard'] - const checkProcess = new ChildProcess('git', checkNewFileArgs) - - try { - let output = '' - await checkProcess.run({ - rejectOnError: true, - rejectOnErrorCode: true, - onStdout: (text) => { - output += text - }, - spawnOptions: { - cwd: projectPath, - }, - }) - return output - } catch (err) { - getLogger().warn(`Failed to check if file is new: ${err}`) - return undefined - } - } - - private async generateNewFileDiff(projectPath: string, projectName: string, relativePath: string): Promise { - let diffContent = '' - - const gitArgs = [ - 'diff', - '--no-index', - `--src-prefix=a/${projectName}/`, - `--dst-prefix=b/${projectName}/`, - '/dev/null', // Use /dev/null as the old file - relativePath, - ] - - const childProcess = new ChildProcess('git', gitArgs) - const runOptions: ChildProcessOptions = { - rejectOnError: false, - rejectOnErrorCode: false, - onStdout: (text) => { - diffContent += text - getLogger().verbose(removeAnsi(text)) - }, - onStderr: (text) => { - getLogger().error(removeAnsi(text)) - }, - spawnOptions: { - cwd: projectPath, - }, - } - - try { - await childProcess.run(runOptions) - return diffContent - } catch (err) { - getLogger().warn(`Failed to run diff command: ${err}`) - return '' - } - } - - private async generateHeadDiff(projectPath: string, projectName: string, relativePath?: string): Promise { - let diffContent = '' - - const gitArgs = [ - 'diff', - 'HEAD', - `--src-prefix=a/${projectName}/`, - `--dst-prefix=b/${projectName}/`, - ...(relativePath ? [relativePath] : []), - ] - - const childProcess = new ChildProcess('git', gitArgs) - - const runOptions: ChildProcessOptions = { - rejectOnError: true, - rejectOnErrorCode: true, - onStdout: (text) => { - diffContent += text - getLogger().verbose(removeAnsi(text)) - }, - onStderr: (text) => { - getLogger().error(removeAnsi(text)) - }, - spawnOptions: { - cwd: projectPath, - }, - } - - try { - await childProcess.run(runOptions) - return diffContent - } catch (err) { - getLogger().warn(`Failed to run command \`${childProcess.toString()}\`: ${err}`) - return '' - } - } - - private async executeGitDiff(options: GitDiffOptions): Promise { - const { projectPath, projectName, filePath, scope } = options - const isProjectScope = scope === CodeWhispererConstants.CodeAnalysisScope.PROJECT - - const untrackedFilesString = await this.getGitUntrackedFiles(projectPath) - const untrackedFilesArray = untrackedFilesString?.trim()?.split('\n')?.filter(Boolean) - - if (isProjectScope && untrackedFilesArray && !untrackedFilesArray.length) { - return await this.generateHeadDiff(projectPath, projectName) - } - - let diffContent = '' - - if (isProjectScope) { - diffContent = await this.generateHeadDiff(projectPath, projectName) - - if (untrackedFilesArray) { - const untrackedDiffs = await Promise.all( - untrackedFilesArray.map((file) => this.generateNewFileDiff(projectPath, projectName, file)) - ) - diffContent += untrackedDiffs.join('') - } - } else if (!isProjectScope && filePath) { - const relativeFilePath = path.relative(projectPath, filePath) - - const newFileDiff = await this.generateNewFileDiff(projectPath, projectName, relativeFilePath) - diffContent = this.rewriteDiff(newFileDiff) - } - return diffContent - } - - private rewriteDiff(inputStr: string): string { - const lines = inputStr.split('\n') - const rewrittenLines = lines.slice(0, 5).map((line) => { - line = line.replace(/\\\\/g, '/') - line = line.replace(/("a\/[^"]*)/g, (match, p1) => p1) - line = line.replace(/("b\/[^"]*)/g, (match, p1) => p1) - line = line.replace(/"/g, '') - - return line - }) - const outputLines = [...rewrittenLines, ...lines.slice(5)] - const outputStr = outputLines.join('\n') - - return outputStr - } - protected async processSourceFiles( zip: ZipStream, languageCount: Map, From 787e1a636fc3908f93d0d6c806f6a714ca34f7af Mon Sep 17 00:00:00 2001 From: hkobew Date: Fri, 21 Mar 2025 17:05:21 -0400 Subject: [PATCH 4/6] fix: make method public so that it can be stubbed --- packages/core/src/codewhisperer/util/zipUtil.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/codewhisperer/util/zipUtil.ts b/packages/core/src/codewhisperer/util/zipUtil.ts index c6a81178ef7..ad14f0f9f95 100644 --- a/packages/core/src/codewhisperer/util/zipUtil.ts +++ b/packages/core/src/codewhisperer/util/zipUtil.ts @@ -70,7 +70,7 @@ export class ZipUtil { return CodeWhispererConstants.projectScanPayloadSizeLimitBytes } - protected reachSizeLimit(size: number, scope: CodeWhispererConstants.CodeAnalysisScope): boolean { + public reachSizeLimit(size: number, scope: CodeWhispererConstants.CodeAnalysisScope): boolean { return CodeWhispererConstants.isFileAnalysisScope(scope) ? size > this.getFileScanPayloadSizeLimitInBytes() : size > this.getProjectScanPayloadSizeLimitInBytes() From 807e9e2f8733e78ad7af8a230fe2c51c7b7d01b4 Mon Sep 17 00:00:00 2001 From: hkobew Date: Fri, 21 Mar 2025 17:19:46 -0400 Subject: [PATCH 5/6] refactor: do not export gitdiff --- .../core/src/amazonq/util/gitDiffUtils.ts | 179 ------------------ .../core/src/codewhisperer/util/zipUtil.ts | 177 ++++++++++++++++- .../shared/utilities/textDocumentUtilities.ts | 6 + .../core/src/shared/utilities/vsCodeUtils.ts | 6 - 4 files changed, 180 insertions(+), 188 deletions(-) delete mode 100644 packages/core/src/amazonq/util/gitDiffUtils.ts diff --git a/packages/core/src/amazonq/util/gitDiffUtils.ts b/packages/core/src/amazonq/util/gitDiffUtils.ts deleted file mode 100644 index 25a02a54b12..00000000000 --- a/packages/core/src/amazonq/util/gitDiffUtils.ts +++ /dev/null @@ -1,179 +0,0 @@ -/*! - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -import path from 'path' -import { CodeWhispererConstants } from '../../codewhisperer/indexNode' -import { ChildProcess, ChildProcessOptions } from '../../shared/utilities/processUtils' -import { getLogger } from '../../shared/logger/logger' -import { removeAnsi } from '../../shared/utilities/textUtilities' - -interface GitDiffOptions { - projectPath: string - projectName: string - filepath?: string - scope?: CodeWhispererConstants.CodeAnalysisScope -} - -export async function getGitDiffContent( - projectPaths: string[], - filepath?: string, - scope?: CodeWhispererConstants.CodeAnalysisScope -) { - let gitDiffContent = '' - for (const projectPath of projectPaths) { - const projectName = path.basename(projectPath) - // Get diff content - gitDiffContent += await executeGitDiff({ - projectPath, - projectName, - filepath, - scope, - }) - } - return gitDiffContent -} - -async function executeGitDiff(options: GitDiffOptions): Promise { - const { projectPath, projectName, filepath: filePath, scope } = options - const isProjectScope = scope === CodeWhispererConstants.CodeAnalysisScope.PROJECT - - const untrackedFilesString = await getGitUntrackedFiles(projectPath) - const untrackedFilesArray = untrackedFilesString?.trim()?.split('\n')?.filter(Boolean) - - if (isProjectScope && untrackedFilesArray && !untrackedFilesArray.length) { - return await generateHeadDiff(projectPath, projectName) - } - - let diffContent = '' - - if (isProjectScope) { - diffContent = await generateHeadDiff(projectPath, projectName) - - if (untrackedFilesArray) { - const untrackedDiffs = await Promise.all( - untrackedFilesArray.map((file) => generateNewFileDiff(projectPath, projectName, file)) - ) - diffContent += untrackedDiffs.join('') - } - } else if (!isProjectScope && filePath) { - const relativeFilePath = path.relative(projectPath, filePath) - - const newFileDiff = await generateNewFileDiff(projectPath, projectName, relativeFilePath) - diffContent = rewriteDiff(newFileDiff) - } - return diffContent -} - -async function getGitUntrackedFiles(projectPath: string): Promise { - const checkNewFileArgs = ['ls-files', '--others', '--exclude-standard'] - const checkProcess = new ChildProcess('git', checkNewFileArgs) - - try { - let output = '' - await checkProcess.run({ - rejectOnError: true, - rejectOnErrorCode: true, - onStdout: (text) => { - output += text - }, - spawnOptions: { - cwd: projectPath, - }, - }) - return output - } catch (err) { - getLogger().warn(`Failed to check if file is new: ${err}`) - return undefined - } -} - -async function generateHeadDiff(projectPath: string, projectName: string, relativePath?: string): Promise { - let diffContent = '' - - const gitArgs = [ - 'diff', - 'HEAD', - `--src-prefix=a/${projectName}/`, - `--dst-prefix=b/${projectName}/`, - ...(relativePath ? [relativePath] : []), - ] - - const childProcess = new ChildProcess('git', gitArgs) - - const runOptions: ChildProcessOptions = { - rejectOnError: true, - rejectOnErrorCode: true, - onStdout: (text) => { - diffContent += text - getLogger().verbose(removeAnsi(text)) - }, - onStderr: (text) => { - getLogger().error(removeAnsi(text)) - }, - spawnOptions: { - cwd: projectPath, - }, - } - - try { - await childProcess.run(runOptions) - return diffContent - } catch (err) { - getLogger().warn(`Failed to run command \`${childProcess.toString()}\`: ${err}`) - return '' - } -} - -async function generateNewFileDiff(projectPath: string, projectName: string, relativePath: string): Promise { - let diffContent = '' - - const gitArgs = [ - 'diff', - '--no-index', - `--src-prefix=a/${projectName}/`, - `--dst-prefix=b/${projectName}/`, - '/dev/null', // Use /dev/null as the old file - relativePath, - ] - - const childProcess = new ChildProcess('git', gitArgs) - const runOptions: ChildProcessOptions = { - rejectOnError: false, - rejectOnErrorCode: false, - onStdout: (text) => { - diffContent += text - getLogger().verbose(removeAnsi(text)) - }, - onStderr: (text) => { - getLogger().error(removeAnsi(text)) - }, - spawnOptions: { - cwd: projectPath, - }, - } - - try { - await childProcess.run(runOptions) - return diffContent - } catch (err) { - getLogger().warn(`Failed to run diff command: ${err}`) - return '' - } -} - -function rewriteDiff(inputStr: string): string { - const lines = inputStr.split('\n') - const rewrittenLines = lines.slice(0, 5).map((line) => { - line = line.replace(/\\\\/g, '/') - line = line.replace(/("a\/[^"]*)/g, (match, p1) => p1) - line = line.replace(/("b\/[^"]*)/g, (match, p1) => p1) - line = line.replace(/"/g, '') - - return line - }) - const outputLines = [...rewrittenLines, ...lines.slice(5)] - const outputStr = outputLines.join('\n') - - return outputStr -} diff --git a/packages/core/src/codewhisperer/util/zipUtil.ts b/packages/core/src/codewhisperer/util/zipUtil.ts index ad14f0f9f95..ad927174b5d 100644 --- a/packages/core/src/codewhisperer/util/zipUtil.ts +++ b/packages/core/src/codewhisperer/util/zipUtil.ts @@ -28,8 +28,9 @@ import { FeatureUseCase } from '../models/constants' import { ProjectZipError } from '../../amazonqTest/error' import { normalize } from '../../shared/utilities/pathUtils' import { ZipStream } from '../../shared/utilities/zipStream' -import { getTextContent } from '../../shared/utilities/vsCodeUtils' -import { getGitDiffContent } from '../../amazonq/util/gitDiffUtils' +import { getTextContent } from '../../shared/utilities/textDocumentUtilities' +import { ChildProcess, ChildProcessOptions } from '../../shared/utilities/processUtils' +import { removeAnsi } from '../../shared/utilities/textUtilities' export interface ZipMetadata { rootDir: string @@ -217,7 +218,7 @@ export class ZipUtil { filePath?: string, scope?: CodeWhispererConstants.CodeAnalysisScope ) { - const gitDiffContent = await getGitDiffContent(projectPaths, filePath) + const gitDiffContent = await getGitDiffContentForProjects(projectPaths, filePath) if (gitDiffContent) { zip.writeString(gitDiffContent, ZipConstants.codeDiffFilePath) } @@ -451,3 +452,173 @@ export class ZipUtil { logger.verbose(`Complete cleaning up temporary files.`) } } + +// TODO: port this to its own utility with tests. +interface GitDiffOptions { + projectPath: string + projectName: string + filepath?: string + scope?: CodeWhispererConstants.CodeAnalysisScope +} + +async function getGitDiffContentForProjects( + projectPaths: string[], + filepath?: string, + scope?: CodeWhispererConstants.CodeAnalysisScope +) { + let gitDiffContent = '' + for (const projectPath of projectPaths) { + const projectName = path.basename(projectPath) + gitDiffContent += await getGitDiffContent({ + projectPath, + projectName, + filepath, + scope, + }) + } + return gitDiffContent +} + +async function getGitDiffContent(options: GitDiffOptions): Promise { + const { projectPath, projectName, filepath: filePath, scope } = options + const isProjectScope = scope === CodeWhispererConstants.CodeAnalysisScope.PROJECT + + const untrackedFilesString = await getGitUntrackedFiles(projectPath) + const untrackedFilesArray = untrackedFilesString?.trim()?.split('\n')?.filter(Boolean) + + if (isProjectScope && untrackedFilesArray && !untrackedFilesArray.length) { + return await generateHeadDiff(projectPath, projectName) + } + + let diffContent = '' + + if (isProjectScope) { + diffContent = await generateHeadDiff(projectPath, projectName) + + if (untrackedFilesArray) { + const untrackedDiffs = await Promise.all( + untrackedFilesArray.map((file) => generateNewFileDiff(projectPath, projectName, file)) + ) + diffContent += untrackedDiffs.join('') + } + } else if (!isProjectScope && filePath) { + const relativeFilePath = path.relative(projectPath, filePath) + + const newFileDiff = await generateNewFileDiff(projectPath, projectName, relativeFilePath) + diffContent = rewriteDiff(newFileDiff) + } + return diffContent +} + +async function getGitUntrackedFiles(projectPath: string): Promise { + const checkNewFileArgs = ['ls-files', '--others', '--exclude-standard'] + const checkProcess = new ChildProcess('git', checkNewFileArgs) + + try { + let output = '' + await checkProcess.run({ + rejectOnError: true, + rejectOnErrorCode: true, + onStdout: (text) => { + output += text + }, + spawnOptions: { + cwd: projectPath, + }, + }) + return output + } catch (err) { + getLogger().warn(`Failed to check if file is new: ${err}`) + return undefined + } +} + +async function generateHeadDiff(projectPath: string, projectName: string, relativePath?: string): Promise { + let diffContent = '' + + const gitArgs = [ + 'diff', + 'HEAD', + `--src-prefix=a/${projectName}/`, + `--dst-prefix=b/${projectName}/`, + ...(relativePath ? [relativePath] : []), + ] + + const childProcess = new ChildProcess('git', gitArgs) + + const runOptions: ChildProcessOptions = { + rejectOnError: true, + rejectOnErrorCode: true, + onStdout: (text) => { + diffContent += text + getLogger().verbose(removeAnsi(text)) + }, + onStderr: (text) => { + getLogger().error(removeAnsi(text)) + }, + spawnOptions: { + cwd: projectPath, + }, + } + + try { + await childProcess.run(runOptions) + return diffContent + } catch (err) { + getLogger().warn(`Failed to run command \`${childProcess.toString()}\`: ${err}`) + return '' + } +} + +async function generateNewFileDiff(projectPath: string, projectName: string, relativePath: string): Promise { + let diffContent = '' + + const gitArgs = [ + 'diff', + '--no-index', + `--src-prefix=a/${projectName}/`, + `--dst-prefix=b/${projectName}/`, + '/dev/null', // Use /dev/null as the old file + relativePath, + ] + + const childProcess = new ChildProcess('git', gitArgs) + const runOptions: ChildProcessOptions = { + rejectOnError: false, + rejectOnErrorCode: false, + onStdout: (text) => { + diffContent += text + getLogger().verbose(removeAnsi(text)) + }, + onStderr: (text) => { + getLogger().error(removeAnsi(text)) + }, + spawnOptions: { + cwd: projectPath, + }, + } + + try { + await childProcess.run(runOptions) + return diffContent + } catch (err) { + getLogger().warn(`Failed to run diff command: ${err}`) + return '' + } +} + +function rewriteDiff(inputStr: string): string { + const lines = inputStr.split('\n') + const rewrittenLines = lines.slice(0, 5).map((line) => { + line = line.replace(/\\\\/g, '/') + line = line.replace(/("a\/[^"]*)/g, (match, p1) => p1) + line = line.replace(/("b\/[^"]*)/g, (match, p1) => p1) + line = line.replace(/"/g, '') + + return line + }) + const outputLines = [...rewrittenLines, ...lines.slice(5)] + const outputStr = outputLines.join('\n') + + return outputStr +} diff --git a/packages/core/src/shared/utilities/textDocumentUtilities.ts b/packages/core/src/shared/utilities/textDocumentUtilities.ts index 48a20a6c44b..9fa030e70b1 100644 --- a/packages/core/src/shared/utilities/textDocumentUtilities.ts +++ b/packages/core/src/shared/utilities/textDocumentUtilities.ts @@ -267,3 +267,9 @@ class ReadonlyDocument { } export const readonlyDocument = new ReadonlyDocument() + +export async function getTextContent(uri: vscode.Uri) { + const document = await vscode.workspace.openTextDocument(uri) + const content = document.getText() + return content +} diff --git a/packages/core/src/shared/utilities/vsCodeUtils.ts b/packages/core/src/shared/utilities/vsCodeUtils.ts index c8c9c58cc37..03229cf104a 100644 --- a/packages/core/src/shared/utilities/vsCodeUtils.ts +++ b/packages/core/src/shared/utilities/vsCodeUtils.ts @@ -254,9 +254,3 @@ export function subscribeOnce(event: vscode.Event): vscode.Event { return result } } - -export async function getTextContent(uri: vscode.Uri) { - const document = await vscode.workspace.openTextDocument(uri) - const content = document.getText() - return content -} From 6e5e2876f4b1a437cb73f8cf17d648288fbcf091 Mon Sep 17 00:00:00 2001 From: hkobew Date: Mon, 24 Mar 2025 10:21:42 -0400 Subject: [PATCH 6/6] docs: update test names --- packages/core/src/test/amazonq/util/workspaceUtils.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/core/src/test/amazonq/util/workspaceUtils.test.ts b/packages/core/src/test/amazonq/util/workspaceUtils.test.ts index 3584ac3bd9a..8a1393919ae 100644 --- a/packages/core/src/test/amazonq/util/workspaceUtils.test.ts +++ b/packages/core/src/test/amazonq/util/workspaceUtils.test.ts @@ -8,16 +8,16 @@ import path from 'path' import { getWorkspaceForFile, getWorkspacePaths } from '../../../shared/utilities/workspaceUtils' import { getTestWorkspaceFolder } from '../../../testInteg/integrationTestsUtilities' -describe('getProjectPaths', function () { +describe('getWorkspace utilities', function () { const workspaceFolder = getTestWorkspaceFolder() const appRoot = path.join(workspaceFolder, 'java11-plain-maven-sam-app') const appCodePath = path.join(appRoot, 'HelloWorldFunction', 'src', 'main', 'java', 'helloworld', 'App.java') - it('Should return the correct project paths', function () { + it('returns the correct workspace paths', function () { assert.deepStrictEqual(getWorkspacePaths(), [workspaceFolder]) }) - it('Should return the correct project path for unit test generation', function () { + it('returns the correct worspace for a filepath', function () { assert.deepStrictEqual(getWorkspaceForFile(appCodePath), workspaceFolder) }) })