From 840f68ff714f073ea0acb3b32aff5393a3fad6ea Mon Sep 17 00:00:00 2001 From: David Hasani Date: Tue, 11 Mar 2025 17:36:46 -0700 Subject: [PATCH 01/20] feat(amazonq): client-side build support --- .../test/e2e/amazonq/transformByQ.test.ts | 29 +++- .../amazonqGumby/transformApiHandler.test.ts | 63 ++++++++ .../chat/controller/controller.ts | 96 ++++++----- .../chat/controller/messenger/messenger.ts | 82 ++++++---- .../controller/messenger/messengerUtils.ts | 11 +- .../src/amazonqGumby/chat/session/session.ts | 3 +- .../commands/startTransformByQ.ts | 43 +---- .../src/codewhisperer/models/constants.ts | 8 +- .../core/src/codewhisperer/models/model.ts | 49 ++++-- .../transformByQ/transformApiHandler.ts | 150 +++++++++++++----- .../transformByQ/transformFileHandler.ts | 65 +++++++- .../transformByQ/transformMavenHandler.ts | 92 ++++++++--- .../transformationHubViewProvider.ts | 4 +- .../transformationResultsViewProvider.ts | 7 +- .../commands/transformByQ.test.ts | 17 ++ 15 files changed, 522 insertions(+), 197 deletions(-) diff --git a/packages/amazonq/test/e2e/amazonq/transformByQ.test.ts b/packages/amazonq/test/e2e/amazonq/transformByQ.test.ts index 784aed4884c..53097b97bf7 100644 --- a/packages/amazonq/test/e2e/amazonq/transformByQ.test.ts +++ b/packages/amazonq/test/e2e/amazonq/transformByQ.test.ts @@ -75,6 +75,9 @@ describe('Amazon Q Code Transformation', function () { }, ]) + transformByQState.setSourceJDKVersion(JDKVersion.JDK8) + transformByQState.setTargetJDKVersion(JDKVersion.JDK17) + tab.addChatMessage({ command: '/transform' }) // wait for /transform to respond with some intro messages and the first user input form @@ -141,20 +144,34 @@ describe('Amazon Q Code Transformation', function () { formItemValues: oneOrMultipleDiffsFormValues, }) - // 2 additional chat messages (including message with 4th form) get sent after 3rd form submitted; wait for both of them + // 2 additional chat messages get sent after 3rd form submitted; wait for both of them await tab.waitForEvent(() => tab.getChatItems().length > 11, { waitTimeoutInMs: 5000, waitIntervalInMs: 1000, }) - const jdkPathPrompt = tab.getChatItems().pop() - assert.strictEqual(jdkPathPrompt?.body?.includes('Enter the path to JDK'), true) - // 2 additional chat messages get sent after 4th form submitted; wait for both of them - tab.addChatMessage({ prompt: '/dummy/path/to/jdk8' }) + const customDependencyVersionPrompt = tab.getChatItems().pop() + assert.strictEqual( + customDependencyVersionPrompt?.body?.includes('You can optionally upload a YAML file'), + true + ) + tab.clickCustomFormButton({ id: 'gumbyTransformFormContinue' }) + + // 2 additional chat messages get sent after Continue button clicked; wait for both of them await tab.waitForEvent(() => tab.getChatItems().length > 13, { waitTimeoutInMs: 5000, waitIntervalInMs: 1000, }) + + const sourceJdkPathPrompt = tab.getChatItems().pop() + assert.strictEqual(sourceJdkPathPrompt?.body?.includes('Enter the path to JDK 8'), true) + + tab.addChatMessage({ prompt: '/dummy/path/to/jdk8' }) + // 2 additional chat messages get sent after JDK path submitted; wait for both of them + await tab.waitForEvent(() => tab.getChatItems().length > 15, { + waitTimeoutInMs: 5000, + waitIntervalInMs: 1000, + }) const jdkPathResponse = tab.getChatItems().pop() // this 'Sorry' message is OK - just making sure that the UI components are working correctly assert.strictEqual(jdkPathResponse?.body?.includes("Sorry, I couldn't locate your Java installation"), true) @@ -173,7 +190,7 @@ describe('Amazon Q Code Transformation', function () { text: 'View summary', }) - await tab.waitForEvent(() => tab.getChatItems().length > 14, { + await tab.waitForEvent(() => tab.getChatItems().length > 16, { waitTimeoutInMs: 5000, waitIntervalInMs: 1000, }) diff --git a/packages/amazonq/test/unit/amazonqGumby/transformApiHandler.test.ts b/packages/amazonq/test/unit/amazonqGumby/transformApiHandler.test.ts index 6ee0f05391d..4d787cca915 100644 --- a/packages/amazonq/test/unit/amazonqGumby/transformApiHandler.test.ts +++ b/packages/amazonq/test/unit/amazonqGumby/transformApiHandler.test.ts @@ -6,6 +6,7 @@ import assert from 'assert' import { TransformationProgressUpdate, TransformationStep, + findDownloadArtifactProgressUpdate, findDownloadArtifactStep, getArtifactsFromProgressUpdate, } from 'aws-core-vscode/codewhisperer/node' @@ -95,4 +96,66 @@ describe('Amazon Q Transform - transformApiHandler tests', function () { assert.strictEqual(progressUpdate, undefined) }) }) + + describe('findDownloadArtifactProgressUpdate', function () { + it('will return correct progress update from transformationStep', function () { + const transformationStepsFixture: TransformationStep[] = [ + { + id: 'dummy-id', + name: 'Step name', + description: 'Step description', + status: 'TRANSFORMING', + progressUpdates: [ + { + name: 'Progress update name', + status: 'AWAITING_CLIENT_ACTION', + description: 'Client-side build happening now', + startTime: new Date(), + endTime: new Date(), + downloadArtifacts: [ + { + downloadArtifactId: 'some-download-artifact-id', + downloadArtifactType: 'some-download-artifact-type', + }, + ], + }, + ], + startTime: new Date(), + endTime: new Date(), + }, + ] + const progressUpdate = findDownloadArtifactProgressUpdate(transformationStepsFixture) + assert.strictEqual(progressUpdate, transformationStepsFixture[0].progressUpdates?.[0]) + }) + + it('will return undefined if step status is NOT AWAITING_CLIENT_ACTION', function () { + const transformationStepsFixture: TransformationStep[] = [ + { + id: 'dummy-id', + name: 'Step name', + description: 'Step description', + status: 'TRANSFORMING', + progressUpdates: [ + { + name: 'Progress update name', + status: 'SOMETHING-BESIDES-AWAITING_CLIENT_ACTION', + description: 'Progress update description', + startTime: new Date(), + endTime: new Date(), + downloadArtifacts: [ + { + downloadArtifactId: 'some-download-artifact-id', + downloadArtifactType: 'some-download-artifact-type', + }, + ], + }, + ], + startTime: new Date(), + endTime: new Date(), + }, + ] + const progressUpdate = findDownloadArtifactProgressUpdate(transformationStepsFixture) + assert.strictEqual(progressUpdate, undefined) + }) + }) }) diff --git a/packages/core/src/amazonqGumby/chat/controller/controller.ts b/packages/core/src/amazonqGumby/chat/controller/controller.ts index c1e3bd34ea3..81dbca73029 100644 --- a/packages/core/src/amazonqGumby/chat/controller/controller.ts +++ b/packages/core/src/amazonqGumby/chat/controller/controller.ts @@ -25,7 +25,6 @@ import { processSQLConversionTransformFormInput, startTransformByQ, stopTransformByQ, - validateCanCompileProject, getValidSQLConversionCandidateProjects, openHilPomFile, } from '../../../codewhisperer/commands/startTransformByQ' @@ -33,7 +32,6 @@ import { JDKVersion, TransformationCandidateProject, transformByQState } from '. import { AbsolutePathDetectedError, AlternateDependencyVersionsNotFoundError, - JavaHomeNotSetError, JobStartError, ModuleUploadError, NoJavaProjectsFoundError, @@ -243,7 +241,7 @@ export class GumbyController { CodeTransformTelemetryState.instance.setSessionId() this.sessionStorage.getSession().conversationState = ConversationState.WAITING_FOR_TRANSFORMATION_OBJECTIVE - this.messenger.sendStaticTextResponse('choose-transformation-objective', message.tabID) + this.messenger.sendMessage(CodeWhispererConstants.chooseTransformationObjective, message.tabID, 'ai-prompt') this.messenger.sendChatInputEnabled(message.tabID, true) this.messenger.sendUpdatePlaceholder( message.tabID, @@ -299,7 +297,7 @@ export class GumbyController { const validProjects = await this.validateSQLConversionProjects(message) if (validProjects.length > 0) { this.sessionStorage.getSession().updateCandidateProjects(validProjects) - await this.messenger.sendSelectSQLMetadataFileMessage(message.tabID) + this.messenger.sendSelectSQLMetadataFileMessage(message.tabID) } }) .catch((err) => { @@ -383,6 +381,13 @@ export class GumbyController { case ButtonActions.SELECT_SQL_CONVERSION_METADATA_FILE: await this.processMetadataFile(message) break + case ButtonActions.SELECT_CUSTOM_DEPENDENCY_VERSION_FILE: + await this.processCustomDependencyVersionFile(message) + break + case ButtonActions.CONTINUE_TRANSFORMATION_FORM: + this.messenger.sendMessage('Ok, I will continue without this information.', message.tabID, 'ai-prompt') + this.promptJavaHome('source', message.tabID) + break case ButtonActions.VIEW_TRANSFORMATION_HUB: await vscode.commands.executeCommand(GumbyCommands.FOCUS_TRANSFORMATION_HUB, CancelActionPositions.Chat) break @@ -403,7 +408,7 @@ export class GumbyController { await this.continueJobWithSelectedDependency(message) break case ButtonActions.CANCEL_DEPENDENCY_FORM: - this.messenger.sendUserPrompt('Cancel', message.tabID) + this.messenger.sendMessage('Cancel', message.tabID, 'prompt') await this.continueTransformationWithoutHIL(message) break case ButtonActions.OPEN_FILE: @@ -448,11 +453,25 @@ export class GumbyController { }) this.messenger.sendOneOrMultipleDiffsMessage(oneOrMultipleDiffsSelection, message.tabID) - // perform local build - await this.validateBuildWithPromptOnError(message) + this.messenger.sendCustomDependencyVersionSelectionMessage(message.tabID) }) } + private promptJavaHome(type: 'source' | 'target', tabID: any) { + let jdkVersion = undefined + if (type === 'source') { + this.sessionStorage.getSession().conversationState = ConversationState.PROMPT_SOURCE_JAVA_HOME + jdkVersion = transformByQState.getSourceJDKVersion() + } else if (type === 'target') { + this.sessionStorage.getSession().conversationState = ConversationState.PROMPT_TARGET_JAVA_HOME + jdkVersion = transformByQState.getTargetJDKVersion() + } + const message = MessengerUtils.createJavaHomePrompt(jdkVersion) + this.messenger.sendMessage(message, tabID, 'ai-prompt') + this.messenger.sendChatInputEnabled(tabID, true) + this.messenger.sendUpdatePlaceholder(tabID, CodeWhispererConstants.enterJavaHomePlaceholder) + } + private async handleUserLanguageUpgradeProjectChoice(message: any) { await telemetry.codeTransform_submitSelection.run(async () => { const pathToProject: string = message.formSelectedValues['GumbyTransformLanguageUpgradeProjectForm'] @@ -521,15 +540,8 @@ export class GumbyController { }) } - private async prepareLanguageUpgradeProject(message: { pathToJavaHome: string; tabID: string }) { - if (message.pathToJavaHome) { - transformByQState.setJavaHome(message.pathToJavaHome) - getLogger().info( - `CodeTransformation: using JAVA_HOME = ${transformByQState.getJavaHome()} since source JDK does not match Maven JDK` - ) - } - - // Pre-build project locally + private async prepareLanguageUpgradeProject(message: any) { + // build project locally try { this.sessionStorage.getSession().conversationState = ConversationState.COMPILING this.messenger.sendCompilationInProgress(message.tabID) @@ -565,22 +577,21 @@ export class GumbyController { await startTransformByQ() } - // only for Language Upgrades - private async validateBuildWithPromptOnError(message: any | undefined = undefined): Promise { - try { - // Check Java Home is set (not yet prebuilding) - await validateCanCompileProject() - } catch (err: any) { - if (err instanceof JavaHomeNotSetError) { - this.sessionStorage.getSession().conversationState = ConversationState.PROMPT_JAVA_HOME - this.messenger.sendStaticTextResponse('java-home-not-set', message.tabID) - this.messenger.sendChatInputEnabled(message.tabID, true) - this.messenger.sendUpdatePlaceholder(message.tabID, 'Enter the path to your Java installation.') - } + private async processCustomDependencyVersionFile(message: any) { + const fileUri = await vscode.window.showOpenDialog({ + canSelectMany: false, + openLabel: 'Select', + filters: { + '.YAML file': ['yaml'], // Restrict user to only pick a .yaml file + }, + }) + if (!fileUri || fileUri.length === 0) { return } - - await this.prepareLanguageUpgradeProject(message) + // TO-DO: validate the YAML file? + this.messenger.sendMessage('Received custom dependency version YAML file!', message.tabID, 'ai-prompt') + transformByQState.setCustomDependencyVersionFilePath(fileUri[0].fsPath) + this.promptJavaHome('source', message.tabID) } private async processMetadataFile(message: any) { @@ -657,19 +668,28 @@ export class GumbyController { } private async processHumanChatMessage(data: { message: string; tabID: string }) { - this.messenger.sendUserPrompt(data.message, data.tabID) + this.messenger.sendMessage(data.message, data.tabID, 'prompt') this.messenger.sendChatInputEnabled(data.tabID, false) - this.messenger.sendUpdatePlaceholder(data.tabID, 'Open a new tab to chat with Q') + this.messenger.sendUpdatePlaceholder(data.tabID, CodeWhispererConstants.openNewTabPlaceholder) const session = this.sessionStorage.getSession() switch (session.conversationState) { - case ConversationState.PROMPT_JAVA_HOME: { + case ConversationState.PROMPT_SOURCE_JAVA_HOME: { const pathToJavaHome = extractPath(data.message) if (pathToJavaHome) { - await this.prepareLanguageUpgradeProject({ - pathToJavaHome, - tabID: data.tabID, - }) + transformByQState.setSourceJavaHome(pathToJavaHome) + this.promptJavaHome('target', data.tabID) // get target JDK path right after saving source JDK path + } else { + this.messenger.sendUnrecoverableErrorResponse('invalid-java-home', data.tabID) + } + break + } + + case ConversationState.PROMPT_TARGET_JAVA_HOME: { + const pathToJavaHome = extractPath(data.message) + if (pathToJavaHome) { + transformByQState.setTargetJavaHome(pathToJavaHome) + await this.prepareLanguageUpgradeProject(data) // build project locally right after saving target JDK path } else { this.messenger.sendUnrecoverableErrorResponse('invalid-java-home', data.tabID) } @@ -747,7 +767,7 @@ export class GumbyController { }) } - this.messenger.sendStaticTextResponse('end-HIL-early', message.tabID) + this.messenger.sendMessage(CodeWhispererConstants.continueWithoutHilMessage, message.tabID, 'ai-prompt') } } diff --git a/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts b/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts index 1f80aaf26c6..d3403478aa4 100644 --- a/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts +++ b/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts @@ -8,6 +8,7 @@ * As much as possible, all strings used in the experience should originate here. */ +import vscode from 'vscode' import { AuthFollowUpType, AuthMessageDataMap } from '../../../../amazonq/auth/model' import { JDKVersion, TransformationCandidateProject, transformByQState } from '../../../../codewhisperer/models/model' import { FeatureAuthState } from '../../../../codewhisperer/util/authUtil' @@ -416,38 +417,12 @@ export class Messenger { this.dispatcher.sendChatMessage(jobSubmittedMessage) } - public sendUserPrompt(prompt: string, tabID: string) { + public sendMessage(prompt: string, tabID: string, type: 'prompt' | 'ai-prompt') { this.dispatcher.sendChatMessage( new ChatMessage( { message: prompt, - messageType: 'prompt', - }, - tabID - ) - ) - } - - public sendStaticTextResponse(messageType: StaticTextResponseType, tabID: string) { - let message = '...' - - switch (messageType) { - case 'java-home-not-set': - message = MessengerUtils.createJavaHomePrompt() - break - case 'end-HIL-early': - message = 'I will continue transforming your code without upgrading this dependency.' - break - case 'choose-transformation-objective': - message = CodeWhispererConstants.chooseTransformationObjective - break - } - - this.dispatcher.sendChatMessage( - new ChatMessage( - { - message, - messageType: 'ai-prompt', + messageType: type, }, tabID ) @@ -772,7 +747,56 @@ ${codeSnippet} ) } - public async sendSelectSQLMetadataFileMessage(tabID: string) { + public async sendCustomDependencyVersionSelectionMessage(tabID: string) { + const message = 'You can optionally upload a YAML file to specify which dependency versions to upgrade to.' + const buttons: ChatItemButton[] = [] + + buttons.push({ + keepCardAfterClick: true, + text: 'Select .yaml file', + id: ButtonActions.SELECT_CUSTOM_DEPENDENCY_VERSION_FILE, + disabled: false, + }) + + buttons.push({ + keepCardAfterClick: false, + text: 'Continue without this', + id: ButtonActions.CONTINUE_TRANSFORMATION_FORM, + disabled: false, + }) + + this.dispatcher.sendChatMessage( + new ChatMessage( + { + message, + messageType: 'ai-prompt', + buttons, + }, + tabID + ) + ) + const sampleYAML = `name: "custom-dependency-management" +description: "Custom dependency version management for Java migration from JDK 8/11/17 to JDK 17/21" + +dependencyManagement: + dependencies: + - identifier: "com.example:library1" + targetVersion: "2.1.0" + versionProperty: "library1.version" # Optional + originType: "FIRST_PARTY" # or "THIRD_PARTY" # Optional + - identifier: "com.example:library2" + targetVersion: "3.0.0" + origin: "THIRD_PARTY" + plugins: + - identifier: "com.example.plugin" + targetVersion: "1.2.0" + versionProperty: "plugin.version" # Optional` + + const doc = await vscode.workspace.openTextDocument({ content: sampleYAML, language: 'yaml' }) + await vscode.window.showTextDocument(doc) + } + + public sendSelectSQLMetadataFileMessage(tabID: string) { const message = CodeWhispererConstants.selectSQLMetadataFileHelpMessage const buttons: ChatItemButton[] = [] diff --git a/packages/core/src/amazonqGumby/chat/controller/messenger/messengerUtils.ts b/packages/core/src/amazonqGumby/chat/controller/messenger/messengerUtils.ts index a7275ac98a3..ad1aade7c7e 100644 --- a/packages/core/src/amazonqGumby/chat/controller/messenger/messengerUtils.ts +++ b/packages/core/src/amazonqGumby/chat/controller/messenger/messengerUtils.ts @@ -5,7 +5,7 @@ */ import * as os from 'os' -import { transformByQState, JDKVersion } from '../../../../codewhisperer/models/model' +import { JDKVersion } from '../../../../codewhisperer/models/model' import * as CodeWhispererConstants from '../../../../codewhisperer/models/constants' import DependencyVersions from '../../../models/dependencies' @@ -20,6 +20,8 @@ export enum ButtonActions { CONFIRM_SKIP_TESTS_FORM = 'gumbyTransformSkipTestsFormConfirm', CONFIRM_SELECTIVE_TRANSFORMATION_FORM = 'gumbyTransformOneOrMultipleDiffsFormConfirm', SELECT_SQL_CONVERSION_METADATA_FILE = 'gumbySQLConversionMetadataTransformFormConfirm', + SELECT_CUSTOM_DEPENDENCY_VERSION_FILE = 'gumbyCustomDependencyVersionTransformFormConfirm', + CONTINUE_TRANSFORMATION_FORM = 'gumbyTransformFormContinue', CONFIRM_DEPENDENCY_FORM = 'gumbyTransformDependencyFormConfirm', CANCEL_DEPENDENCY_FORM = 'gumbyTransformDependencyFormCancel', CONFIRM_JAVA_HOME_FORM = 'gumbyJavaHomeFormConfirm', @@ -35,14 +37,11 @@ export enum GumbyCommands { } export default class MessengerUtils { - static createJavaHomePrompt = (): string => { - let javaHomePrompt = `${ - CodeWhispererConstants.enterJavaHomeChatMessage - } ${transformByQState.getSourceJDKVersion()}. \n` + static createJavaHomePrompt = (jdkVersion: JDKVersion | undefined): string => { + let javaHomePrompt = `${CodeWhispererConstants.enterJavaHomeChatMessage} ${jdkVersion}. \n` if (os.platform() === 'win32') { javaHomePrompt += CodeWhispererConstants.windowsJavaHomeHelpChatMessage } else if (os.platform() === 'darwin') { - const jdkVersion = transformByQState.getSourceJDKVersion() if (jdkVersion === JDKVersion.JDK8) { javaHomePrompt += ` ${CodeWhispererConstants.macJavaVersionHomeHelpChatMessage(1.8)}` } else if (jdkVersion === JDKVersion.JDK11) { diff --git a/packages/core/src/amazonqGumby/chat/session/session.ts b/packages/core/src/amazonqGumby/chat/session/session.ts index b0ed125c7d8..f1a69eb60ff 100644 --- a/packages/core/src/amazonqGumby/chat/session/session.ts +++ b/packages/core/src/amazonqGumby/chat/session/session.ts @@ -7,7 +7,8 @@ import { TransformationCandidateProject } from '../../../codewhisperer/models/mo export enum ConversationState { IDLE, - PROMPT_JAVA_HOME, + PROMPT_SOURCE_JAVA_HOME, + PROMPT_TARGET_JAVA_HOME, COMPILING, JOB_SUBMITTED, WAITING_FOR_HIL_INPUT, diff --git a/packages/core/src/codewhisperer/commands/startTransformByQ.ts b/packages/core/src/codewhisperer/commands/startTransformByQ.ts index 37b85be3fad..f888db4a10a 100644 --- a/packages/core/src/codewhisperer/commands/startTransformByQ.ts +++ b/packages/core/src/codewhisperer/commands/startTransformByQ.ts @@ -56,7 +56,6 @@ import { submitFeedback } from '../../feedback/vue/submitFeedback' import { placeholder } from '../../shared/vscode/commands2' import { AlternateDependencyVersionsNotFoundError, - JavaHomeNotSetError, JobStartError, ModuleUploadError, PollJobError, @@ -69,8 +68,7 @@ import { getJsonValuesFromManifestFile, highlightPomIssueInProject, parseVersionsListFromPomFile, - setMaven, - writeLogs, + writeAndShowBuildLogs, } from '../service/transformByQ/transformFileHandler' import { sleep } from '../../shared/utilities/timeoutUtils' import DependencyVersions from '../../amazonqGumby/models/dependencies' @@ -111,37 +109,6 @@ export async function processSQLConversionTransformFormInput(pathToProject: stri transformByQState.setTargetJDKVersion(JDKVersion.JDK17) } -async function validateJavaHome(): Promise { - const versionData = await getVersionData() - let javaVersionUsedByMaven = versionData[1] - if (javaVersionUsedByMaven !== undefined) { - javaVersionUsedByMaven = javaVersionUsedByMaven.slice(0, 3) - if (javaVersionUsedByMaven === '1.8') { - javaVersionUsedByMaven = JDKVersion.JDK8 - } else if (javaVersionUsedByMaven === '11.') { - javaVersionUsedByMaven = JDKVersion.JDK11 - } else if (javaVersionUsedByMaven === '17.') { - javaVersionUsedByMaven = JDKVersion.JDK17 - } else if (javaVersionUsedByMaven === '21.') { - javaVersionUsedByMaven = JDKVersion.JDK21 - } - } - if (javaVersionUsedByMaven !== transformByQState.getSourceJDKVersion()) { - // means either javaVersionUsedByMaven is undefined or it does not match the project JDK - return false - } - - return true -} - -export async function validateCanCompileProject() { - await setMaven() - const javaHomeFound = await validateJavaHome() - if (!javaHomeFound) { - throw new JavaHomeNotSetError() - } -} - export async function compileProject() { try { const dependenciesFolder: FolderInfo = getDependenciesFolderInfo() @@ -150,9 +117,7 @@ export async function compileProject() { await prepareProjectDependencies(dependenciesFolder, modulePath) } catch (err) { // open build-logs.txt file to show user error logs - const logFilePath = await writeLogs() - const doc = await vscode.workspace.openTextDocument(logFilePath) - await vscode.window.showTextDocument(doc) + await writeAndShowBuildLogs() throw err } } @@ -300,7 +265,7 @@ export async function initiateHumanInTheLoopPrompt(jobId: string) { const profile = AuthUtil.instance.regionProfileManager.activeRegionProfile const humanInTheLoopManager = HumanInTheLoopManager.instance // 1) We need to call GetTransformationPlan to get artifactId - const transformationSteps = await getTransformationSteps(jobId, false, profile) + const transformationSteps = await getTransformationSteps(jobId, profile) const { transformationStep, progressUpdate } = findDownloadArtifactStep(transformationSteps) if (!transformationStep || !progressUpdate) { @@ -566,7 +531,7 @@ export async function pollTransformationStatusUntilPlanReady(jobId: string, prof try { const tempToolkitFolder = await makeTemporaryToolkitFolder() const tempBuildLogsDir = path.join(tempToolkitFolder, 'q-transformation-build-logs') - await downloadAndExtractResultArchive(jobId, undefined, tempBuildLogsDir, 'Logs') + await downloadAndExtractResultArchive(jobId, tempBuildLogsDir) pathToLog = path.join(tempBuildLogsDir, 'buildCommandOutput.log') transformByQState.setPreBuildLogFilePath(pathToLog) } catch (e) { diff --git a/packages/core/src/codewhisperer/models/constants.ts b/packages/core/src/codewhisperer/models/constants.ts index 9d17b166cb8..71741c83c22 100644 --- a/packages/core/src/codewhisperer/models/constants.ts +++ b/packages/core/src/codewhisperer/models/constants.ts @@ -655,6 +655,12 @@ export const jobCancelledChatMessage = export const jobCancelledNotification = 'You cancelled the transformation.' +export const continueWithoutHilMessage = 'I will continue transforming your code without upgrading this dependency.' + +export const enterJavaHomePlaceholder = 'Enter the path to your Java installation' + +export const openNewTabPlaceholder = 'Open a new tab to chat with Q' + export const diffMessage = (multipleDiffs: boolean) => { return multipleDiffs ? 'You can review the diffs to see my proposed changes and accept or reject them. You will be able to accept changes from one diff at a time. If you reject changes in one diff, you will not be able to view or accept changes in the other diffs.' @@ -756,7 +762,7 @@ export const cleanInstallErrorChatMessage = `Sorry, I couldn\'t run the Maven cl export const cleanInstallErrorNotification = `Amazon Q could not run the Maven clean install command to build your project. For more information, see the [Amazon Q documentation](${codeTransformTroubleshootMvnFailure}).` -export const enterJavaHomeChatMessage = 'Enter the path to JDK ' +export const enterJavaHomeChatMessage = 'Enter the path to JDK' export const projectPromptChatMessage = 'I can upgrade your Java project. To start the transformation, I need some information from you. Choose the project you want to upgrade and the target code version to upgrade to. Then, choose Confirm.' diff --git a/packages/core/src/codewhisperer/models/model.ts b/packages/core/src/codewhisperer/models/model.ts index 99689748320..3962ae37195 100644 --- a/packages/core/src/codewhisperer/models/model.ts +++ b/packages/core/src/codewhisperer/models/model.ts @@ -684,7 +684,7 @@ export class ZipManifest { buildLogs: string = 'build-logs.txt' version: string = '1.0' hilCapabilities: string[] = ['HIL_1pDependency_VersionUpgrade'] - transformCapabilities: string[] = ['EXPLAINABILITY_V1'] + transformCapabilities: string[] = ['EXPLAINABILITY_V1', 'CLIENT_SIDE_BUILD'] customBuildCommand: string = 'clean test' requestedConversions?: { sqlConversion?: { @@ -771,6 +771,8 @@ export class TransformByQState { private metadataPathSQL: string = '' + private customVersionPath: string = '' + private linesOfCodeSubmitted: number | undefined = undefined private planFilePath: string = '' @@ -790,11 +792,13 @@ export class TransformByQState { private jobFailureErrorChatMessage: string | undefined = undefined - private errorLog: string = '' + private buildLog: string = '' private mavenName: string = '' - private javaHome: string | undefined = undefined + private sourceJavaHome: string | undefined = undefined + + private targetJavaHome: string | undefined = undefined private chatControllers: ChatControllerEventEmitters | undefined = undefined private chatMessenger: Messenger | undefined = undefined @@ -897,6 +901,10 @@ export class TransformByQState { return this.metadataPathSQL } + public getCustomDependencyVersionFilePath() { + return this.customVersionPath + } + public getStatus() { return this.transformByQState } @@ -937,16 +945,20 @@ export class TransformByQState { return this.jobFailureErrorChatMessage } - public getErrorLog() { - return this.errorLog + public getBuildLog() { + return this.buildLog } public getMavenName() { return this.mavenName } - public getJavaHome() { - return this.javaHome + public getSourceJavaHome() { + return this.sourceJavaHome + } + + public getTargetJavaHome() { + return this.targetJavaHome } public getChatControllers() { @@ -969,8 +981,12 @@ export class TransformByQState { return this.intervalId } - public appendToErrorLog(message: string) { - this.errorLog += `${message}\n\n` + public appendToBuildLog(message: string) { + this.buildLog += `${message}\n\n` + } + + public clearBuildLog() { + this.buildLog = '' } public setToNotStarted() { @@ -1061,6 +1077,10 @@ export class TransformByQState { this.metadataPathSQL = path } + public setCustomDependencyVersionFilePath(path: string) { + this.customVersionPath = path + } + public setPlanFilePath(filePath: string) { this.planFilePath = filePath } @@ -1101,8 +1121,12 @@ export class TransformByQState { this.mavenName = mavenName } - public setJavaHome(javaHome: string) { - this.javaHome = javaHome + public setSourceJavaHome(javaHome: string) { + this.sourceJavaHome = javaHome + } + + public setTargetJavaHome(javaHome: string) { + this.targetJavaHome = javaHome } public setChatControllers(controllers: ChatControllerEventEmitters) { @@ -1144,6 +1168,7 @@ export class TransformByQState { this.jobFailureMetadata = '' this.payloadFilePath = '' this.metadataPathSQL = '' + this.customVersionPath = '' this.sourceJDKVersion = undefined this.targetJDKVersion = undefined this.sourceDB = undefined @@ -1151,7 +1176,7 @@ export class TransformByQState { this.sourceServerName = '' this.schemaOptions.clear() this.schema = '' - this.errorLog = '' + this.buildLog = '' this.customBuildCommand = '' this.intervalId = undefined this.produceMultipleDiffs = false diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts index 97dacc1a664..143277abbc2 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts @@ -41,10 +41,10 @@ import { calculateTotalLatency } from '../../../amazonqGumby/telemetry/codeTrans import { MetadataResult } from '../../../shared/telemetry/telemetryClient' import request from '../../../shared/request' import { JobStoppedError, ZipExceedsSizeLimitError } from '../../../amazonqGumby/errors' -import { writeLogs } from './transformFileHandler' +import { copyDirectory, loadManifestFile, writeAndShowBuildLogs } from './transformFileHandler' import { createCodeWhispererChatStreamingClient } from '../../../shared/clients/codewhispererChatClient' import { downloadExportResultArchive } from '../../../shared/utilities/download' -import { ExportIntent, TransformationDownloadArtifactType } from '@amzn/codewhisperer-streaming' +import { ExportContext, ExportIntent, TransformationDownloadArtifactType } from '@amzn/codewhisperer-streaming' import fs from '../../../shared/fs/fs' import { ChatSessionManager } from '../../../amazonqGumby/chat/storages/chatSession' import { encodeHTML } from '../../../shared/utilities/textUtilities' @@ -52,6 +52,8 @@ import { convertToTimeString } from '../../../shared/datetime' import { getAuthType } from '../../../auth/utils' import { UserWrittenCodeTracker } from '../../tracker/userWrittenCodeTracker' import { AuthUtil } from '../../util/authUtil' +import { DiffModel } from './transformationResultsViewProvider' +import { runClientSideBuild } from './transformMavenHandler' export function getSha256(buffer: Buffer) { const hasher = crypto.createHash('sha256') @@ -167,10 +169,10 @@ export async function resumeTransformationJob(jobId: string, userActionStatus: T transformationJobId: jobId, userActionStatus, // can be "COMPLETED" or "REJECTED" }) - if (response) { - // always store request ID, but it will only show up in a notification if an error occurs - return response.transformationStatus - } + getLogger().info( + `CodeTransformation: resumeTransformation API status code = ${response.$response.httpResponse.statusCode}` + ) + return response.transformationStatus } catch (e: any) { const errorMessage = `Resuming the job failed due to: ${(e as Error).message}` getLogger().error(`CodeTransformation: ResumeTransformation error = %O`, e) @@ -219,6 +221,8 @@ export async function uploadPayload( throw new Error(errorMessage) } + getLogger().info('CodeTransformation: created upload URL successfully') + try { await uploadArtifactToS3(payloadFileName, response, sha256, buffer) } catch (e: any) { @@ -251,7 +255,8 @@ export async function uploadPayload( */ const mavenExcludedExtensions = ['.repositories', '.sha1'] -const sourceExcludedExtensions = ['.DS_Store'] +// TO-DO: should we exclude mvnw and mvnw.cmd? +const sourceExcludedExtensions = ['.DS_Store', 'mvnw', 'mvnw.cmd'] /** * Determines if the specified file path corresponds to a Maven metadata file @@ -396,7 +401,7 @@ export async function zipCode( throwIfCancelled() // add text file with logs from mvn clean install and mvn copy-dependencies - logFilePath = await writeLogs() + logFilePath = await writeAndShowBuildLogs() // We don't add build-logs.txt file to the manifest if we are // uploading HIL artifacts if (!humanInTheLoopFlag) { @@ -635,14 +640,9 @@ export async function getTransformationPlan(jobId: string, profile: RegionProfil export async function getTransformationSteps( jobId: string, - handleThrottleFlag: boolean, profile: RegionProfile | undefined ) { try { - // prevent ThrottlingException - if (handleThrottleFlag) { - await sleep(2000) - } const response = await codeWhisperer.codeWhispererClient.codeModernizerGetCodeTransformationPlan({ transformationJobId: jobId, profileArn: profile?.arn, @@ -693,6 +693,15 @@ export async function pollTransformationJob(jobId: string, validStates: string[] if (validStates.includes(status)) { break } + + if ( + status === 'TRANSFORMING' && + transformByQState.getTransformationType() === TransformationType.LANGUAGE_UPGRADE + ) { + // client-side build is N/A for SQL conversions + await attemptLocalBuild() + } + /** * If we find a paused state, we need the user to take action. We will set the global * state for polling status and early exit. @@ -718,7 +727,61 @@ export async function pollTransformationJob(jobId: string, validStates: string[] return status } -export function getArtifactsFromProgressUpdate(progressUpdate?: TransformationProgressUpdate) { +async function attemptLocalBuild() { + const jobId = transformByQState.getJobId() + const artifactId = await getClientInstructionArtifactId(jobId) + getLogger().info(`CodeTransformation: found artifactId = ${artifactId}`) + if (artifactId) { + const clientInstructionsPath = await downloadClientInstructions(jobId, artifactId) + getLogger().info( + `CodeTransformation: downloaded clientInstructions with diff.patch at: ${clientInstructionsPath}` + ) + await processClientInstructions(jobId, clientInstructionsPath, artifactId) + } +} + +async function getClientInstructionArtifactId(jobId: string) { + const steps = await getTransformationSteps(jobId, AuthUtil.instance.regionProfileManager.activeRegionProfile) + const progressUpdate = findDownloadArtifactProgressUpdate(steps) + + let artifactId = undefined + if (progressUpdate?.downloadArtifacts) { + artifactId = progressUpdate.downloadArtifacts[0].downloadArtifactId + } + return artifactId +} + +async function downloadClientInstructions(jobId: string, artifactId: string) { + const exportDestination = `downloadClientInstructions_${jobId}_${artifactId}` + const exportZipPath = path.join(os.tmpdir(), `${exportDestination}.zip`) + + const exportContext: ExportContext = { + transformationExportContext: { + downloadArtifactType: TransformationDownloadArtifactType.CLIENT_INSTRUCTIONS, + downloadArtifactId: artifactId, + }, + } + + await downloadAndExtractResultArchive(jobId, exportZipPath, exportContext) + + const clientInstructionsManifest = await loadManifestFile(exportZipPath) + return path.join(exportZipPath, clientInstructionsManifest.diffFileName) +} + +async function processClientInstructions(jobId: string, clientInstructionsPath: any, artifactId: string) { + const sourcePath = transformByQState.getProjectPath() + const destinationPath = path.join(os.tmpdir(), jobId, artifactId, 'originalCopy') + await copyDirectory(sourcePath, destinationPath) + getLogger().info(`CodeTransformation: copied project to ${destinationPath}`) + const diffModel = new DiffModel() + diffModel.parseDiff(clientInstructionsPath, destinationPath, undefined, 1, true) + // show user the diff.patch + const doc = await vscode.workspace.openTextDocument(clientInstructionsPath) + await vscode.window.showTextDocument(doc, { viewColumn: vscode.ViewColumn.One }) + await runClientSideBuild(transformByQState.getProjectCopyFilePath(), artifactId) +} + +export function getArtifactsFromProgressUpdate(progressUpdate: TransformationProgressUpdate) { const artifactType = progressUpdate?.downloadArtifacts?.[0]?.downloadArtifactType const artifactId = progressUpdate?.downloadArtifacts?.[0]?.downloadArtifactId return { @@ -727,6 +790,25 @@ export function getArtifactsFromProgressUpdate(progressUpdate?: TransformationPr } } +export function findDownloadArtifactProgressUpdate(transformationSteps: TransformationSteps) { + for (let i = 0; i < transformationSteps.length; i++) { + const progressUpdates = transformationSteps[i].progressUpdates + if (progressUpdates) { + for (let j = 0; j < progressUpdates.length; j++) { + if ( + progressUpdates[j].status === 'AWAITING_CLIENT_ACTION' && + progressUpdates[j].downloadArtifacts?.[0]?.downloadArtifactId + ) { + // TO-DO: make sure length is always 1 + console.log(`found progress update; length = ${progressUpdates[j].downloadArtifacts?.length}`) + return progressUpdates[j] + } + } + } + } + return undefined +} + export function findDownloadArtifactStep(transformationSteps: TransformationSteps) { for (let i = 0; i < transformationSteps.length; i++) { const progressUpdates = transformationSteps[i].progressUpdates @@ -750,24 +832,21 @@ export function findDownloadArtifactStep(transformationSteps: TransformationStep } } -export async function downloadResultArchive( - jobId: string, - downloadArtifactId: string | undefined, - pathToArchive: string, - downloadArtifactType: TransformationDownloadArtifactType -) { +export async function downloadResultArchive(jobId: string, pathToArchive: string, exportContext?: ExportContext) { const cwStreamingClient = await createCodeWhispererChatStreamingClient() try { - await downloadExportResultArchive( - cwStreamingClient, - { - exportId: jobId, - exportIntent: ExportIntent.TRANSFORMATION, - }, - pathToArchive, - AuthUtil.instance.regionProfileManager.activeRegionProfile - ) + const args = exportContext + ? { + exportId: jobId, + exportIntent: ExportIntent.TRANSFORMATION, + exportContext: exportContext, + } + : { + exportId: jobId, + exportIntent: ExportIntent.TRANSFORMATION, + } + await downloadExportResultArchive(cwStreamingClient, args, pathToArchive, AuthUtil.instance.regionProfileManager.activeRegionProfile) } catch (e: any) { getLogger().error(`CodeTransformation: ExportResultArchive error = %O`, e) throw e @@ -779,9 +858,8 @@ export async function downloadResultArchive( export async function downloadAndExtractResultArchive( jobId: string, - downloadArtifactId: string | undefined, pathToArchiveDir: string, - downloadArtifactType: TransformationDownloadArtifactType + exportContext?: ExportContext ) { const archivePathExists = await fs.existsDir(pathToArchiveDir) if (!archivePathExists) { @@ -793,9 +871,10 @@ export async function downloadAndExtractResultArchive( let downloadErrorMessage = undefined try { // Download and deserialize the zip - await downloadResultArchive(jobId, downloadArtifactId, pathToArchive, downloadArtifactType) + await downloadResultArchive(jobId, pathToArchive, exportContext) const zip = new AdmZip(pathToArchive) zip.extractAllTo(pathToArchiveDir) + getLogger().info(`CodeTransformation: downloaded result archive to: ${pathToArchiveDir}`) } catch (e) { downloadErrorMessage = (e as Error).message getLogger().error(`CodeTransformation: ExportResultArchive error = %O`, e) @@ -804,12 +883,7 @@ export async function downloadAndExtractResultArchive( } export async function downloadHilResultArchive(jobId: string, downloadArtifactId: string, pathToArchiveDir: string) { - await downloadAndExtractResultArchive( - jobId, - downloadArtifactId, - pathToArchiveDir, - TransformationDownloadArtifactType.CLIENT_INSTRUCTIONS - ) + await downloadAndExtractResultArchive(jobId, pathToArchiveDir) // manifest.json // pomFolder/pom.xml or manifest has pomFolderName path diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts index c2a0617c15f..d55bfb0f931 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts @@ -17,6 +17,7 @@ import { ChatSessionManager } from '../../../amazonqGumby/chat/storages/chatSess import { AbsolutePathDetectedError } from '../../../amazonqGumby/errors' import { getLogger } from '../../../shared/logger/logger' import { isWin } from '../../../shared/vscode/env' +import AdmZip from 'adm-zip' export function getDependenciesFolderInfo(): FolderInfo { const dependencyFolderName = `${CodeWhispererConstants.dependencyFolderName}${globals.clock.Date.now()}` @@ -27,12 +28,72 @@ export function getDependenciesFolderInfo(): FolderInfo { } } -export async function writeLogs() { +export async function writeAndShowBuildLogs() { const logFilePath = path.join(os.tmpdir(), 'build-logs.txt') - writeFileSync(logFilePath, transformByQState.getErrorLog()) + writeFileSync(logFilePath, transformByQState.getBuildLog()) + const doc = await vscode.workspace.openTextDocument(logFilePath) + if (!transformByQState.getBuildLog().includes('clean install succeeded')) { + // only show the log if the build failed + await vscode.window.showTextDocument(doc, { viewColumn: vscode.ViewColumn.Two }) + } return logFilePath } +export async function loadManifestFile(directory: string) { + const manifestFile = path.join(directory, 'manifest.json') + const data = await fs.readFileText(manifestFile) + const manifest = JSON.parse(data) + getLogger().info(`CodeTransformation: loaded and parsed manifest file from ${manifestFile}`) + return manifest +} + +export async function copyDirectory(sourcePath: string, destinationPath: string) { + fs.mkdir(destinationPath) + const files = await fs.readdir(sourcePath) + + for (const file of files) { + const sourceFilePath = path.join(sourcePath, file[0]) + const destinationFilePath = path.join(destinationPath, file[0]) + if (file[1] === vscode.FileType.Directory) { + // if the item is a directory, recursively copy it + const destinationFilePath = path.join(destinationPath, path.relative(sourcePath, sourceFilePath)) + await copyDirectory(sourceFilePath, destinationFilePath) + } else { + // if the item is a file, copy its contents + try { + await fs.copy(sourceFilePath, destinationFilePath) + } catch (err: any) { + getLogger().error( + `CodeTransformation: error copying file ${sourceFilePath} to ${destinationFilePath}: ${err}` + ) + } + } + } +} + +export async function createLocalBuildUploadZip(baseDir: string, exitCode: number | null, stdout: string) { + const manifestFilePath = path.join(baseDir, 'manifest.json') + const buildResultsManifest = { + capability: 'CLIENT_SIDE_BUILD', + exitCode: exitCode, + commandLogFileName: 'clientBuildLogs.log', + } + const formattedManifest = JSON.stringify(buildResultsManifest, null, 2) + await fs.writeFile(manifestFilePath, formattedManifest, 'utf8') + + const buildLogsFilePath = path.join(baseDir, 'clientBuildLogs.log') + await fs.writeFile(buildLogsFilePath, stdout, 'utf8') + + const zip = new AdmZip() + zip.addLocalFile(buildLogsFilePath) + zip.addLocalFile(manifestFilePath) + + const zipPath = `${baseDir}.zip` + zip.writeZip(zipPath) + getLogger().info(`CodeTransformation: created local build upload zip at ${zipPath}`) + return zipPath +} + export async function checkBuildSystem(projectPath: string) { const mavenBuildFilePath = path.join(projectPath, 'pom.xml') if (existsSync(mavenBuildFilePath)) { diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts index dacd78f6dc3..7c612b829c3 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts @@ -11,9 +11,13 @@ import { spawnSync } from 'child_process' // eslint-disable-line no-restricted-i import { CodeTransformBuildCommand, telemetry } from '../../../shared/telemetry/telemetry' import { CodeTransformTelemetryState } from '../../../amazonqGumby/telemetry/codeTransformTelemetryState' import { ToolkitError } from '../../../shared/errors' -import { setMaven, writeLogs } from './transformFileHandler' -import { throwIfCancelled } from './transformApiHandler' +import { createLocalBuildUploadZip, setMaven, writeAndShowBuildLogs } from './transformFileHandler' +import { resumeTransformationJob, throwIfCancelled, uploadPayload } from './transformApiHandler' import { sleep } from '../../../shared/utilities/timeoutUtils' +import path from 'path' +import os from 'os' +import { fs } from '../../../shared' +import { UploadContext } from '../../client/codewhispereruserclient' // run 'install' with either 'mvnw.cmd', './mvnw', or 'mvn' (if wrapper exists, we use that, otherwise we use regular 'mvn') function installProjectDependencies(dependenciesFolder: FolderInfo, modulePath: string) { @@ -23,19 +27,19 @@ function installProjectDependencies(dependenciesFolder: FolderInfo, modulePath: // baseCommand will be one of: '.\mvnw.cmd', './mvnw', 'mvn' const baseCommand = transformByQState.getMavenName() - transformByQState.appendToErrorLog(`Running command ${baseCommand} clean install`) - // Note: IntelliJ runs 'clean' separately from 'install'. Evaluate benefits (if any) of this. const args = [`-Dmaven.repo.local=${dependenciesFolder.path}`, 'clean', 'install', '-q'] + transformByQState.appendToBuildLog(`Running ${baseCommand} ${args.join(' ')}`) + if (transformByQState.getCustomBuildCommand() === CodeWhispererConstants.skipUnitTestsBuildCommand) { args.push('-DskipTests') } let environment = process.env - if (transformByQState.getJavaHome() !== undefined) { - environment = { ...process.env, JAVA_HOME: transformByQState.getJavaHome() } + if (transformByQState.getSourceJavaHome()) { + environment = { ...process.env, JAVA_HOME: transformByQState.getSourceJavaHome() } } const argString = args.join(' ') @@ -61,13 +65,13 @@ function installProjectDependencies(dependenciesFolder: FolderInfo, modulePath: let errorLog = '' errorLog += spawnResult.error ? JSON.stringify(spawnResult.error) : '' errorLog += `${spawnResult.stderr}\n${spawnResult.stdout}` - transformByQState.appendToErrorLog(`${baseCommand} ${argString} failed: \n ${errorLog}`) + transformByQState.appendToBuildLog(`${baseCommand} ${argString} failed: \n ${errorLog}`) getLogger().error( - `CodeTransformation: Error in running Maven ${argString} command ${baseCommand} = ${errorLog}` + `CodeTransformation: Error in running Maven command ${baseCommand} ${argString} = ${errorLog}` ) throw new ToolkitError(`Maven ${argString} error`, { code: 'MavenExecutionError' }) } else { - transformByQState.appendToErrorLog(`${baseCommand} ${argString} succeeded`) + transformByQState.appendToBuildLog(`Maven clean install succeeded`) } }) } @@ -76,8 +80,6 @@ function copyProjectDependencies(dependenciesFolder: FolderInfo, modulePath: str // baseCommand will be one of: '.\mvnw.cmd', './mvnw', 'mvn' const baseCommand = transformByQState.getMavenName() - transformByQState.appendToErrorLog(`Running command ${baseCommand} copy-dependencies`) - const args = [ 'dependency:copy-dependencies', `-DoutputDirectory=${dependenciesFolder.path}`, @@ -87,9 +89,11 @@ function copyProjectDependencies(dependenciesFolder: FolderInfo, modulePath: str '-q', ] + transformByQState.appendToBuildLog(`Running ${baseCommand} ${args.join(' ')}`) + let environment = process.env - if (transformByQState.getJavaHome() !== undefined) { - environment = { ...process.env, JAVA_HOME: transformByQState.getJavaHome() } + if (transformByQState.getSourceJavaHome()) { + environment = { ...process.env, JAVA_HOME: transformByQState.getSourceJavaHome() } } const spawnResult = spawnSync(baseCommand, args, { @@ -103,13 +107,13 @@ function copyProjectDependencies(dependenciesFolder: FolderInfo, modulePath: str let errorLog = '' errorLog += spawnResult.error ? JSON.stringify(spawnResult.error) : '' errorLog += `${spawnResult.stderr}\n${spawnResult.stdout}` - transformByQState.appendToErrorLog(`${baseCommand} copy-dependencies failed: \n ${errorLog}`) + transformByQState.appendToBuildLog(`${baseCommand} ${args} failed: \n ${errorLog}`) getLogger().info( - `CodeTransformation: Maven copy-dependencies command ${baseCommand} failed, but still continuing with transformation: ${errorLog}` + `CodeTransformation: Maven command ${baseCommand} ${args} failed, but still continuing with transformation: ${errorLog}` ) throw new Error('Maven copy-deps error') } else { - transformByQState.appendToErrorLog(`${baseCommand} copy-dependencies succeeded`) + transformByQState.appendToBuildLog(`Maven dependency:copy-dependencies succeeded`) } } @@ -133,9 +137,7 @@ export async function prepareProjectDependencies(dependenciesFolder: FolderInfo, } catch (err) { void vscode.window.showErrorMessage(CodeWhispererConstants.cleanInstallErrorNotification) // open build-logs.txt file to show user error logs - const logFilePath = await writeLogs() - const doc = await vscode.workspace.openTextDocument(logFilePath) - await vscode.window.showTextDocument(doc) + await writeAndShowBuildLogs() throw err } @@ -143,6 +145,52 @@ export async function prepareProjectDependencies(dependenciesFolder: FolderInfo, void vscode.window.showInformationMessage(CodeWhispererConstants.buildSucceededNotification) } +export async function runClientSideBuild(projectPath: string, clientInstructionArtifactId: string) { + // baseCommand will be one of: '.\mvnw.cmd', './mvnw', 'mvn' + const baseCommand = transformByQState.getMavenName() + const args = ['test'] + // TO-DO / QUESTION: why not use the build command from the downloaded manifest? + transformByQState.appendToBuildLog(`Running ${baseCommand} ${args}`) + const environment = { ...process.env, JAVA_HOME: transformByQState.getTargetJavaHome() } + + const argString = args.join(' ') + const spawnResult = spawnSync(baseCommand, args, { + cwd: projectPath, + shell: true, + encoding: 'utf-8', + env: environment, + }) + + const buildLogs = `Intermediate build result from running ${baseCommand} ${argString}:\n\n${spawnResult.stdout}` + transformByQState.clearBuildLog() + transformByQState.appendToBuildLog(buildLogs) + await writeAndShowBuildLogs() + + const baseDir = path.join( + os.tmpdir(), + `clientInstructionsResult_${transformByQState.getJobId()}_${clientInstructionArtifactId}` + ) + const zipPath = await createLocalBuildUploadZip(baseDir, spawnResult.status, spawnResult.stdout) + + // upload build results + const uploadContext: UploadContext = { + transformationUploadContext: { + jobId: transformByQState.getJobId(), + uploadArtifactType: 'ClientBuildResult', + }, + } + getLogger().info(`CodeTransformation: uploading client build results at ${zipPath} and resuming job now`) + await uploadPayload(zipPath, uploadContext) + await resumeTransformationJob(transformByQState.getJobId(), 'COMPLETED') + try { + await fs.delete(transformByQState.getProjectCopyFilePath(), { recursive: true }) + } catch { + getLogger().error( + `CodeTransformation: failed to delete project copy at ${transformByQState.getProjectCopyFilePath()} after client-side build` + ) + } +} + export async function getVersionData() { const baseCommand = transformByQState.getMavenName() // will be one of: 'mvnw.cmd', './mvnw', 'mvn' const projectPath = transformByQState.getProjectPath() @@ -188,9 +236,9 @@ export function runMavenDependencyUpdateCommands(dependenciesFolder: FolderInfo) ] let environment = process.env - // if JAVA_HOME not found or not matching project JDK, get user input for it and set here - if (transformByQState.getJavaHome() !== undefined) { - environment = { ...process.env, JAVA_HOME: transformByQState.getJavaHome() } + + if (transformByQState.getSourceJavaHome()) { + environment = { ...process.env, JAVA_HOME: transformByQState.getSourceJavaHome() } } const spawnResult = spawnSync(baseCommand, args, { diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformationHubViewProvider.ts b/packages/core/src/codewhisperer/service/transformByQ/transformationHubViewProvider.ts index 2d0585085a9..9bbb0dcefa5 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformationHubViewProvider.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformationHubViewProvider.ts @@ -193,6 +193,8 @@ export class TransformationHubViewProvider implements vscode.WebviewViewProvider return '

' case 'COMPLETED': return '

✓

' + case 'AWAITING_CLIENT_ACTION': + return '

' case 'FAILED': default: return '

𐔧

' @@ -327,7 +329,7 @@ export class TransformationHubViewProvider implements vscode.WebviewViewProvider transformByQState.isRunning() ) { const profile = AuthUtil.instance.regionProfileManager.activeRegionProfile - planSteps = await getTransformationSteps(transformByQState.getJobId(), false, profile) + planSteps = await getTransformationSteps(transformByQState.getJobId(), profile) transformByQState.setPlanSteps(planSteps) } let progressHtml diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts b/packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts index d1c6f368259..24353adcadc 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts @@ -168,7 +168,8 @@ export class DiffModel { pathToDiff: string, pathToWorkspace: string, diffDescription: PatchInfo | undefined, - totalDiffPatches: number + totalDiffPatches: number, + isIntermediateBuild: boolean = false ): PatchFileNode { this.patchFileNodes = [] const diffContents = fs.readFileSync(pathToDiff, 'utf8') @@ -181,7 +182,9 @@ export class DiffModel { const changedFiles = parsePatch(diffContents) getLogger().info('CodeTransformation: parsed patch file successfully') // path to the directory containing copy of the changed files in the transformed project - const pathToTmpSrcDir = this.copyProject(pathToWorkspace, changedFiles) + // if doing intermediate client-side build, pathToWorkspace is the already-copied project directory + // otherwise, we are at the very end of the transformation and need to copy the project to show the changes + const pathToTmpSrcDir = isIntermediateBuild ? pathToWorkspace : this.copyProject(pathToWorkspace, changedFiles) transformByQState.setProjectCopyFilePath(pathToTmpSrcDir) applyPatches(changedFiles, { diff --git a/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts b/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts index 3ec69e70b04..23e29a0fd09 100644 --- a/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts +++ b/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts @@ -41,6 +41,7 @@ import { setMaven, parseBuildFile, validateSQLMetadataFile, + createLocalBuildUploadZip, } from '../../../codewhisperer/service/transformByQ/transformFileHandler' import { uploadArtifactToS3 } from '../../../codewhisperer/indexNode' import request from '../../../shared/request' @@ -288,6 +289,22 @@ describe('transformByQ', function () { assert.strictEqual(transformByQState.getMavenName(), '.\\mvnw.cmd') }) + it(`WHEN local build zip created THEN zip contains all expected files and no unexpected files`, async function () { + const zipPath = await createLocalBuildUploadZip(tempDir, 0, 'sample stdout after running local build') + const zip = new AdmZip(zipPath) + const manifestEntry = zip.getEntry('manifest.json') + if (!manifestEntry) { + fail('manifest.json not found in the zip') + } + const manifestBuffer = manifestEntry.getData() + const manifestText = manifestBuffer.toString('utf8') + const manifest = JSON.parse(manifestText) + assert.strictEqual(manifest.capability, 'CLIENT_SIDE_BUILD') + assert.strictEqual(manifest.exitCode, 0) + assert.strictEqual(manifest.commandLogFileName, 'clientBuildLogs.log') + assert.strictEqual(zip.getEntries().length, 2) // expecting only manifest.json and clientBuildLogs.log + }) + it(`WHEN zip created THEN manifest.json contains test-compile custom build command`, async function () { const tempFileName = `testfile-${globals.clock.Date.now()}.zip` transformByQState.setProjectPath(tempDir) From 692f8909618f3408ddd1864423f7bef8a06026a9 Mon Sep 17 00:00:00 2001 From: David Hasani Date: Tue, 11 Mar 2025 17:55:16 -0700 Subject: [PATCH 02/20] fix imports --- .../transformByQ/transformApiHandler.ts | 55 ++++++++++++++++++- .../transformByQ/transformMavenHandler.ts | 54 +----------------- 2 files changed, 55 insertions(+), 54 deletions(-) diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts index 143277abbc2..b4d2eefebfd 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts @@ -41,7 +41,12 @@ import { calculateTotalLatency } from '../../../amazonqGumby/telemetry/codeTrans import { MetadataResult } from '../../../shared/telemetry/telemetryClient' import request from '../../../shared/request' import { JobStoppedError, ZipExceedsSizeLimitError } from '../../../amazonqGumby/errors' -import { copyDirectory, loadManifestFile, writeAndShowBuildLogs } from './transformFileHandler' +import { + copyDirectory, + createLocalBuildUploadZip, + loadManifestFile, + writeAndShowBuildLogs, +} from './transformFileHandler' import { createCodeWhispererChatStreamingClient } from '../../../shared/clients/codewhispererChatClient' import { downloadExportResultArchive } from '../../../shared/utilities/download' import { ExportContext, ExportIntent, TransformationDownloadArtifactType } from '@amzn/codewhisperer-streaming' @@ -53,7 +58,7 @@ import { getAuthType } from '../../../auth/utils' import { UserWrittenCodeTracker } from '../../tracker/userWrittenCodeTracker' import { AuthUtil } from '../../util/authUtil' import { DiffModel } from './transformationResultsViewProvider' -import { runClientSideBuild } from './transformMavenHandler' +import { spawnSync } from 'child_process' export function getSha256(buffer: Buffer) { const hasher = crypto.createHash('sha256') @@ -781,6 +786,52 @@ async function processClientInstructions(jobId: string, clientInstructionsPath: await runClientSideBuild(transformByQState.getProjectCopyFilePath(), artifactId) } +export async function runClientSideBuild(projectPath: string, clientInstructionArtifactId: string) { + // baseCommand will be one of: '.\mvnw.cmd', './mvnw', 'mvn' + const baseCommand = transformByQState.getMavenName() + const args = ['test'] + // TO-DO / QUESTION: why not use the build command from the downloaded manifest? + transformByQState.appendToBuildLog(`Running ${baseCommand} ${args}`) + const environment = { ...process.env, JAVA_HOME: transformByQState.getTargetJavaHome() } + + const argString = args.join(' ') + const spawnResult = spawnSync(baseCommand, args, { + cwd: projectPath, + shell: true, + encoding: 'utf-8', + env: environment, + }) + + const buildLogs = `Intermediate build result from running ${baseCommand} ${argString}:\n\n${spawnResult.stdout}` + transformByQState.clearBuildLog() + transformByQState.appendToBuildLog(buildLogs) + await writeAndShowBuildLogs() + + const baseDir = path.join( + os.tmpdir(), + `clientInstructionsResult_${transformByQState.getJobId()}_${clientInstructionArtifactId}` + ) + const zipPath = await createLocalBuildUploadZip(baseDir, spawnResult.status, spawnResult.stdout) + + // upload build results + const uploadContext: UploadContext = { + transformationUploadContext: { + jobId: transformByQState.getJobId(), + uploadArtifactType: 'ClientBuildResult', + }, + } + getLogger().info(`CodeTransformation: uploading client build results at ${zipPath} and resuming job now`) + await uploadPayload(zipPath, uploadContext) + await resumeTransformationJob(transformByQState.getJobId(), 'COMPLETED') + try { + await fs.delete(transformByQState.getProjectCopyFilePath(), { recursive: true }) + } catch { + getLogger().error( + `CodeTransformation: failed to delete project copy at ${transformByQState.getProjectCopyFilePath()} after client-side build` + ) + } +} + export function getArtifactsFromProgressUpdate(progressUpdate: TransformationProgressUpdate) { const artifactType = progressUpdate?.downloadArtifacts?.[0]?.downloadArtifactType const artifactId = progressUpdate?.downloadArtifacts?.[0]?.downloadArtifactId diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts index 7c612b829c3..f5eac536e13 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts @@ -11,13 +11,9 @@ import { spawnSync } from 'child_process' // eslint-disable-line no-restricted-i import { CodeTransformBuildCommand, telemetry } from '../../../shared/telemetry/telemetry' import { CodeTransformTelemetryState } from '../../../amazonqGumby/telemetry/codeTransformTelemetryState' import { ToolkitError } from '../../../shared/errors' -import { createLocalBuildUploadZip, setMaven, writeAndShowBuildLogs } from './transformFileHandler' -import { resumeTransformationJob, throwIfCancelled, uploadPayload } from './transformApiHandler' +import { setMaven, writeAndShowBuildLogs } from './transformFileHandler' +import { throwIfCancelled } from './transformApiHandler' import { sleep } from '../../../shared/utilities/timeoutUtils' -import path from 'path' -import os from 'os' -import { fs } from '../../../shared' -import { UploadContext } from '../../client/codewhispereruserclient' // run 'install' with either 'mvnw.cmd', './mvnw', or 'mvn' (if wrapper exists, we use that, otherwise we use regular 'mvn') function installProjectDependencies(dependenciesFolder: FolderInfo, modulePath: string) { @@ -145,52 +141,6 @@ export async function prepareProjectDependencies(dependenciesFolder: FolderInfo, void vscode.window.showInformationMessage(CodeWhispererConstants.buildSucceededNotification) } -export async function runClientSideBuild(projectPath: string, clientInstructionArtifactId: string) { - // baseCommand will be one of: '.\mvnw.cmd', './mvnw', 'mvn' - const baseCommand = transformByQState.getMavenName() - const args = ['test'] - // TO-DO / QUESTION: why not use the build command from the downloaded manifest? - transformByQState.appendToBuildLog(`Running ${baseCommand} ${args}`) - const environment = { ...process.env, JAVA_HOME: transformByQState.getTargetJavaHome() } - - const argString = args.join(' ') - const spawnResult = spawnSync(baseCommand, args, { - cwd: projectPath, - shell: true, - encoding: 'utf-8', - env: environment, - }) - - const buildLogs = `Intermediate build result from running ${baseCommand} ${argString}:\n\n${spawnResult.stdout}` - transformByQState.clearBuildLog() - transformByQState.appendToBuildLog(buildLogs) - await writeAndShowBuildLogs() - - const baseDir = path.join( - os.tmpdir(), - `clientInstructionsResult_${transformByQState.getJobId()}_${clientInstructionArtifactId}` - ) - const zipPath = await createLocalBuildUploadZip(baseDir, spawnResult.status, spawnResult.stdout) - - // upload build results - const uploadContext: UploadContext = { - transformationUploadContext: { - jobId: transformByQState.getJobId(), - uploadArtifactType: 'ClientBuildResult', - }, - } - getLogger().info(`CodeTransformation: uploading client build results at ${zipPath} and resuming job now`) - await uploadPayload(zipPath, uploadContext) - await resumeTransformationJob(transformByQState.getJobId(), 'COMPLETED') - try { - await fs.delete(transformByQState.getProjectCopyFilePath(), { recursive: true }) - } catch { - getLogger().error( - `CodeTransformation: failed to delete project copy at ${transformByQState.getProjectCopyFilePath()} after client-side build` - ) - } -} - export async function getVersionData() { const baseCommand = transformByQState.getMavenName() // will be one of: 'mvnw.cmd', './mvnw', 'mvn' const projectPath = transformByQState.getProjectPath() From 58b5208b621297bba17b4cf66b1c914647619a25 Mon Sep 17 00:00:00 2001 From: David Hasani Date: Wed, 12 Mar 2025 10:41:34 -0700 Subject: [PATCH 03/20] add yaml file to zip --- .../src/amazonqGumby/chat/controller/controller.ts | 3 ++- .../chat/controller/messenger/messenger.ts | 2 +- packages/core/src/codewhisperer/models/model.ts | 1 + .../service/transformByQ/transformApiHandler.ts | 10 +++++++++- .../service/transformByQ/transformFileHandler.ts | 4 ++-- 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/core/src/amazonqGumby/chat/controller/controller.ts b/packages/core/src/amazonqGumby/chat/controller/controller.ts index 81dbca73029..c919dfecc7d 100644 --- a/packages/core/src/amazonqGumby/chat/controller/controller.ts +++ b/packages/core/src/amazonqGumby/chat/controller/controller.ts @@ -386,6 +386,7 @@ export class GumbyController { break case ButtonActions.CONTINUE_TRANSFORMATION_FORM: this.messenger.sendMessage('Ok, I will continue without this information.', message.tabID, 'ai-prompt') + transformByQState.setCustomDependencyVersionFilePath('') this.promptJavaHome('source', message.tabID) break case ButtonActions.VIEW_TRANSFORMATION_HUB: @@ -453,7 +454,7 @@ export class GumbyController { }) this.messenger.sendOneOrMultipleDiffsMessage(oneOrMultipleDiffsSelection, message.tabID) - this.messenger.sendCustomDependencyVersionSelectionMessage(message.tabID) + await this.messenger.sendCustomDependencyVersionMessage(message.tabID) }) } diff --git a/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts b/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts index d3403478aa4..f1060cec392 100644 --- a/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts +++ b/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts @@ -747,7 +747,7 @@ ${codeSnippet} ) } - public async sendCustomDependencyVersionSelectionMessage(tabID: string) { + public async sendCustomDependencyVersionMessage(tabID: string) { const message = 'You can optionally upload a YAML file to specify which dependency versions to upgrade to.' const buttons: ChatItemButton[] = [] diff --git a/packages/core/src/codewhisperer/models/model.ts b/packages/core/src/codewhisperer/models/model.ts index 3962ae37195..084d685c82a 100644 --- a/packages/core/src/codewhisperer/models/model.ts +++ b/packages/core/src/codewhisperer/models/model.ts @@ -678,6 +678,7 @@ export enum BuildSystem { Unknown = 'Unknown', } +// TO-DO: include the custom YAML file path here somewhere? export class ZipManifest { sourcesRoot: string = 'sources/' dependenciesRoot: string = 'dependencies/' diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts index b4d2eefebfd..1c5852cd310 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts @@ -58,7 +58,7 @@ import { getAuthType } from '../../../auth/utils' import { UserWrittenCodeTracker } from '../../tracker/userWrittenCodeTracker' import { AuthUtil } from '../../util/authUtil' import { DiffModel } from './transformationResultsViewProvider' -import { spawnSync } from 'child_process' +import { spawnSync } from 'child_process' // eslint-disable-line no-restricted-imports export function getSha256(buffer: Buffer) { const hasher = crypto.createHash('sha256') @@ -401,6 +401,14 @@ export async function zipCode( dependenciesCopied = true } + if (transformByQState.getCustomDependencyVersionFilePath() && zipManifest instanceof ZipManifest) { + zip.addLocalFile( + transformByQState.getCustomDependencyVersionFilePath(), + 'custom-upgrades', + 'dependency-versions.yaml' + ) + } + zip.addFile('manifest.json', Buffer.from(JSON.stringify(zipManifest)), 'utf-8') throwIfCancelled() diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts index d55bfb0f931..27ef885898f 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts @@ -48,7 +48,7 @@ export async function loadManifestFile(directory: string) { } export async function copyDirectory(sourcePath: string, destinationPath: string) { - fs.mkdir(destinationPath) + await fs.mkdir(destinationPath) const files = await fs.readdir(sourcePath) for (const file of files) { @@ -78,7 +78,7 @@ export async function createLocalBuildUploadZip(baseDir: string, exitCode: numbe exitCode: exitCode, commandLogFileName: 'clientBuildLogs.log', } - const formattedManifest = JSON.stringify(buildResultsManifest, null, 2) + const formattedManifest = JSON.stringify(buildResultsManifest) await fs.writeFile(manifestFilePath, formattedManifest, 'utf8') const buildLogsFilePath = path.join(baseDir, 'clientBuildLogs.log') From 3e0c19a3ec55578c57ee1d5e693f315fc0010bec Mon Sep 17 00:00:00 2001 From: David Hasani Date: Tue, 18 Mar 2025 15:34:14 -0700 Subject: [PATCH 04/20] add permissions prompt --- .../chat/controller/controller.ts | 15 ++++++-- .../chat/controller/messenger/messenger.ts | 35 ++++++++++++++++++- .../controller/messenger/messengerUtils.ts | 1 + .../src/codewhisperer/models/constants.ts | 6 ++++ .../transformByQ/transformApiHandler.ts | 7 +++- .../transformByQ/transformFileHandler.ts | 13 +++++++ .../transformationHubViewProvider.ts | 2 ++ .../commands/transformByQ.test.ts | 25 +++++++++++++ 8 files changed, 99 insertions(+), 5 deletions(-) diff --git a/packages/core/src/amazonqGumby/chat/controller/controller.ts b/packages/core/src/amazonqGumby/chat/controller/controller.ts index c919dfecc7d..2555af6d991 100644 --- a/packages/core/src/amazonqGumby/chat/controller/controller.ts +++ b/packages/core/src/amazonqGumby/chat/controller/controller.ts @@ -57,8 +57,10 @@ import { openBuildLogFile, parseBuildFile, validateSQLMetadataFile, + validateYamlFile, } from '../../../codewhisperer/service/transformByQ/transformFileHandler' import { getAuthType } from '../../../auth/utils' +import fs from '../../../shared/fs/fs' // These events can be interactions within the chat, // or elsewhere in the IDE @@ -389,6 +391,9 @@ export class GumbyController { transformByQState.setCustomDependencyVersionFilePath('') this.promptJavaHome('source', message.tabID) break + case ButtonActions.AGREE_TO_LOCAL_BUILD: + await this.prepareLanguageUpgradeProject(message) // build project locally right after user agrees to do so + break case ButtonActions.VIEW_TRANSFORMATION_HUB: await vscode.commands.executeCommand(GumbyCommands.FOCUS_TRANSFORMATION_HUB, CancelActionPositions.Chat) break @@ -589,8 +594,12 @@ export class GumbyController { if (!fileUri || fileUri.length === 0) { return } - // TO-DO: validate the YAML file? - this.messenger.sendMessage('Received custom dependency version YAML file!', message.tabID, 'ai-prompt') + const fileContents = await fs.readFileText(fileUri[0].fsPath) + const isValidYaml = await validateYamlFile(fileContents, message) + if (!isValidYaml) { + return + } + this.messenger.sendMessage('Received custom dependency version YAML file.', message.tabID, 'ai-prompt') transformByQState.setCustomDependencyVersionFilePath(fileUri[0].fsPath) this.promptJavaHome('source', message.tabID) } @@ -690,7 +699,7 @@ export class GumbyController { const pathToJavaHome = extractPath(data.message) if (pathToJavaHome) { transformByQState.setTargetJavaHome(pathToJavaHome) - await this.prepareLanguageUpgradeProject(data) // build project locally right after saving target JDK path + this.messenger.sendPermissionToBuildMessage(data.tabID) } else { this.messenger.sendUnrecoverableErrorResponse('invalid-java-home', data.tabID) } diff --git a/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts b/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts index f1060cec392..ac6286f7751 100644 --- a/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts +++ b/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts @@ -50,6 +50,7 @@ export type UnrecoverableErrorType = | 'job-start-failed' | 'unsupported-source-db' | 'unsupported-target-db' + | 'missing-yaml-key' | 'error-parsing-sct-file' | 'invalid-zip-no-sct-file' | 'invalid-from-to-jdk' @@ -345,6 +346,35 @@ export class Messenger { ) } + public sendPermissionToBuildMessage(tabID: string) { + const message = CodeWhispererConstants.buildLocallyChatMessage + + const buttons: ChatItemButton[] = [] + buttons.push({ + keepCardAfterClick: false, + text: 'Agree', + id: ButtonActions.AGREE_TO_LOCAL_BUILD, + position: 'outside', + }) + buttons.push({ + keepCardAfterClick: false, + text: 'No, stop the transformation', + id: ButtonActions.CANCEL_TRANSFORMATION_FORM, + position: 'outside', + }) + + this.dispatcher.sendChatMessage( + new ChatMessage( + { + message, + messageType: 'ai-prompt', + buttons, + }, + tabID + ) + ) + } + public sendAsyncEventProgress( tabID: string, inProgress: boolean, @@ -461,6 +491,9 @@ export class Messenger { case 'unsupported-target-db': message = CodeWhispererConstants.invalidMetadataFileUnsupportedTargetDB break + case 'missing-yaml-key': + message = CodeWhispererConstants.invalidYamlFileMissingKey + break case 'error-parsing-sct-file': message = CodeWhispererConstants.invalidMetadataFileErrorParsing break @@ -786,7 +819,7 @@ dependencyManagement: originType: "FIRST_PARTY" # or "THIRD_PARTY" # Optional - identifier: "com.example:library2" targetVersion: "3.0.0" - origin: "THIRD_PARTY" + originType: "THIRD_PARTY" plugins: - identifier: "com.example.plugin" targetVersion: "1.2.0" diff --git a/packages/core/src/amazonqGumby/chat/controller/messenger/messengerUtils.ts b/packages/core/src/amazonqGumby/chat/controller/messenger/messengerUtils.ts index ad1aade7c7e..58227e0b7af 100644 --- a/packages/core/src/amazonqGumby/chat/controller/messenger/messengerUtils.ts +++ b/packages/core/src/amazonqGumby/chat/controller/messenger/messengerUtils.ts @@ -22,6 +22,7 @@ export enum ButtonActions { SELECT_SQL_CONVERSION_METADATA_FILE = 'gumbySQLConversionMetadataTransformFormConfirm', SELECT_CUSTOM_DEPENDENCY_VERSION_FILE = 'gumbyCustomDependencyVersionTransformFormConfirm', CONTINUE_TRANSFORMATION_FORM = 'gumbyTransformFormContinue', + AGREE_TO_LOCAL_BUILD = 'gumbyAgreeToLocalBuild', CONFIRM_DEPENDENCY_FORM = 'gumbyTransformDependencyFormConfirm', CANCEL_DEPENDENCY_FORM = 'gumbyTransformDependencyFormCancel', CONFIRM_JAVA_HOME_FORM = 'gumbyJavaHomeFormConfirm', diff --git a/packages/core/src/codewhisperer/models/constants.ts b/packages/core/src/codewhisperer/models/constants.ts index 71741c83c22..e7312e12e38 100644 --- a/packages/core/src/codewhisperer/models/constants.ts +++ b/packages/core/src/codewhisperer/models/constants.ts @@ -587,6 +587,9 @@ export const buildSucceededChatMessage = 'I was able to build your project and w export const buildSucceededNotification = 'Amazon Q was able to build your project and will start transforming your code soon.' +export const buildLocallyChatMessage = + 'I will build your project on this machine throughout the transformation process.' + export const absolutePathDetectedMessage = (numPaths: number, buildFile: string, listOfPaths: string) => `I detected ${numPaths} potential absolute file path(s) in your ${buildFile} file: **${listOfPaths}**. Absolute file paths might cause issues when I build your code. Any errors will show up in the build log.` @@ -599,6 +602,9 @@ export const invalidMetadataFileUnsupportedSourceDB = export const invalidMetadataFileUnsupportedTargetDB = 'I can only convert SQL for migrations to Aurora PostgreSQL or Amazon RDS for PostgreSQL target databases. The provided .sct file indicates another target database for this migration.' +export const invalidYamlFileMissingKey = + 'Your YAML file is not formatted correctly. Make sure that the .yaml file you upload follows the format of the sample file provided.' + export const invalidMetadataFileErrorParsing = "It looks like the .sct file you provided isn't valid. Make sure that you've uploaded the .zip file you retrieved from your schema conversion in AWS DMS." diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts index 1c5852cd310..ae1ba1e790d 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts @@ -797,7 +797,12 @@ async function processClientInstructions(jobId: string, clientInstructionsPath: export async function runClientSideBuild(projectPath: string, clientInstructionArtifactId: string) { // baseCommand will be one of: '.\mvnw.cmd', './mvnw', 'mvn' const baseCommand = transformByQState.getMavenName() - const args = ['test'] + const args = ['clean'] + if (transformByQState.getCustomBuildCommand() === CodeWhispererConstants.skipUnitTestsBuildCommand) { + args.push('test-compile') + } else { + args.push('test') + } // TO-DO / QUESTION: why not use the build command from the downloaded manifest? transformByQState.appendToBuildLog(`Running ${baseCommand} ${args}`) const environment = { ...process.env, JAVA_HOME: transformByQState.getTargetJavaHome() } diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts index 27ef885898f..a43188e0873 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts @@ -28,6 +28,7 @@ export function getDependenciesFolderInfo(): FolderInfo { } } +// TO-DO: what happens when intermediate build logs are massive from downloading dependencies? exlude those lines? export async function writeAndShowBuildLogs() { const logFilePath = path.join(os.tmpdir(), 'build-logs.txt') writeFileSync(logFilePath, transformByQState.getBuildLog()) @@ -137,6 +138,18 @@ export async function parseBuildFile() { return undefined } +export async function validateYamlFile(fileContents: string, message: any) { + const requiredKeys = ['dependencyManagement:', 'identifier:', 'targetVersion:'] + for (const key of requiredKeys) { + if (!fileContents.includes(key)) { + getLogger().info(`CodeTransformation: missing yaml key: ${key}`) + transformByQState.getChatMessenger()?.sendUnrecoverableErrorResponse('missing-yaml-key', message.tabID) + return false + } + } + return true +} + export async function validateSQLMetadataFile(fileContents: string, message: any) { try { const sctData = await xml2js.parseStringPromise(fileContents) diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformationHubViewProvider.ts b/packages/core/src/codewhisperer/service/transformByQ/transformationHubViewProvider.ts index 9bbb0dcefa5..6edc2ddd69b 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformationHubViewProvider.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformationHubViewProvider.ts @@ -351,6 +351,8 @@ export class TransformationHubViewProvider implements vscode.WebviewViewProvider CodeWhispererConstants.uploadingCodeStepMessage, activeStepId === 0 ) + // TO-DO: remove this step entirely since we do entirely client-side builds + // TO-DO: do we still show the "Building in Java 17/21 environment" progress update? const buildMarkup = activeStepId >= 1 && transformByQState.getTransformationType() !== TransformationType.SQL_CONVERSION // for SQL conversions, don't show buildCode step ? simpleStep( diff --git a/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts b/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts index 23e29a0fd09..6829a8bb892 100644 --- a/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts +++ b/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts @@ -42,6 +42,7 @@ import { parseBuildFile, validateSQLMetadataFile, createLocalBuildUploadZip, + validateYamlFile, } from '../../../codewhisperer/service/transformByQ/transformFileHandler' import { uploadArtifactToS3 } from '../../../codewhisperer/indexNode' import request from '../../../shared/request' @@ -50,6 +51,19 @@ import * as nodefs from 'fs' // eslint-disable-line no-restricted-imports describe('transformByQ', function () { let fetchStub: sinon.SinonStub let tempDir: string + const validYamlFile = `name: "custom-dependency-management" +description: "Custom dependency version management for Java migration from JDK 8/11/17 to JDK 17/21" +dependencyManagement: + dependencies: + - identifier: "com.example:library1" + targetVersion: "2.1.0" + versionProperty: "library1.version" + originType: "FIRST_PARTY" + plugins: + - identifier: "com.example.plugin" + targetVersion: "1.2.0" + versionProperty: "plugin.version"` + const validSctFile = ` @@ -470,6 +484,17 @@ describe('transformByQ', function () { assert.strictEqual(expectedWarning, warningMessage) }) + it(`WHEN validateYamlFile on fully valid .yaml file THEN passes validation`, async function () { + const isValidYaml = await validateYamlFile(validYamlFile, { tabID: 'abc123' }) + assert.strictEqual(isValidYaml, true) + }) + + it(`WHEN validateYamlFile on invalid .yaml file THEN fails validation`, async function () { + const invalidYamlFile = validYamlFile.replace('dependencyManagement', 'invalidKey') + const isValidYaml = await validateYamlFile(invalidYamlFile, { tabID: 'abc123' }) + assert.strictEqual(isValidYaml, false) + }) + it(`WHEN validateMetadataFile on fully valid .sct file THEN passes validation`, async function () { const isValidMetadata = await validateSQLMetadataFile(validSctFile, { tabID: 'abc123' }) assert.strictEqual(isValidMetadata, true) From 0f965f41fbe25072211e1463fd349818702eedc4 Mon Sep 17 00:00:00 2001 From: David Hasani Date: Mon, 31 Mar 2025 09:27:41 -0700 Subject: [PATCH 05/20] cleanup --- .../service/transformByQ/transformApiHandler.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts index ae1ba1e790d..2093cac356f 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts @@ -260,8 +260,7 @@ export async function uploadPayload( */ const mavenExcludedExtensions = ['.repositories', '.sha1'] -// TO-DO: should we exclude mvnw and mvnw.cmd? -const sourceExcludedExtensions = ['.DS_Store', 'mvnw', 'mvnw.cmd'] +const sourceExcludedExtensions = ['.DS_Store'] /** * Determines if the specified file path corresponds to a Maven metadata file @@ -370,7 +369,6 @@ export async function zipCode( sctFileName: metadataZip.getEntries().filter((entry) => entry.name.endsWith('.sct'))[0].name, }, } - // TO-DO: later consider making this add to path.join(zipManifest.dependenciesRoot, 'qct-sct-metadata', entry.entryName) so that it's more organized for (const entry of metadataZip.getEntries()) { zip.addFile(path.join(zipManifest.dependenciesRoot, entry.name), entry.getData()) } @@ -863,8 +861,6 @@ export function findDownloadArtifactProgressUpdate(transformationSteps: Transfor progressUpdates[j].status === 'AWAITING_CLIENT_ACTION' && progressUpdates[j].downloadArtifacts?.[0]?.downloadArtifactId ) { - // TO-DO: make sure length is always 1 - console.log(`found progress update; length = ${progressUpdates[j].downloadArtifacts?.length}`) return progressUpdates[j] } } From b90859392966baf00c3c766713eb694483b23e3f Mon Sep 17 00:00:00 2001 From: David Hasani Date: Wed, 9 Apr 2025 01:00:24 -0700 Subject: [PATCH 06/20] more refactoring --- .../commands/startTransformByQ.ts | 3 +- .../transformByQ/transformApiHandler.ts | 47 +++++++------- .../transformByQ/transformFileHandler.ts | 63 ++++++------------- .../transformByQ/transformMavenHandler.ts | 21 ++----- .../transformationHubViewProvider.ts | 15 +++-- .../transformationResultsViewProvider.ts | 5 +- .../commands/transformByQ.test.ts | 36 ++++++++++- 7 files changed, 97 insertions(+), 93 deletions(-) diff --git a/packages/core/src/codewhisperer/commands/startTransformByQ.ts b/packages/core/src/codewhisperer/commands/startTransformByQ.ts index f888db4a10a..fe8fd43fab2 100644 --- a/packages/core/src/codewhisperer/commands/startTransformByQ.ts +++ b/packages/core/src/codewhisperer/commands/startTransformByQ.ts @@ -753,7 +753,8 @@ export async function postTransformationJob() { } if (transformByQState.getPayloadFilePath() !== '') { - fs.rmSync(transformByQState.getPayloadFilePath(), { recursive: true, force: true }) // delete ZIP if it exists + // delete original upload ZIP at very end of transformation + fs.rmSync(transformByQState.getPayloadFilePath(), { recursive: true, force: true }) } // attempt download for user diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts index 2093cac356f..a34ef1bccce 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts @@ -42,8 +42,8 @@ import { MetadataResult } from '../../../shared/telemetry/telemetryClient' import request from '../../../shared/request' import { JobStoppedError, ZipExceedsSizeLimitError } from '../../../amazonqGumby/errors' import { - copyDirectory, createLocalBuildUploadZip, + extractOriginalProjectSources, loadManifestFile, writeAndShowBuildLogs, } from './transformFileHandler' @@ -260,7 +260,8 @@ export async function uploadPayload( */ const mavenExcludedExtensions = ['.repositories', '.sha1'] -const sourceExcludedExtensions = ['.DS_Store'] +// exclude .DS_Store (not relevant) and Maven executables (can cause permissions issues when building if user has not ran 'chmod') +const sourceExcludedExtensions = ['.DS_Store', 'mvnw', 'mvnw.cmd'] /** * Determines if the specified file path corresponds to a Maven metadata file @@ -740,8 +741,14 @@ export async function pollTransformationJob(jobId: string, validStates: string[] async function attemptLocalBuild() { const jobId = transformByQState.getJobId() - const artifactId = await getClientInstructionArtifactId(jobId) - getLogger().info(`CodeTransformation: found artifactId = ${artifactId}`) + let artifactId + try { + artifactId = await getClientInstructionArtifactId(jobId) + getLogger().info(`CodeTransformation: found artifactId = ${artifactId}`) + } catch (e: any) { + // don't throw error so that we can try to get progress updates again in next polling cycle + getLogger().error(`CodeTransformation: failed to get client instruction artifact ID = %O`, e) + } if (artifactId) { const clientInstructionsPath = await downloadClientInstructions(jobId, artifactId) getLogger().info( @@ -764,7 +771,7 @@ async function getClientInstructionArtifactId(jobId: string) { async function downloadClientInstructions(jobId: string, artifactId: string) { const exportDestination = `downloadClientInstructions_${jobId}_${artifactId}` - const exportZipPath = path.join(os.tmpdir(), `${exportDestination}.zip`) + const exportZipPath = path.join(os.tmpdir(), exportDestination) const exportContext: ExportContext = { transformationExportContext: { @@ -780,34 +787,30 @@ async function downloadClientInstructions(jobId: string, artifactId: string) { } async function processClientInstructions(jobId: string, clientInstructionsPath: any, artifactId: string) { - const sourcePath = transformByQState.getProjectPath() - const destinationPath = path.join(os.tmpdir(), jobId, artifactId, 'originalCopy') - await copyDirectory(sourcePath, destinationPath) + const destinationPath = path.join(os.tmpdir(), `originalCopy_${jobId}_${artifactId}`) + await extractOriginalProjectSources(destinationPath) getLogger().info(`CodeTransformation: copied project to ${destinationPath}`) const diffModel = new DiffModel() - diffModel.parseDiff(clientInstructionsPath, destinationPath, undefined, 1, true) + diffModel.parseDiff(clientInstructionsPath, path.join(destinationPath, 'sources'), undefined, 1, true) // show user the diff.patch const doc = await vscode.workspace.openTextDocument(clientInstructionsPath) await vscode.window.showTextDocument(doc, { viewColumn: vscode.ViewColumn.One }) await runClientSideBuild(transformByQState.getProjectCopyFilePath(), artifactId) } -export async function runClientSideBuild(projectPath: string, clientInstructionArtifactId: string) { - // baseCommand will be one of: '.\mvnw.cmd', './mvnw', 'mvn' +export async function runClientSideBuild(projectCopyPath: string, clientInstructionArtifactId: string) { const baseCommand = transformByQState.getMavenName() - const args = ['clean'] + const args = [] if (transformByQState.getCustomBuildCommand() === CodeWhispererConstants.skipUnitTestsBuildCommand) { args.push('test-compile') } else { args.push('test') } - // TO-DO / QUESTION: why not use the build command from the downloaded manifest? - transformByQState.appendToBuildLog(`Running ${baseCommand} ${args}`) const environment = { ...process.env, JAVA_HOME: transformByQState.getTargetJavaHome() } const argString = args.join(' ') const spawnResult = spawnSync(baseCommand, args, { - cwd: projectPath, + cwd: projectCopyPath, shell: true, encoding: 'utf-8', env: environment, @@ -818,11 +821,11 @@ export async function runClientSideBuild(projectPath: string, clientInstructionA transformByQState.appendToBuildLog(buildLogs) await writeAndShowBuildLogs() - const baseDir = path.join( + const uploadZipBaseDir = path.join( os.tmpdir(), `clientInstructionsResult_${transformByQState.getJobId()}_${clientInstructionArtifactId}` ) - const zipPath = await createLocalBuildUploadZip(baseDir, spawnResult.status, spawnResult.stdout) + const uploadZipPath = await createLocalBuildUploadZip(uploadZipBaseDir, spawnResult.status, spawnResult.stdout) // upload build results const uploadContext: UploadContext = { @@ -831,14 +834,16 @@ export async function runClientSideBuild(projectPath: string, clientInstructionA uploadArtifactType: 'ClientBuildResult', }, } - getLogger().info(`CodeTransformation: uploading client build results at ${zipPath} and resuming job now`) - await uploadPayload(zipPath, uploadContext) + getLogger().info(`CodeTransformation: uploading client build results at ${uploadZipPath} and resuming job now`) + await uploadPayload(uploadZipPath, AuthUtil.instance.regionProfileManager.activeRegionProfile, uploadContext) await resumeTransformationJob(transformByQState.getJobId(), 'COMPLETED') try { - await fs.delete(transformByQState.getProjectCopyFilePath(), { recursive: true }) + await fs.delete(projectCopyPath, { recursive: true }) + await fs.delete(uploadZipBaseDir, { recursive: true }) + // TODO: do we need to delete the downloaded client instructions and uploadZipPath? they can help in debugging } catch { getLogger().error( - `CodeTransformation: failed to delete project copy at ${transformByQState.getProjectCopyFilePath()} after client-side build` + `CodeTransformation: failed to delete project copy and uploadZipBaseDir after client-side build` ) } } diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts index a43188e0873..ea6baf23dcb 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts @@ -16,7 +16,6 @@ import globals from '../../../shared/extensionGlobals' import { ChatSessionManager } from '../../../amazonqGumby/chat/storages/chatSession' import { AbsolutePathDetectedError } from '../../../amazonqGumby/errors' import { getLogger } from '../../../shared/logger/logger' -import { isWin } from '../../../shared/vscode/env' import AdmZip from 'adm-zip' export function getDependenciesFolderInfo(): FolderInfo { @@ -28,7 +27,6 @@ export function getDependenciesFolderInfo(): FolderInfo { } } -// TO-DO: what happens when intermediate build logs are massive from downloading dependencies? exlude those lines? export async function writeAndShowBuildLogs() { const logFilePath = path.join(os.tmpdir(), 'build-logs.txt') writeFileSync(logFilePath, transformByQState.getBuildLog()) @@ -48,42 +46,18 @@ export async function loadManifestFile(directory: string) { return manifest } -export async function copyDirectory(sourcePath: string, destinationPath: string) { - await fs.mkdir(destinationPath) - const files = await fs.readdir(sourcePath) - - for (const file of files) { - const sourceFilePath = path.join(sourcePath, file[0]) - const destinationFilePath = path.join(destinationPath, file[0]) - if (file[1] === vscode.FileType.Directory) { - // if the item is a directory, recursively copy it - const destinationFilePath = path.join(destinationPath, path.relative(sourcePath, sourceFilePath)) - await copyDirectory(sourceFilePath, destinationFilePath) - } else { - // if the item is a file, copy its contents - try { - await fs.copy(sourceFilePath, destinationFilePath) - } catch (err: any) { - getLogger().error( - `CodeTransformation: error copying file ${sourceFilePath} to ${destinationFilePath}: ${err}` - ) - } - } - } -} - export async function createLocalBuildUploadZip(baseDir: string, exitCode: number | null, stdout: string) { const manifestFilePath = path.join(baseDir, 'manifest.json') const buildResultsManifest = { capability: 'CLIENT_SIDE_BUILD', exitCode: exitCode, - commandLogFileName: 'clientBuildLogs.log', + commandLogFileName: 'build-output.log', } const formattedManifest = JSON.stringify(buildResultsManifest) - await fs.writeFile(manifestFilePath, formattedManifest, 'utf8') + await fs.writeFile(manifestFilePath, formattedManifest) - const buildLogsFilePath = path.join(baseDir, 'clientBuildLogs.log') - await fs.writeFile(buildLogsFilePath, stdout, 'utf8') + const buildLogsFilePath = path.join(baseDir, 'build-output.log') + await fs.writeFile(buildLogsFilePath, stdout) const zip = new AdmZip() zip.addLocalFile(buildLogsFilePath) @@ -95,6 +69,17 @@ export async function createLocalBuildUploadZip(baseDir: string, exitCode: numbe return zipPath } +// extract the 'sources' directory of the upload ZIP so that we can apply the diff.patch to a copy of the source code +export async function extractOriginalProjectSources(destinationPath: string) { + const zip = new AdmZip(transformByQState.getPayloadFilePath()) + const zipEntries = zip.getEntries() + for (const zipEntry of zipEntries) { + if (zipEntry.entryName.startsWith('sources')) { + zip.extractEntryTo(zipEntry, destinationPath, true, true) + } + } +} + export async function checkBuildSystem(projectPath: string) { const mavenBuildFilePath = path.join(projectPath, 'pom.xml') if (existsSync(mavenBuildFilePath)) { @@ -193,20 +178,10 @@ export async function validateSQLMetadataFile(fileContents: string, message: any return true } -export async function setMaven() { - let mavenWrapperExecutableName = isWin() ? 'mvnw.cmd' : 'mvnw' - const mavenWrapperExecutablePath = path.join(transformByQState.getProjectPath(), mavenWrapperExecutableName) - if (existsSync(mavenWrapperExecutablePath)) { - if (mavenWrapperExecutableName === 'mvnw') { - mavenWrapperExecutableName = './mvnw' // add the './' for non-Windows - } else if (mavenWrapperExecutableName === 'mvnw.cmd') { - mavenWrapperExecutableName = '.\\mvnw.cmd' // add the '.\' for Windows - } - transformByQState.setMavenName(mavenWrapperExecutableName) - } else { - transformByQState.setMavenName('mvn') - } - getLogger().info(`CodeTransformation: using Maven ${transformByQState.getMavenName()}`) +export function setMaven() { + // for now, just use regular Maven since the Maven executables can + // cause permissions issues when building if user has not ran 'chmod' + transformByQState.setMavenName('mvn') } export async function openBuildLogFile() { diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts index f5eac536e13..c4004ab26bf 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts @@ -15,15 +15,13 @@ import { setMaven, writeAndShowBuildLogs } from './transformFileHandler' import { throwIfCancelled } from './transformApiHandler' import { sleep } from '../../../shared/utilities/timeoutUtils' -// run 'install' with either 'mvnw.cmd', './mvnw', or 'mvn' (if wrapper exists, we use that, otherwise we use regular 'mvn') function installProjectDependencies(dependenciesFolder: FolderInfo, modulePath: string) { telemetry.codeTransform_localBuildProject.run(() => { telemetry.record({ codeTransformSessionId: CodeTransformTelemetryState.instance.getSessionId() }) - // baseCommand will be one of: '.\mvnw.cmd', './mvnw', 'mvn' + // will always be 'mvn' const baseCommand = transformByQState.getMavenName() - // Note: IntelliJ runs 'clean' separately from 'install'. Evaluate benefits (if any) of this. const args = [`-Dmaven.repo.local=${dependenciesFolder.path}`, 'clean', 'install', '-q'] transformByQState.appendToBuildLog(`Running ${baseCommand} ${args.join(' ')}`) @@ -47,14 +45,7 @@ function installProjectDependencies(dependenciesFolder: FolderInfo, modulePath: maxBuffer: CodeWhispererConstants.maxBufferSize, }) - let mavenBuildCommand = transformByQState.getMavenName() - // slashes not allowed in telemetry - if (mavenBuildCommand === './mvnw') { - mavenBuildCommand = 'mvnw' - } else if (mavenBuildCommand === '.\\mvnw.cmd') { - mavenBuildCommand = 'mvnw.cmd' - } - + const mavenBuildCommand = transformByQState.getMavenName() telemetry.record({ codeTransformBuildCommand: mavenBuildCommand as CodeTransformBuildCommand }) if (spawnResult.status !== 0) { @@ -73,7 +64,6 @@ function installProjectDependencies(dependenciesFolder: FolderInfo, modulePath: } function copyProjectDependencies(dependenciesFolder: FolderInfo, modulePath: string) { - // baseCommand will be one of: '.\mvnw.cmd', './mvnw', 'mvn' const baseCommand = transformByQState.getMavenName() const args = [ @@ -142,7 +132,7 @@ export async function prepareProjectDependencies(dependenciesFolder: FolderInfo, } export async function getVersionData() { - const baseCommand = transformByQState.getMavenName() // will be one of: 'mvnw.cmd', './mvnw', 'mvn' + const baseCommand = transformByQState.getMavenName() const projectPath = transformByQState.getProjectPath() const args = ['-v'] const spawnResult = spawnSync(baseCommand, args, { cwd: projectPath, shell: true, encoding: 'utf-8' }) @@ -172,12 +162,9 @@ export async function getVersionData() { return [localMavenVersion, localJavaVersion] } -// run maven 'versions:dependency-updates-aggregate-report' with either 'mvnw.cmd', './mvnw', or 'mvn' (if wrapper exists, we use that, otherwise we use regular 'mvn') export function runMavenDependencyUpdateCommands(dependenciesFolder: FolderInfo) { - // baseCommand will be one of: '.\mvnw.cmd', './mvnw', 'mvn' - const baseCommand = transformByQState.getMavenName() // will be one of: 'mvnw.cmd', './mvnw', 'mvn' + const baseCommand = transformByQState.getMavenName() - // Note: IntelliJ runs 'clean' separately from 'install'. Evaluate benefits (if any) of this. const args = [ 'versions:dependency-updates-aggregate-report', `-DoutputDirectory=${dependenciesFolder.path}`, diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformationHubViewProvider.ts b/packages/core/src/codewhisperer/service/transformByQ/transformationHubViewProvider.ts index 6edc2ddd69b..bf88a49f680 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformationHubViewProvider.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformationHubViewProvider.ts @@ -328,9 +328,16 @@ export class TransformationHubViewProvider implements vscode.WebviewViewProvider jobPlanProgress['generatePlan'] === StepProgress.Succeeded && transformByQState.isRunning() ) { - const profile = AuthUtil.instance.regionProfileManager.activeRegionProfile - planSteps = await getTransformationSteps(transformByQState.getJobId(), profile) - transformByQState.setPlanSteps(planSteps) + try { + planSteps = await getTransformationSteps(transformByQState.getJobId(), AuthUtil.instance.regionProfileManager.activeRegionProfile) + transformByQState.setPlanSteps(planSteps) + } catch (e: any) { + // no-op; re-use current plan steps and try again in next polling cycle + getLogger().error( + `CodeTransformation: failed to get plan steps to show updates in transformation hub, continuing transformation; error = %O`, + e + ) + } } let progressHtml // for each step that has succeeded, increment activeStepId by 1 @@ -351,8 +358,6 @@ export class TransformationHubViewProvider implements vscode.WebviewViewProvider CodeWhispererConstants.uploadingCodeStepMessage, activeStepId === 0 ) - // TO-DO: remove this step entirely since we do entirely client-side builds - // TO-DO: do we still show the "Building in Java 17/21 environment" progress update? const buildMarkup = activeStepId >= 1 && transformByQState.getTransformationType() !== TransformationType.SQL_CONVERSION // for SQL conversions, don't show buildCode step ? simpleStep( diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts b/packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts index 24353adcadc..411571f0693 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts @@ -181,9 +181,8 @@ export class DiffModel { const changedFiles = parsePatch(diffContents) getLogger().info('CodeTransformation: parsed patch file successfully') - // path to the directory containing copy of the changed files in the transformed project - // if doing intermediate client-side build, pathToWorkspace is the already-copied project directory - // otherwise, we are at the very end of the transformation and need to copy the project to show the changes + // if doing intermediate client-side build, pathToWorkspace is the path to the unzipped project's 'sources' directory (re-using upload ZIP) + // otherwise, we are at the very end of the transformation and need to copy the changed files in the project to show the diff(s) const pathToTmpSrcDir = isIntermediateBuild ? pathToWorkspace : this.copyProject(pathToWorkspace, changedFiles) transformByQState.setProjectCopyFilePath(pathToTmpSrcDir) diff --git a/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts b/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts index 6829a8bb892..fbe549f6136 100644 --- a/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts +++ b/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts @@ -43,6 +43,7 @@ import { validateSQLMetadataFile, createLocalBuildUploadZip, validateYamlFile, + extractOriginalProjectSources, } from '../../../codewhisperer/service/transformByQ/transformFileHandler' import { uploadArtifactToS3 } from '../../../codewhisperer/indexNode' import request from '../../../shared/request' @@ -299,8 +300,9 @@ dependencyManagement: const tempFilePath = path.join(tempDir, tempFileName) await toFile('', tempFilePath) transformByQState.setProjectPath(tempDir) - await setMaven() - assert.strictEqual(transformByQState.getMavenName(), '.\\mvnw.cmd') + setMaven() + // mavenName should always be 'mvn' + assert.strictEqual(transformByQState.getMavenName(), 'mvn') }) it(`WHEN local build zip created THEN zip contains all expected files and no unexpected files`, async function () { @@ -319,6 +321,36 @@ dependencyManagement: assert.strictEqual(zip.getEntries().length, 2) // expecting only manifest.json and clientBuildLogs.log }) + it('WHEN extractOriginalProjectSources THEN only source files are extracted to destination', async function () { + const tempDir = (await TestFolder.create()).path + const destinationPath = path.join(tempDir, 'originalCopy_jobId_artifactId') + await fs.mkdir(destinationPath) + + const zip = new AdmZip() + const testFiles = [ + { path: 'sources/file1.java', content: 'test content 1' }, + { path: 'sources/dir/file2.java', content: 'test content 2' }, + { path: 'dependencies/file3.jar', content: 'should not extract' }, + { path: 'manifest.json', content: '{"version": "1.0"}' }, + ] + + for (const file of testFiles) { + zip.addFile(file.path, Buffer.from(file.content)) + } + + const zipPath = path.join(tempDir, 'test.zip') + zip.writeZip(zipPath) + + transformByQState.setPayloadFilePath(zipPath) + + await extractOriginalProjectSources(destinationPath) + + const extractedFiles = getFilesRecursively(destinationPath, false) + assert.strictEqual(extractedFiles.length, 2) + assert(extractedFiles.includes(path.join(destinationPath, 'sources', 'file1.java'))) + assert(extractedFiles.includes(path.join(destinationPath, 'sources', 'dir', 'file2.java'))) + }) + it(`WHEN zip created THEN manifest.json contains test-compile custom build command`, async function () { const tempFileName = `testfile-${globals.clock.Date.now()}.zip` transformByQState.setProjectPath(tempDir) From 8d3441317b377cef300998048282fe0de5f2827c Mon Sep 17 00:00:00 2001 From: David Hasani Date: Wed, 9 Apr 2025 01:11:20 -0700 Subject: [PATCH 07/20] fix lint --- packages/amazonq/test/e2e/amazonq/transformByQ.test.ts | 2 +- .../codewhisperer/service/transformByQ/transformMavenHandler.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/amazonq/test/e2e/amazonq/transformByQ.test.ts b/packages/amazonq/test/e2e/amazonq/transformByQ.test.ts index 53097b97bf7..df28086a6bd 100644 --- a/packages/amazonq/test/e2e/amazonq/transformByQ.test.ts +++ b/packages/amazonq/test/e2e/amazonq/transformByQ.test.ts @@ -418,7 +418,7 @@ describe('Amazon Q Code Transformation', function () { it('WHEN transforming a Java 8 project E2E THEN job is successful', async function () { transformByQState.setTransformationType(TransformationType.LANGUAGE_UPGRADE) - await setMaven() + setMaven() await startTransformByQ.processLanguageUpgradeTransformFormInput(tempDir, JDKVersion.JDK8, JDKVersion.JDK17) await startTransformByQ.startTransformByQ() assert.strictEqual(transformByQState.getPolledJobStatus(), 'COMPLETED') diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts index c4004ab26bf..ffcf159af98 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts @@ -104,7 +104,7 @@ function copyProjectDependencies(dependenciesFolder: FolderInfo, modulePath: str } export async function prepareProjectDependencies(dependenciesFolder: FolderInfo, rootPomPath: string) { - await setMaven() + setMaven() getLogger().info('CodeTransformation: running Maven copy-dependencies') // pause to give chat time to update await sleep(100) From 8a83dc7381ca4db5d7be92d4954d7eb513e5c43d Mon Sep 17 00:00:00 2001 From: David Hasani Date: Wed, 9 Apr 2025 01:12:36 -0700 Subject: [PATCH 08/20] add comment --- .../codewhisperer/service/transformByQ/transformApiHandler.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts index a34ef1bccce..26d9c7a765c 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts @@ -840,7 +840,8 @@ export async function runClientSideBuild(projectCopyPath: string, clientInstruct try { await fs.delete(projectCopyPath, { recursive: true }) await fs.delete(uploadZipBaseDir, { recursive: true }) - // TODO: do we need to delete the downloaded client instructions and uploadZipPath? they can help in debugging + // TODO: do we need to delete the downloaded client instructions and uploadZipPath? + // Check with AppSec, but they can help in debugging } catch { getLogger().error( `CodeTransformation: failed to delete project copy and uploadZipBaseDir after client-side build` From 3d3e2c211ead7094b18fc9314316653541fbaf9c Mon Sep 17 00:00:00 2001 From: David Hasani Date: Wed, 9 Apr 2025 01:28:23 -0700 Subject: [PATCH 09/20] fix test --- .../core/src/test/codewhisperer/commands/transformByQ.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts b/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts index fbe549f6136..84c09fc487b 100644 --- a/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts +++ b/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts @@ -317,8 +317,8 @@ dependencyManagement: const manifest = JSON.parse(manifestText) assert.strictEqual(manifest.capability, 'CLIENT_SIDE_BUILD') assert.strictEqual(manifest.exitCode, 0) - assert.strictEqual(manifest.commandLogFileName, 'clientBuildLogs.log') - assert.strictEqual(zip.getEntries().length, 2) // expecting only manifest.json and clientBuildLogs.log + assert.strictEqual(manifest.commandLogFileName, 'build-output.log') + assert.strictEqual(zip.getEntries().length, 2) // expecting only manifest.json and build-output.log }) it('WHEN extractOriginalProjectSources THEN only source files are extracted to destination', async function () { From cd2d4ef743b011375ec0e5a69ad2ffb85ae07d81 Mon Sep 17 00:00:00 2001 From: David Hasani Date: Fri, 11 Apr 2025 10:28:42 -0700 Subject: [PATCH 10/20] refactor --- .../transformByQ/transformApiHandler.ts | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts index 26d9c7a765c..0d17a3e2a9a 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts @@ -858,23 +858,16 @@ export function getArtifactsFromProgressUpdate(progressUpdate: TransformationPro } } +// used for client-side build export function findDownloadArtifactProgressUpdate(transformationSteps: TransformationSteps) { - for (let i = 0; i < transformationSteps.length; i++) { - const progressUpdates = transformationSteps[i].progressUpdates - if (progressUpdates) { - for (let j = 0; j < progressUpdates.length; j++) { - if ( - progressUpdates[j].status === 'AWAITING_CLIENT_ACTION' && - progressUpdates[j].downloadArtifacts?.[0]?.downloadArtifactId - ) { - return progressUpdates[j] - } - } - } - } - return undefined + return transformationSteps + .flatMap((step) => step.progressUpdates ?? []) + .find( + (update) => update.status === 'AWAITING_CLIENT_ACTION' && update.downloadArtifacts?.[0]?.downloadArtifactId + ) } +// used for HIL export function findDownloadArtifactStep(transformationSteps: TransformationSteps) { for (let i = 0; i < transformationSteps.length; i++) { const progressUpdates = transformationSteps[i].progressUpdates From 14c8ab5bc8efdb41d8ee06c47cc72f22022a0c7f Mon Sep 17 00:00:00 2001 From: David Hasani Date: Tue, 15 Apr 2025 12:15:50 -0700 Subject: [PATCH 11/20] cleanup --- .../src/amazonqGumby/chat/controller/controller.ts | 10 ++++++++-- .../chat/controller/messenger/messenger.ts | 2 +- .../core/src/codewhisperer/models/constants.ts | 5 +++++ .../service/transformByQ/transformApiHandler.ts | 14 +++++--------- .../service/transformByQ/transformFileHandler.ts | 8 -------- 5 files changed, 19 insertions(+), 20 deletions(-) diff --git a/packages/core/src/amazonqGumby/chat/controller/controller.ts b/packages/core/src/amazonqGumby/chat/controller/controller.ts index 2555af6d991..4f3d34ffb89 100644 --- a/packages/core/src/amazonqGumby/chat/controller/controller.ts +++ b/packages/core/src/amazonqGumby/chat/controller/controller.ts @@ -387,7 +387,11 @@ export class GumbyController { await this.processCustomDependencyVersionFile(message) break case ButtonActions.CONTINUE_TRANSFORMATION_FORM: - this.messenger.sendMessage('Ok, I will continue without this information.', message.tabID, 'ai-prompt') + this.messenger.sendMessage( + CodeWhispererConstants.continueWithoutYamlMessage, + message.tabID, + 'ai-prompt' + ) transformByQState.setCustomDependencyVersionFilePath('') this.promptJavaHome('source', message.tabID) break @@ -459,7 +463,9 @@ export class GumbyController { }) this.messenger.sendOneOrMultipleDiffsMessage(oneOrMultipleDiffsSelection, message.tabID) - await this.messenger.sendCustomDependencyVersionMessage(message.tabID) + this.promptJavaHome('source', message.tabID) + // When custom 1P upgrades ready, delete line above and uncomment line below + // await this.messenger.sendCustomDependencyVersionMessage(message.tabID) }) } diff --git a/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts b/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts index ac6286f7751..ff8a1361139 100644 --- a/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts +++ b/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts @@ -781,7 +781,7 @@ ${codeSnippet} } public async sendCustomDependencyVersionMessage(tabID: string) { - const message = 'You can optionally upload a YAML file to specify which dependency versions to upgrade to.' + const message = CodeWhispererConstants.chooseYamlMessage const buttons: ChatItemButton[] = [] buttons.push({ diff --git a/packages/core/src/codewhisperer/models/constants.ts b/packages/core/src/codewhisperer/models/constants.ts index e7312e12e38..d553f14a236 100644 --- a/packages/core/src/codewhisperer/models/constants.ts +++ b/packages/core/src/codewhisperer/models/constants.ts @@ -663,6 +663,11 @@ export const jobCancelledNotification = 'You cancelled the transformation.' export const continueWithoutHilMessage = 'I will continue transforming your code without upgrading this dependency.' +export const continueWithoutYamlMessage = 'Ok, I will continue without this information.' + +export const chooseYamlMessage = + 'You can optionally upload a YAML file to specify which dependency versions to upgrade to.' + export const enterJavaHomePlaceholder = 'Enter the path to your Java installation' export const openNewTabPlaceholder = 'Open a new tab to chat with Q' diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts index 0d17a3e2a9a..74788e81465 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts @@ -41,12 +41,7 @@ import { calculateTotalLatency } from '../../../amazonqGumby/telemetry/codeTrans import { MetadataResult } from '../../../shared/telemetry/telemetryClient' import request from '../../../shared/request' import { JobStoppedError, ZipExceedsSizeLimitError } from '../../../amazonqGumby/errors' -import { - createLocalBuildUploadZip, - extractOriginalProjectSources, - loadManifestFile, - writeAndShowBuildLogs, -} from './transformFileHandler' +import { createLocalBuildUploadZip, extractOriginalProjectSources, writeAndShowBuildLogs } from './transformFileHandler' import { createCodeWhispererChatStreamingClient } from '../../../shared/clients/codewhispererChatClient' import { downloadExportResultArchive } from '../../../shared/utilities/download' import { ExportContext, ExportIntent, TransformationDownloadArtifactType } from '@amzn/codewhisperer-streaming' @@ -695,6 +690,9 @@ export async function pollTransformationJob(jobId: string, validStates: string[] const errorMessage = response.transformationJob.reason if (errorMessage !== undefined) { + getLogger().error( + `CodeTransformation: GetTransformation returned transformation error reason = ${errorMessage}` + ) transformByQState.setJobFailureErrorChatMessage( `${CodeWhispererConstants.failedToCompleteJobGenericChatMessage} ${errorMessage}` ) @@ -781,9 +779,7 @@ async function downloadClientInstructions(jobId: string, artifactId: string) { } await downloadAndExtractResultArchive(jobId, exportZipPath, exportContext) - - const clientInstructionsManifest = await loadManifestFile(exportZipPath) - return path.join(exportZipPath, clientInstructionsManifest.diffFileName) + return path.join(exportZipPath, 'diff.patch') } async function processClientInstructions(jobId: string, clientInstructionsPath: any, artifactId: string) { diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts index ea6baf23dcb..0512d170304 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts @@ -38,14 +38,6 @@ export async function writeAndShowBuildLogs() { return logFilePath } -export async function loadManifestFile(directory: string) { - const manifestFile = path.join(directory, 'manifest.json') - const data = await fs.readFileText(manifestFile) - const manifest = JSON.parse(data) - getLogger().info(`CodeTransformation: loaded and parsed manifest file from ${manifestFile}`) - return manifest -} - export async function createLocalBuildUploadZip(baseDir: string, exitCode: number | null, stdout: string) { const manifestFilePath = path.join(baseDir, 'manifest.json') const buildResultsManifest = { From ecd0725e69b10a60d99a0a737f524658ffa0de99 Mon Sep 17 00:00:00 2001 From: David Hasani Date: Wed, 16 Apr 2025 11:31:58 -0700 Subject: [PATCH 12/20] add changelog --- .../Feature-73d53c1b-6a88-448a-a4a3-144e99bd95e6.json | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 packages/amazonq/.changes/next-release/Feature-73d53c1b-6a88-448a-a4a3-144e99bd95e6.json diff --git a/packages/amazonq/.changes/next-release/Feature-73d53c1b-6a88-448a-a4a3-144e99bd95e6.json b/packages/amazonq/.changes/next-release/Feature-73d53c1b-6a88-448a-a4a3-144e99bd95e6.json new file mode 100644 index 00000000000..4699cdea38a --- /dev/null +++ b/packages/amazonq/.changes/next-release/Feature-73d53c1b-6a88-448a-a4a3-144e99bd95e6.json @@ -0,0 +1,4 @@ +{ + "type": "Feature", + "description": "/transform: run all builds client-side instead of server-side" +} From 7c9e2473676f8f8327e41ef74b62bb9975c57a66 Mon Sep 17 00:00:00 2001 From: David Hasani Date: Wed, 16 Apr 2025 12:03:11 -0700 Subject: [PATCH 13/20] fix e2e test and prettier --- .../amazonq/test/e2e/amazonq/transformByQ.test.ts | 8 ++++++-- .../service/transformByQ/transformApiHandler.ts | 12 +++++++----- .../transformByQ/transformationHubViewProvider.ts | 5 ++++- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/packages/amazonq/test/e2e/amazonq/transformByQ.test.ts b/packages/amazonq/test/e2e/amazonq/transformByQ.test.ts index df28086a6bd..a22697293c2 100644 --- a/packages/amazonq/test/e2e/amazonq/transformByQ.test.ts +++ b/packages/amazonq/test/e2e/amazonq/transformByQ.test.ts @@ -150,6 +150,8 @@ describe('Amazon Q Code Transformation', function () { waitIntervalInMs: 1000, }) + // Add this back when custom 1P upgrades support is ready + /* const customDependencyVersionPrompt = tab.getChatItems().pop() assert.strictEqual( customDependencyVersionPrompt?.body?.includes('You can optionally upload a YAML file'), @@ -162,13 +164,15 @@ describe('Amazon Q Code Transformation', function () { waitTimeoutInMs: 5000, waitIntervalInMs: 1000, }) + */ const sourceJdkPathPrompt = tab.getChatItems().pop() assert.strictEqual(sourceJdkPathPrompt?.body?.includes('Enter the path to JDK 8'), true) tab.addChatMessage({ prompt: '/dummy/path/to/jdk8' }) + // 2 additional chat messages get sent after JDK path submitted; wait for both of them - await tab.waitForEvent(() => tab.getChatItems().length > 15, { + await tab.waitForEvent(() => tab.getChatItems().length > 13, { waitTimeoutInMs: 5000, waitIntervalInMs: 1000, }) @@ -190,7 +194,7 @@ describe('Amazon Q Code Transformation', function () { text: 'View summary', }) - await tab.waitForEvent(() => tab.getChatItems().length > 16, { + await tab.waitForEvent(() => tab.getChatItems().length > 14, { waitTimeoutInMs: 5000, waitIntervalInMs: 1000, }) diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts index 74788e81465..34e0b4682d8 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts @@ -645,10 +645,7 @@ export async function getTransformationPlan(jobId: string, profile: RegionProfil } } -export async function getTransformationSteps( - jobId: string, - profile: RegionProfile | undefined -) { +export async function getTransformationSteps(jobId: string, profile: RegionProfile | undefined) { try { const response = await codeWhisperer.codeWhispererClient.codeModernizerGetCodeTransformationPlan({ transformationJobId: jobId, @@ -901,7 +898,12 @@ export async function downloadResultArchive(jobId: string, pathToArchive: string exportId: jobId, exportIntent: ExportIntent.TRANSFORMATION, } - await downloadExportResultArchive(cwStreamingClient, args, pathToArchive, AuthUtil.instance.regionProfileManager.activeRegionProfile) + await downloadExportResultArchive( + cwStreamingClient, + args, + pathToArchive, + AuthUtil.instance.regionProfileManager.activeRegionProfile + ) } catch (e: any) { getLogger().error(`CodeTransformation: ExportResultArchive error = %O`, e) throw e diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformationHubViewProvider.ts b/packages/core/src/codewhisperer/service/transformByQ/transformationHubViewProvider.ts index bf88a49f680..052ef53b56c 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformationHubViewProvider.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformationHubViewProvider.ts @@ -329,7 +329,10 @@ export class TransformationHubViewProvider implements vscode.WebviewViewProvider transformByQState.isRunning() ) { try { - planSteps = await getTransformationSteps(transformByQState.getJobId(), AuthUtil.instance.regionProfileManager.activeRegionProfile) + planSteps = await getTransformationSteps( + transformByQState.getJobId(), + AuthUtil.instance.regionProfileManager.activeRegionProfile + ) transformByQState.setPlanSteps(planSteps) } catch (e: any) { // no-op; re-use current plan steps and try again in next polling cycle From 55b7bf073db51a1c7ffe169b6246f0872240d78d Mon Sep 17 00:00:00 2001 From: David Hasani Date: Wed, 16 Apr 2025 12:27:45 -0700 Subject: [PATCH 14/20] remove agree to local build form --- .../chat/controller/controller.ts | 24 +++++---------- .../chat/controller/messenger/messenger.ts | 29 ------------------- .../controller/messenger/messengerUtils.ts | 1 - .../src/codewhisperer/models/constants.ts | 3 -- 4 files changed, 8 insertions(+), 49 deletions(-) diff --git a/packages/core/src/amazonqGumby/chat/controller/controller.ts b/packages/core/src/amazonqGumby/chat/controller/controller.ts index 4f3d34ffb89..238b51aa221 100644 --- a/packages/core/src/amazonqGumby/chat/controller/controller.ts +++ b/packages/core/src/amazonqGumby/chat/controller/controller.ts @@ -395,9 +395,6 @@ export class GumbyController { transformByQState.setCustomDependencyVersionFilePath('') this.promptJavaHome('source', message.tabID) break - case ButtonActions.AGREE_TO_LOCAL_BUILD: - await this.prepareLanguageUpgradeProject(message) // build project locally right after user agrees to do so - break case ButtonActions.VIEW_TRANSFORMATION_HUB: await vscode.commands.executeCommand(GumbyCommands.FOCUS_TRANSFORMATION_HUB, CancelActionPositions.Chat) break @@ -552,25 +549,25 @@ export class GumbyController { }) } - private async prepareLanguageUpgradeProject(message: any) { + private async prepareLanguageUpgradeProject(tabID: string) { // build project locally try { this.sessionStorage.getSession().conversationState = ConversationState.COMPILING - this.messenger.sendCompilationInProgress(message.tabID) + this.messenger.sendCompilationInProgress(tabID) await compileProject() } catch (err: any) { - this.messenger.sendUnrecoverableErrorResponse('could-not-compile-project', message.tabID) + this.messenger.sendUnrecoverableErrorResponse('could-not-compile-project', tabID) // reset state to allow "Start a new transformation" button to work this.sessionStorage.getSession().conversationState = ConversationState.IDLE throw err } - this.messenger.sendCompilationFinished(message.tabID) + this.messenger.sendCompilationFinished(tabID) // since compilation can potentially take a long time, double check auth const authState = await AuthUtil.instance.getChatAuthState() if (authState.amazonQ !== 'connected') { - void this.messenger.sendAuthNeededExceptionMessage(authState, message.tabID) + void this.messenger.sendAuthNeededExceptionMessage(authState, tabID) this.sessionStorage.getSession().isAuthenticating = true return } @@ -578,13 +575,8 @@ export class GumbyController { // give user a non-blocking warning if build file appears to contain absolute paths await parseBuildFile() - this.messenger.sendAsyncEventProgress( - message.tabID, - true, - undefined, - GumbyNamedMessages.JOB_SUBMISSION_STATUS_MESSAGE - ) - this.messenger.sendJobSubmittedMessage(message.tabID) + this.messenger.sendAsyncEventProgress(tabID, true, undefined, GumbyNamedMessages.JOB_SUBMISSION_STATUS_MESSAGE) + this.messenger.sendJobSubmittedMessage(tabID) this.sessionStorage.getSession().conversationState = ConversationState.JOB_SUBMITTED await startTransformByQ() } @@ -705,7 +697,7 @@ export class GumbyController { const pathToJavaHome = extractPath(data.message) if (pathToJavaHome) { transformByQState.setTargetJavaHome(pathToJavaHome) - this.messenger.sendPermissionToBuildMessage(data.tabID) + await this.prepareLanguageUpgradeProject(data.tabID) // build right after we get target JDK path } else { this.messenger.sendUnrecoverableErrorResponse('invalid-java-home', data.tabID) } diff --git a/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts b/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts index ff8a1361139..7e66e311c91 100644 --- a/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts +++ b/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts @@ -346,35 +346,6 @@ export class Messenger { ) } - public sendPermissionToBuildMessage(tabID: string) { - const message = CodeWhispererConstants.buildLocallyChatMessage - - const buttons: ChatItemButton[] = [] - buttons.push({ - keepCardAfterClick: false, - text: 'Agree', - id: ButtonActions.AGREE_TO_LOCAL_BUILD, - position: 'outside', - }) - buttons.push({ - keepCardAfterClick: false, - text: 'No, stop the transformation', - id: ButtonActions.CANCEL_TRANSFORMATION_FORM, - position: 'outside', - }) - - this.dispatcher.sendChatMessage( - new ChatMessage( - { - message, - messageType: 'ai-prompt', - buttons, - }, - tabID - ) - ) - } - public sendAsyncEventProgress( tabID: string, inProgress: boolean, diff --git a/packages/core/src/amazonqGumby/chat/controller/messenger/messengerUtils.ts b/packages/core/src/amazonqGumby/chat/controller/messenger/messengerUtils.ts index 58227e0b7af..ad1aade7c7e 100644 --- a/packages/core/src/amazonqGumby/chat/controller/messenger/messengerUtils.ts +++ b/packages/core/src/amazonqGumby/chat/controller/messenger/messengerUtils.ts @@ -22,7 +22,6 @@ export enum ButtonActions { SELECT_SQL_CONVERSION_METADATA_FILE = 'gumbySQLConversionMetadataTransformFormConfirm', SELECT_CUSTOM_DEPENDENCY_VERSION_FILE = 'gumbyCustomDependencyVersionTransformFormConfirm', CONTINUE_TRANSFORMATION_FORM = 'gumbyTransformFormContinue', - AGREE_TO_LOCAL_BUILD = 'gumbyAgreeToLocalBuild', CONFIRM_DEPENDENCY_FORM = 'gumbyTransformDependencyFormConfirm', CANCEL_DEPENDENCY_FORM = 'gumbyTransformDependencyFormCancel', CONFIRM_JAVA_HOME_FORM = 'gumbyJavaHomeFormConfirm', diff --git a/packages/core/src/codewhisperer/models/constants.ts b/packages/core/src/codewhisperer/models/constants.ts index d553f14a236..9bbd4d8e431 100644 --- a/packages/core/src/codewhisperer/models/constants.ts +++ b/packages/core/src/codewhisperer/models/constants.ts @@ -587,9 +587,6 @@ export const buildSucceededChatMessage = 'I was able to build your project and w export const buildSucceededNotification = 'Amazon Q was able to build your project and will start transforming your code soon.' -export const buildLocallyChatMessage = - 'I will build your project on this machine throughout the transformation process.' - export const absolutePathDetectedMessage = (numPaths: number, buildFile: string, listOfPaths: string) => `I detected ${numPaths} potential absolute file path(s) in your ${buildFile} file: **${listOfPaths}**. Absolute file paths might cause issues when I build your code. Any errors will show up in the build log.` From b09591e640ffb7f052ff8cf1a25f4b0aeb70dad9 Mon Sep 17 00:00:00 2001 From: David Hasani Date: Wed, 16 Apr 2025 15:43:54 -0700 Subject: [PATCH 15/20] delete artifacts --- packages/core/src/codewhisperer/models/constants.ts | 1 + .../service/transformByQ/transformApiHandler.ts | 12 ++++-------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/packages/core/src/codewhisperer/models/constants.ts b/packages/core/src/codewhisperer/models/constants.ts index 9bbd4d8e431..f097b7938dc 100644 --- a/packages/core/src/codewhisperer/models/constants.ts +++ b/packages/core/src/codewhisperer/models/constants.ts @@ -806,6 +806,7 @@ export const skipUnitTestsFormTitle = 'Choose to skip unit tests' export const selectiveTransformationFormTitle = 'Choose how to receive proposed changes' +// TO-DO: get text from Allie for this export const skipUnitTestsFormMessage = 'I will build your project using `mvn clean test` by default. If you would like me to build your project without running unit tests, I will use `mvn clean test-compile`.' diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts index 34e0b4682d8..587f64f5459 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts @@ -828,17 +828,13 @@ export async function runClientSideBuild(projectCopyPath: string, clientInstruct }, } getLogger().info(`CodeTransformation: uploading client build results at ${uploadZipPath} and resuming job now`) - await uploadPayload(uploadZipPath, AuthUtil.instance.regionProfileManager.activeRegionProfile, uploadContext) - await resumeTransformationJob(transformByQState.getJobId(), 'COMPLETED') try { + await uploadPayload(uploadZipPath, AuthUtil.instance.regionProfileManager.activeRegionProfile, uploadContext) + await resumeTransformationJob(transformByQState.getJobId(), 'COMPLETED') + } finally { await fs.delete(projectCopyPath, { recursive: true }) await fs.delete(uploadZipBaseDir, { recursive: true }) - // TODO: do we need to delete the downloaded client instructions and uploadZipPath? - // Check with AppSec, but they can help in debugging - } catch { - getLogger().error( - `CodeTransformation: failed to delete project copy and uploadZipBaseDir after client-side build` - ) + getLogger().info(`CodeTransformation: Just deleted project copy and uploadZipBaseDir after client-side build`) } } From a3374eafd79b007fd524049c3aaac88fce3a2226 Mon Sep 17 00:00:00 2001 From: David Hasani Date: Thu, 17 Apr 2025 14:50:59 -0700 Subject: [PATCH 16/20] fix test --- .../amazonqGumby/transformApiHandler.test.ts | 19 ++++--------------- .../src/codewhisperer/models/constants.ts | 3 +-- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/packages/amazonq/test/unit/amazonqGumby/transformApiHandler.test.ts b/packages/amazonq/test/unit/amazonqGumby/transformApiHandler.test.ts index 4d787cca915..1441171bafc 100644 --- a/packages/amazonq/test/unit/amazonqGumby/transformApiHandler.test.ts +++ b/packages/amazonq/test/unit/amazonqGumby/transformApiHandler.test.ts @@ -131,27 +131,16 @@ describe('Amazon Q Transform - transformApiHandler tests', function () { it('will return undefined if step status is NOT AWAITING_CLIENT_ACTION', function () { const transformationStepsFixture: TransformationStep[] = [ { - id: 'dummy-id', - name: 'Step name', - description: 'Step description', + id: 'random-id', + name: 'not-awaiting-client-action step name', + description: 'not-awaiting-client-action step description', status: 'TRANSFORMING', progressUpdates: [ { - name: 'Progress update name', + name: 'some progress update name', status: 'SOMETHING-BESIDES-AWAITING_CLIENT_ACTION', - description: 'Progress update description', - startTime: new Date(), - endTime: new Date(), - downloadArtifacts: [ - { - downloadArtifactId: 'some-download-artifact-id', - downloadArtifactType: 'some-download-artifact-type', - }, - ], }, ], - startTime: new Date(), - endTime: new Date(), }, ] const progressUpdate = findDownloadArtifactProgressUpdate(transformationStepsFixture) diff --git a/packages/core/src/codewhisperer/models/constants.ts b/packages/core/src/codewhisperer/models/constants.ts index f097b7938dc..57df26ed42c 100644 --- a/packages/core/src/codewhisperer/models/constants.ts +++ b/packages/core/src/codewhisperer/models/constants.ts @@ -806,9 +806,8 @@ export const skipUnitTestsFormTitle = 'Choose to skip unit tests' export const selectiveTransformationFormTitle = 'Choose how to receive proposed changes' -// TO-DO: get text from Allie for this export const skipUnitTestsFormMessage = - 'I will build your project using `mvn clean test` by default. If you would like me to build your project without running unit tests, I will use `mvn clean test-compile`.' + 'I will build generated code in your local environment, not on the server side. For information on how I scan code to reduce security risks associated with building the code in your local environment, see the [Amazon Q Developer documentation](https://docs.aws.amazon.com/amazonq/latest/qdeveloper-ug/code-transformation.html#java-local-builds).\n\nI will build your project using `mvn clean test` by default. If you would like me to build your project without running unit tests, I will use `mvn clean test-compile`.' export const runUnitTestsMessage = 'Run unit tests' From 0a59e8af26e2db7e32dd9718d52adaadb68cf16e Mon Sep 17 00:00:00 2001 From: David Hasani Date: Thu, 24 Apr 2025 13:30:48 -0700 Subject: [PATCH 17/20] final csb commit --- .../test/e2e/amazonq/transformByQ.test.ts | 2 +- .../chat/controller/controller.ts | 20 +++++++++++++------ .../chat/controller/messenger/messenger.ts | 8 ++++---- .../commands/startTransformByQ.ts | 2 +- .../src/codewhisperer/models/constants.ts | 4 ++-- .../core/src/codewhisperer/models/model.ts | 3 ++- .../transformByQ/transformApiHandler.ts | 4 ++++ .../transformByQ/transformFileHandler.ts | 12 +++++------ .../transformByQ/transformMavenHandler.ts | 11 ++-------- packages/core/src/dev/config.ts | 3 +++ .../commands/transformByQ.test.ts | 18 ++++++++--------- 11 files changed, 48 insertions(+), 39 deletions(-) diff --git a/packages/amazonq/test/e2e/amazonq/transformByQ.test.ts b/packages/amazonq/test/e2e/amazonq/transformByQ.test.ts index a22697293c2..74f788732dd 100644 --- a/packages/amazonq/test/e2e/amazonq/transformByQ.test.ts +++ b/packages/amazonq/test/e2e/amazonq/transformByQ.test.ts @@ -150,7 +150,7 @@ describe('Amazon Q Code Transformation', function () { waitIntervalInMs: 1000, }) - // Add this back when custom 1P upgrades support is ready + // TO-DO: add this back when releasing CSB /* const customDependencyVersionPrompt = tab.getChatItems().pop() assert.strictEqual( diff --git a/packages/core/src/amazonqGumby/chat/controller/controller.ts b/packages/core/src/amazonqGumby/chat/controller/controller.ts index 238b51aa221..af3f462bf95 100644 --- a/packages/core/src/amazonqGumby/chat/controller/controller.ts +++ b/packages/core/src/amazonqGumby/chat/controller/controller.ts @@ -57,7 +57,7 @@ import { openBuildLogFile, parseBuildFile, validateSQLMetadataFile, - validateYamlFile, + validateCustomVersionsFile, } from '../../../codewhisperer/service/transformByQ/transformFileHandler' import { getAuthType } from '../../../auth/utils' import fs from '../../../shared/fs/fs' @@ -461,7 +461,7 @@ export class GumbyController { this.messenger.sendOneOrMultipleDiffsMessage(oneOrMultipleDiffsSelection, message.tabID) this.promptJavaHome('source', message.tabID) - // When custom 1P upgrades ready, delete line above and uncomment line below + // TO-DO: delete line above and uncomment line below when releasing CSB // await this.messenger.sendCustomDependencyVersionMessage(message.tabID) }) } @@ -586,15 +586,17 @@ export class GumbyController { canSelectMany: false, openLabel: 'Select', filters: { - '.YAML file': ['yaml'], // Restrict user to only pick a .yaml file + 'YAML file': ['yaml'], // restrict user to only pick a .yaml file }, }) if (!fileUri || fileUri.length === 0) { return } const fileContents = await fs.readFileText(fileUri[0].fsPath) - const isValidYaml = await validateYamlFile(fileContents, message) - if (!isValidYaml) { + const isValidFile = await validateCustomVersionsFile(fileContents) + + if (!isValidFile) { + this.messenger.sendUnrecoverableErrorResponse('invalid-custom-versions-file', message.tabID) return } this.messenger.sendMessage('Received custom dependency version YAML file.', message.tabID, 'ai-prompt') @@ -686,7 +688,13 @@ export class GumbyController { const pathToJavaHome = extractPath(data.message) if (pathToJavaHome) { transformByQState.setSourceJavaHome(pathToJavaHome) - this.promptJavaHome('target', data.tabID) // get target JDK path right after saving source JDK path + // if source and target JDK versions are the same, just re-use the source JAVA_HOME and start the build + if (transformByQState.getTargetJDKVersion() === transformByQState.getSourceJDKVersion()) { + transformByQState.setTargetJavaHome(pathToJavaHome) + await this.prepareLanguageUpgradeProject(data.tabID) + } else { + this.promptJavaHome('target', data.tabID) + } } else { this.messenger.sendUnrecoverableErrorResponse('invalid-java-home', data.tabID) } diff --git a/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts b/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts index 7e66e311c91..30324bab06f 100644 --- a/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts +++ b/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts @@ -50,7 +50,7 @@ export type UnrecoverableErrorType = | 'job-start-failed' | 'unsupported-source-db' | 'unsupported-target-db' - | 'missing-yaml-key' + | 'invalid-custom-versions-file' | 'error-parsing-sct-file' | 'invalid-zip-no-sct-file' | 'invalid-from-to-jdk' @@ -462,8 +462,8 @@ export class Messenger { case 'unsupported-target-db': message = CodeWhispererConstants.invalidMetadataFileUnsupportedTargetDB break - case 'missing-yaml-key': - message = CodeWhispererConstants.invalidYamlFileMissingKey + case 'invalid-custom-versions-file': + message = CodeWhispererConstants.invalidCustomVersionsFileMessage break case 'error-parsing-sct-file': message = CodeWhispererConstants.invalidMetadataFileErrorParsing @@ -647,7 +647,7 @@ ${codeSnippet} this.sendInProgressMessage(tabID, message) } - public sendInProgressMessage(tabID: string, message: string, messageName?: string) { + public sendInProgressMessage(tabID: string, message: string) { this.dispatcher.sendAsyncEventProgress( new AsyncEventProgressMessage(tabID, { inProgress: true, message: undefined }) ) diff --git a/packages/core/src/codewhisperer/commands/startTransformByQ.ts b/packages/core/src/codewhisperer/commands/startTransformByQ.ts index fe8fd43fab2..eb31839686d 100644 --- a/packages/core/src/codewhisperer/commands/startTransformByQ.ts +++ b/packages/core/src/codewhisperer/commands/startTransformByQ.ts @@ -117,7 +117,7 @@ export async function compileProject() { await prepareProjectDependencies(dependenciesFolder, modulePath) } catch (err) { // open build-logs.txt file to show user error logs - await writeAndShowBuildLogs() + await writeAndShowBuildLogs(true) throw err } } diff --git a/packages/core/src/codewhisperer/models/constants.ts b/packages/core/src/codewhisperer/models/constants.ts index 57df26ed42c..274199a6267 100644 --- a/packages/core/src/codewhisperer/models/constants.ts +++ b/packages/core/src/codewhisperer/models/constants.ts @@ -599,8 +599,8 @@ export const invalidMetadataFileUnsupportedSourceDB = export const invalidMetadataFileUnsupportedTargetDB = 'I can only convert SQL for migrations to Aurora PostgreSQL or Amazon RDS for PostgreSQL target databases. The provided .sct file indicates another target database for this migration.' -export const invalidYamlFileMissingKey = - 'Your YAML file is not formatted correctly. Make sure that the .yaml file you upload follows the format of the sample file provided.' +export const invalidCustomVersionsFileMessage = + 'Your .YAML file is not formatted correctly. Make sure that the .YAML file you upload follows the format of the sample file provided.' export const invalidMetadataFileErrorParsing = "It looks like the .sct file you provided isn't valid. Make sure that you've uploaded the .zip file you retrieved from your schema conversion in AWS DMS." diff --git a/packages/core/src/codewhisperer/models/model.ts b/packages/core/src/codewhisperer/models/model.ts index 084d685c82a..28072249371 100644 --- a/packages/core/src/codewhisperer/models/model.ts +++ b/packages/core/src/codewhisperer/models/model.ts @@ -685,7 +685,8 @@ export class ZipManifest { buildLogs: string = 'build-logs.txt' version: string = '1.0' hilCapabilities: string[] = ['HIL_1pDependency_VersionUpgrade'] - transformCapabilities: string[] = ['EXPLAINABILITY_V1', 'CLIENT_SIDE_BUILD'] + // TO-DO: add 'CLIENT_SIDE_BUILD' here when releasing + transformCapabilities: string[] = ['EXPLAINABILITY_V1'] customBuildCommand: string = 'clean test' requestedConversions?: { sqlConversion?: { diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts index 587f64f5459..476123f2d6d 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts @@ -54,6 +54,7 @@ import { UserWrittenCodeTracker } from '../../tracker/userWrittenCodeTracker' import { AuthUtil } from '../../util/authUtil' import { DiffModel } from './transformationResultsViewProvider' import { spawnSync } from 'child_process' // eslint-disable-line no-restricted-imports +import { isClientSideBuildEnabled } from '../../../dev/config' export function getSha256(buffer: Buffer) { const hasher = crypto.createHash('sha256') @@ -395,6 +396,7 @@ export async function zipCode( dependenciesCopied = true } + // TO-DO: decide where exactly to put the YAML file / what to name it if (transformByQState.getCustomDependencyVersionFilePath() && zipManifest instanceof ZipManifest) { zip.addLocalFile( transformByQState.getCustomDependencyVersionFilePath(), @@ -701,7 +703,9 @@ export async function pollTransformationJob(jobId: string, validStates: string[] break } + // TO-DO: remove isClientSideBuildEnabled when releasing CSB if ( + isClientSideBuildEnabled && status === 'TRANSFORMING' && transformByQState.getTransformationType() === TransformationType.LANGUAGE_UPGRADE ) { diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts index 0512d170304..ec004087066 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts @@ -27,13 +27,14 @@ export function getDependenciesFolderInfo(): FolderInfo { } } -export async function writeAndShowBuildLogs() { +export async function writeAndShowBuildLogs(isLocalInstall: boolean = false) { const logFilePath = path.join(os.tmpdir(), 'build-logs.txt') writeFileSync(logFilePath, transformByQState.getBuildLog()) const doc = await vscode.workspace.openTextDocument(logFilePath) if (!transformByQState.getBuildLog().includes('clean install succeeded')) { - // only show the log if the build failed - await vscode.window.showTextDocument(doc, { viewColumn: vscode.ViewColumn.Two }) + // only show the log if the build failed; show it in second column for intermediate builds only + const options = isLocalInstall ? undefined : { viewColumn: vscode.ViewColumn.Two } + await vscode.window.showTextDocument(doc, options) } return logFilePath } @@ -115,12 +116,11 @@ export async function parseBuildFile() { return undefined } -export async function validateYamlFile(fileContents: string, message: any) { +export async function validateCustomVersionsFile(fileContents: string) { const requiredKeys = ['dependencyManagement:', 'identifier:', 'targetVersion:'] for (const key of requiredKeys) { if (!fileContents.includes(key)) { - getLogger().info(`CodeTransformation: missing yaml key: ${key}`) - transformByQState.getChatMessenger()?.sendUnrecoverableErrorResponse('missing-yaml-key', message.tabID) + getLogger().info(`CodeTransformation: .YAML file is missing required key: ${key}`) return false } } diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts index ffcf159af98..ebcbfec8970 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts @@ -11,7 +11,7 @@ import { spawnSync } from 'child_process' // eslint-disable-line no-restricted-i import { CodeTransformBuildCommand, telemetry } from '../../../shared/telemetry/telemetry' import { CodeTransformTelemetryState } from '../../../amazonqGumby/telemetry/codeTransformTelemetryState' import { ToolkitError } from '../../../shared/errors' -import { setMaven, writeAndShowBuildLogs } from './transformFileHandler' +import { setMaven } from './transformFileHandler' import { throwIfCancelled } from './transformApiHandler' import { sleep } from '../../../shared/utilities/timeoutUtils' @@ -58,7 +58,7 @@ function installProjectDependencies(dependenciesFolder: FolderInfo, modulePath: ) throw new ToolkitError(`Maven ${argString} error`, { code: 'MavenExecutionError' }) } else { - transformByQState.appendToBuildLog(`Maven clean install succeeded`) + transformByQState.appendToBuildLog(`mvn clean install succeeded`) } }) } @@ -75,8 +75,6 @@ function copyProjectDependencies(dependenciesFolder: FolderInfo, modulePath: str '-q', ] - transformByQState.appendToBuildLog(`Running ${baseCommand} ${args.join(' ')}`) - let environment = process.env if (transformByQState.getSourceJavaHome()) { environment = { ...process.env, JAVA_HOME: transformByQState.getSourceJavaHome() } @@ -93,13 +91,10 @@ function copyProjectDependencies(dependenciesFolder: FolderInfo, modulePath: str let errorLog = '' errorLog += spawnResult.error ? JSON.stringify(spawnResult.error) : '' errorLog += `${spawnResult.stderr}\n${spawnResult.stdout}` - transformByQState.appendToBuildLog(`${baseCommand} ${args} failed: \n ${errorLog}`) getLogger().info( `CodeTransformation: Maven command ${baseCommand} ${args} failed, but still continuing with transformation: ${errorLog}` ) throw new Error('Maven copy-deps error') - } else { - transformByQState.appendToBuildLog(`Maven dependency:copy-dependencies succeeded`) } } @@ -122,8 +117,6 @@ export async function prepareProjectDependencies(dependenciesFolder: FolderInfo, installProjectDependencies(dependenciesFolder, rootPomPath) } catch (err) { void vscode.window.showErrorMessage(CodeWhispererConstants.cleanInstallErrorNotification) - // open build-logs.txt file to show user error logs - await writeAndShowBuildLogs() throw err } diff --git a/packages/core/src/dev/config.ts b/packages/core/src/dev/config.ts index d5fa49b2426..b4df78f64b0 100644 --- a/packages/core/src/dev/config.ts +++ b/packages/core/src/dev/config.ts @@ -10,3 +10,6 @@ export const betaUrl = { amazonq: '', toolkit: '', } + +// TO-DO: remove when releasing CSB +export const isClientSideBuildEnabled = false diff --git a/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts b/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts index 84c09fc487b..4b478e1876e 100644 --- a/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts +++ b/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts @@ -42,7 +42,7 @@ import { parseBuildFile, validateSQLMetadataFile, createLocalBuildUploadZip, - validateYamlFile, + validateCustomVersionsFile, extractOriginalProjectSources, } from '../../../codewhisperer/service/transformByQ/transformFileHandler' import { uploadArtifactToS3 } from '../../../codewhisperer/indexNode' @@ -52,7 +52,7 @@ import * as nodefs from 'fs' // eslint-disable-line no-restricted-imports describe('transformByQ', function () { let fetchStub: sinon.SinonStub let tempDir: string - const validYamlFile = `name: "custom-dependency-management" + const validCustomVersionsFile = `name: "custom-dependency-management" description: "Custom dependency version management for Java migration from JDK 8/11/17 to JDK 17/21" dependencyManagement: dependencies: @@ -516,15 +516,15 @@ dependencyManagement: assert.strictEqual(expectedWarning, warningMessage) }) - it(`WHEN validateYamlFile on fully valid .yaml file THEN passes validation`, async function () { - const isValidYaml = await validateYamlFile(validYamlFile, { tabID: 'abc123' }) - assert.strictEqual(isValidYaml, true) + it(`WHEN validateCustomVersionsFile on fully valid .yaml file THEN passes validation`, async function () { + const isValidFile = await validateCustomVersionsFile(validCustomVersionsFile) + assert.strictEqual(isValidFile, true) }) - it(`WHEN validateYamlFile on invalid .yaml file THEN fails validation`, async function () { - const invalidYamlFile = validYamlFile.replace('dependencyManagement', 'invalidKey') - const isValidYaml = await validateYamlFile(invalidYamlFile, { tabID: 'abc123' }) - assert.strictEqual(isValidYaml, false) + it(`WHEN validateCustomVersionsFile on invalid .yaml file THEN fails validation`, async function () { + const invalidFile = validCustomVersionsFile.replace('dependencyManagement', 'invalidKey') + const isValidFile = await validateCustomVersionsFile(invalidFile) + assert.strictEqual(isValidFile, false) }) it(`WHEN validateMetadataFile on fully valid .sct file THEN passes validation`, async function () { From 0eb57171f40b5c56a789514fb0a60f393b2a4abe Mon Sep 17 00:00:00 2001 From: David Hasani Date: Thu, 24 Apr 2025 13:32:00 -0700 Subject: [PATCH 18/20] remove changelog --- .../Feature-73d53c1b-6a88-448a-a4a3-144e99bd95e6.json | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 packages/amazonq/.changes/next-release/Feature-73d53c1b-6a88-448a-a4a3-144e99bd95e6.json diff --git a/packages/amazonq/.changes/next-release/Feature-73d53c1b-6a88-448a-a4a3-144e99bd95e6.json b/packages/amazonq/.changes/next-release/Feature-73d53c1b-6a88-448a-a4a3-144e99bd95e6.json deleted file mode 100644 index 4699cdea38a..00000000000 --- a/packages/amazonq/.changes/next-release/Feature-73d53c1b-6a88-448a-a4a3-144e99bd95e6.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "type": "Feature", - "description": "/transform: run all builds client-side instead of server-side" -} From 91abf266b00e8670eed7d2289a589c091782ec40 Mon Sep 17 00:00:00 2001 From: David Hasani Date: Thu, 24 Apr 2025 15:09:34 -0700 Subject: [PATCH 19/20] fix message --- packages/core/src/codewhisperer/models/constants.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/codewhisperer/models/constants.ts b/packages/core/src/codewhisperer/models/constants.ts index 274199a6267..2fb3dd10069 100644 --- a/packages/core/src/codewhisperer/models/constants.ts +++ b/packages/core/src/codewhisperer/models/constants.ts @@ -807,7 +807,7 @@ export const skipUnitTestsFormTitle = 'Choose to skip unit tests' export const selectiveTransformationFormTitle = 'Choose how to receive proposed changes' export const skipUnitTestsFormMessage = - 'I will build generated code in your local environment, not on the server side. For information on how I scan code to reduce security risks associated with building the code in your local environment, see the [Amazon Q Developer documentation](https://docs.aws.amazon.com/amazonq/latest/qdeveloper-ug/code-transformation.html#java-local-builds).\n\nI will build your project using `mvn clean test` by default. If you would like me to build your project without running unit tests, I will use `mvn clean test-compile`.' + 'I will build your project using `mvn clean test` by default. If you would like me to build your project without running unit tests, I will use `mvn clean test-compile`.' export const runUnitTestsMessage = 'Run unit tests' From f35c8d0702dfdb71880cf8b25357f1babd03384a Mon Sep 17 00:00:00 2001 From: David Hasani Date: Mon, 28 Apr 2025 14:25:00 -0700 Subject: [PATCH 20/20] don't show logs for sql conversions --- .../service/transformByQ/transformFileHandler.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts index ec004087066..fd74ca7b147 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts @@ -9,7 +9,7 @@ import * as os from 'os' import xml2js = require('xml2js') import * as CodeWhispererConstants from '../../models/constants' import { existsSync, readFileSync, writeFileSync } from 'fs' // eslint-disable-line no-restricted-imports -import { BuildSystem, DB, FolderInfo, transformByQState } from '../../models/model' +import { BuildSystem, DB, FolderInfo, TransformationType, transformByQState } from '../../models/model' import { IManifestFile } from '../../../amazonqFeatureDev/models' import fs from '../../../shared/fs/fs' import globals from '../../../shared/extensionGlobals' @@ -31,7 +31,10 @@ export async function writeAndShowBuildLogs(isLocalInstall: boolean = false) { const logFilePath = path.join(os.tmpdir(), 'build-logs.txt') writeFileSync(logFilePath, transformByQState.getBuildLog()) const doc = await vscode.workspace.openTextDocument(logFilePath) - if (!transformByQState.getBuildLog().includes('clean install succeeded')) { + if ( + !transformByQState.getBuildLog().includes('clean install succeeded') && + transformByQState.getTransformationType() !== TransformationType.SQL_CONVERSION + ) { // only show the log if the build failed; show it in second column for intermediate builds only const options = isLocalInstall ? undefined : { viewColumn: vscode.ViewColumn.Two } await vscode.window.showTextDocument(doc, options)