diff --git a/packages/amazonq/test/e2e/amazonq/transformByQ.test.ts b/packages/amazonq/test/e2e/amazonq/transformByQ.test.ts index 784aed4884c..74f788732dd 100644 --- a/packages/amazonq/test/e2e/amazonq/transformByQ.test.ts +++ b/packages/amazonq/test/e2e/amazonq/transformByQ.test.ts @@ -75,6 +75,9 @@ describe('Amazon Q Code Transformation', function () { }, ]) + transformByQState.setSourceJDKVersion(JDKVersion.JDK8) + transformByQState.setTargetJDKVersion(JDKVersion.JDK17) + tab.addChatMessage({ command: '/transform' }) // wait for /transform to respond with some intro messages and the first user input form @@ -141,16 +144,34 @@ describe('Amazon Q Code Transformation', function () { formItemValues: oneOrMultipleDiffsFormValues, }) - // 2 additional chat messages (including message with 4th form) get sent after 3rd form submitted; wait for both of them + // 2 additional chat messages get sent after 3rd form submitted; wait for both of them await tab.waitForEvent(() => tab.getChatItems().length > 11, { waitTimeoutInMs: 5000, waitIntervalInMs: 1000, }) - const jdkPathPrompt = tab.getChatItems().pop() - assert.strictEqual(jdkPathPrompt?.body?.includes('Enter the path to JDK'), true) - // 2 additional chat messages get sent after 4th form submitted; wait for both of them + // TO-DO: add this back when releasing CSB + /* + const customDependencyVersionPrompt = tab.getChatItems().pop() + assert.strictEqual( + customDependencyVersionPrompt?.body?.includes('You can optionally upload a YAML file'), + true + ) + tab.clickCustomFormButton({ id: 'gumbyTransformFormContinue' }) + + // 2 additional chat messages get sent after Continue button clicked; wait for both of them + await tab.waitForEvent(() => tab.getChatItems().length > 13, { + waitTimeoutInMs: 5000, + waitIntervalInMs: 1000, + }) + */ + + const sourceJdkPathPrompt = tab.getChatItems().pop() + assert.strictEqual(sourceJdkPathPrompt?.body?.includes('Enter the path to JDK 8'), true) + tab.addChatMessage({ prompt: '/dummy/path/to/jdk8' }) + + // 2 additional chat messages get sent after JDK path submitted; wait for both of them await tab.waitForEvent(() => tab.getChatItems().length > 13, { waitTimeoutInMs: 5000, waitIntervalInMs: 1000, @@ -401,7 +422,7 @@ describe('Amazon Q Code Transformation', function () { it('WHEN transforming a Java 8 project E2E THEN job is successful', async function () { transformByQState.setTransformationType(TransformationType.LANGUAGE_UPGRADE) - await setMaven() + setMaven() await startTransformByQ.processLanguageUpgradeTransformFormInput(tempDir, JDKVersion.JDK8, JDKVersion.JDK17) await startTransformByQ.startTransformByQ() assert.strictEqual(transformByQState.getPolledJobStatus(), 'COMPLETED') diff --git a/packages/amazonq/test/unit/amazonqGumby/transformApiHandler.test.ts b/packages/amazonq/test/unit/amazonqGumby/transformApiHandler.test.ts index 6ee0f05391d..1441171bafc 100644 --- a/packages/amazonq/test/unit/amazonqGumby/transformApiHandler.test.ts +++ b/packages/amazonq/test/unit/amazonqGumby/transformApiHandler.test.ts @@ -6,6 +6,7 @@ import assert from 'assert' import { TransformationProgressUpdate, TransformationStep, + findDownloadArtifactProgressUpdate, findDownloadArtifactStep, getArtifactsFromProgressUpdate, } from 'aws-core-vscode/codewhisperer/node' @@ -95,4 +96,55 @@ describe('Amazon Q Transform - transformApiHandler tests', function () { assert.strictEqual(progressUpdate, undefined) }) }) + + describe('findDownloadArtifactProgressUpdate', function () { + it('will return correct progress update from transformationStep', function () { + const transformationStepsFixture: TransformationStep[] = [ + { + id: 'dummy-id', + name: 'Step name', + description: 'Step description', + status: 'TRANSFORMING', + progressUpdates: [ + { + name: 'Progress update name', + status: 'AWAITING_CLIENT_ACTION', + description: 'Client-side build happening now', + startTime: new Date(), + endTime: new Date(), + downloadArtifacts: [ + { + downloadArtifactId: 'some-download-artifact-id', + downloadArtifactType: 'some-download-artifact-type', + }, + ], + }, + ], + startTime: new Date(), + endTime: new Date(), + }, + ] + const progressUpdate = findDownloadArtifactProgressUpdate(transformationStepsFixture) + assert.strictEqual(progressUpdate, transformationStepsFixture[0].progressUpdates?.[0]) + }) + + it('will return undefined if step status is NOT AWAITING_CLIENT_ACTION', function () { + const transformationStepsFixture: TransformationStep[] = [ + { + id: 'random-id', + name: 'not-awaiting-client-action step name', + description: 'not-awaiting-client-action step description', + status: 'TRANSFORMING', + progressUpdates: [ + { + name: 'some progress update name', + status: 'SOMETHING-BESIDES-AWAITING_CLIENT_ACTION', + }, + ], + }, + ] + const progressUpdate = findDownloadArtifactProgressUpdate(transformationStepsFixture) + assert.strictEqual(progressUpdate, undefined) + }) + }) }) diff --git a/packages/core/src/amazonqGumby/chat/controller/controller.ts b/packages/core/src/amazonqGumby/chat/controller/controller.ts index c1e3bd34ea3..af3f462bf95 100644 --- a/packages/core/src/amazonqGumby/chat/controller/controller.ts +++ b/packages/core/src/amazonqGumby/chat/controller/controller.ts @@ -25,7 +25,6 @@ import { processSQLConversionTransformFormInput, startTransformByQ, stopTransformByQ, - validateCanCompileProject, getValidSQLConversionCandidateProjects, openHilPomFile, } from '../../../codewhisperer/commands/startTransformByQ' @@ -33,7 +32,6 @@ import { JDKVersion, TransformationCandidateProject, transformByQState } from '. import { AbsolutePathDetectedError, AlternateDependencyVersionsNotFoundError, - JavaHomeNotSetError, JobStartError, ModuleUploadError, NoJavaProjectsFoundError, @@ -59,8 +57,10 @@ import { openBuildLogFile, parseBuildFile, validateSQLMetadataFile, + validateCustomVersionsFile, } from '../../../codewhisperer/service/transformByQ/transformFileHandler' import { getAuthType } from '../../../auth/utils' +import fs from '../../../shared/fs/fs' // These events can be interactions within the chat, // or elsewhere in the IDE @@ -243,7 +243,7 @@ export class GumbyController { CodeTransformTelemetryState.instance.setSessionId() this.sessionStorage.getSession().conversationState = ConversationState.WAITING_FOR_TRANSFORMATION_OBJECTIVE - this.messenger.sendStaticTextResponse('choose-transformation-objective', message.tabID) + this.messenger.sendMessage(CodeWhispererConstants.chooseTransformationObjective, message.tabID, 'ai-prompt') this.messenger.sendChatInputEnabled(message.tabID, true) this.messenger.sendUpdatePlaceholder( message.tabID, @@ -299,7 +299,7 @@ export class GumbyController { const validProjects = await this.validateSQLConversionProjects(message) if (validProjects.length > 0) { this.sessionStorage.getSession().updateCandidateProjects(validProjects) - await this.messenger.sendSelectSQLMetadataFileMessage(message.tabID) + this.messenger.sendSelectSQLMetadataFileMessage(message.tabID) } }) .catch((err) => { @@ -383,6 +383,18 @@ export class GumbyController { case ButtonActions.SELECT_SQL_CONVERSION_METADATA_FILE: await this.processMetadataFile(message) break + case ButtonActions.SELECT_CUSTOM_DEPENDENCY_VERSION_FILE: + await this.processCustomDependencyVersionFile(message) + break + case ButtonActions.CONTINUE_TRANSFORMATION_FORM: + this.messenger.sendMessage( + CodeWhispererConstants.continueWithoutYamlMessage, + message.tabID, + 'ai-prompt' + ) + transformByQState.setCustomDependencyVersionFilePath('') + this.promptJavaHome('source', message.tabID) + break case ButtonActions.VIEW_TRANSFORMATION_HUB: await vscode.commands.executeCommand(GumbyCommands.FOCUS_TRANSFORMATION_HUB, CancelActionPositions.Chat) break @@ -403,7 +415,7 @@ export class GumbyController { await this.continueJobWithSelectedDependency(message) break case ButtonActions.CANCEL_DEPENDENCY_FORM: - this.messenger.sendUserPrompt('Cancel', message.tabID) + this.messenger.sendMessage('Cancel', message.tabID, 'prompt') await this.continueTransformationWithoutHIL(message) break case ButtonActions.OPEN_FILE: @@ -448,11 +460,27 @@ export class GumbyController { }) this.messenger.sendOneOrMultipleDiffsMessage(oneOrMultipleDiffsSelection, message.tabID) - // perform local build - await this.validateBuildWithPromptOnError(message) + this.promptJavaHome('source', message.tabID) + // TO-DO: delete line above and uncomment line below when releasing CSB + // await this.messenger.sendCustomDependencyVersionMessage(message.tabID) }) } + private promptJavaHome(type: 'source' | 'target', tabID: any) { + let jdkVersion = undefined + if (type === 'source') { + this.sessionStorage.getSession().conversationState = ConversationState.PROMPT_SOURCE_JAVA_HOME + jdkVersion = transformByQState.getSourceJDKVersion() + } else if (type === 'target') { + this.sessionStorage.getSession().conversationState = ConversationState.PROMPT_TARGET_JAVA_HOME + jdkVersion = transformByQState.getTargetJDKVersion() + } + const message = MessengerUtils.createJavaHomePrompt(jdkVersion) + this.messenger.sendMessage(message, tabID, 'ai-prompt') + this.messenger.sendChatInputEnabled(tabID, true) + this.messenger.sendUpdatePlaceholder(tabID, CodeWhispererConstants.enterJavaHomePlaceholder) + } + private async handleUserLanguageUpgradeProjectChoice(message: any) { await telemetry.codeTransform_submitSelection.run(async () => { const pathToProject: string = message.formSelectedValues['GumbyTransformLanguageUpgradeProjectForm'] @@ -521,32 +549,25 @@ export class GumbyController { }) } - private async prepareLanguageUpgradeProject(message: { pathToJavaHome: string; tabID: string }) { - if (message.pathToJavaHome) { - transformByQState.setJavaHome(message.pathToJavaHome) - getLogger().info( - `CodeTransformation: using JAVA_HOME = ${transformByQState.getJavaHome()} since source JDK does not match Maven JDK` - ) - } - - // Pre-build project locally + private async prepareLanguageUpgradeProject(tabID: string) { + // build project locally try { this.sessionStorage.getSession().conversationState = ConversationState.COMPILING - this.messenger.sendCompilationInProgress(message.tabID) + this.messenger.sendCompilationInProgress(tabID) await compileProject() } catch (err: any) { - this.messenger.sendUnrecoverableErrorResponse('could-not-compile-project', message.tabID) + this.messenger.sendUnrecoverableErrorResponse('could-not-compile-project', tabID) // reset state to allow "Start a new transformation" button to work this.sessionStorage.getSession().conversationState = ConversationState.IDLE throw err } - this.messenger.sendCompilationFinished(message.tabID) + this.messenger.sendCompilationFinished(tabID) // since compilation can potentially take a long time, double check auth const authState = await AuthUtil.instance.getChatAuthState() if (authState.amazonQ !== 'connected') { - void this.messenger.sendAuthNeededExceptionMessage(authState, message.tabID) + void this.messenger.sendAuthNeededExceptionMessage(authState, tabID) this.sessionStorage.getSession().isAuthenticating = true return } @@ -554,33 +575,33 @@ export class GumbyController { // give user a non-blocking warning if build file appears to contain absolute paths await parseBuildFile() - this.messenger.sendAsyncEventProgress( - message.tabID, - true, - undefined, - GumbyNamedMessages.JOB_SUBMISSION_STATUS_MESSAGE - ) - this.messenger.sendJobSubmittedMessage(message.tabID) + this.messenger.sendAsyncEventProgress(tabID, true, undefined, GumbyNamedMessages.JOB_SUBMISSION_STATUS_MESSAGE) + this.messenger.sendJobSubmittedMessage(tabID) this.sessionStorage.getSession().conversationState = ConversationState.JOB_SUBMITTED await startTransformByQ() } - // only for Language Upgrades - private async validateBuildWithPromptOnError(message: any | undefined = undefined): Promise { - try { - // Check Java Home is set (not yet prebuilding) - await validateCanCompileProject() - } catch (err: any) { - if (err instanceof JavaHomeNotSetError) { - this.sessionStorage.getSession().conversationState = ConversationState.PROMPT_JAVA_HOME - this.messenger.sendStaticTextResponse('java-home-not-set', message.tabID) - this.messenger.sendChatInputEnabled(message.tabID, true) - this.messenger.sendUpdatePlaceholder(message.tabID, 'Enter the path to your Java installation.') - } + private async processCustomDependencyVersionFile(message: any) { + const fileUri = await vscode.window.showOpenDialog({ + canSelectMany: false, + openLabel: 'Select', + filters: { + 'YAML file': ['yaml'], // restrict user to only pick a .yaml file + }, + }) + if (!fileUri || fileUri.length === 0) { return } + const fileContents = await fs.readFileText(fileUri[0].fsPath) + const isValidFile = await validateCustomVersionsFile(fileContents) - await this.prepareLanguageUpgradeProject(message) + if (!isValidFile) { + this.messenger.sendUnrecoverableErrorResponse('invalid-custom-versions-file', message.tabID) + return + } + this.messenger.sendMessage('Received custom dependency version YAML file.', message.tabID, 'ai-prompt') + transformByQState.setCustomDependencyVersionFilePath(fileUri[0].fsPath) + this.promptJavaHome('source', message.tabID) } private async processMetadataFile(message: any) { @@ -657,19 +678,34 @@ export class GumbyController { } private async processHumanChatMessage(data: { message: string; tabID: string }) { - this.messenger.sendUserPrompt(data.message, data.tabID) + this.messenger.sendMessage(data.message, data.tabID, 'prompt') this.messenger.sendChatInputEnabled(data.tabID, false) - this.messenger.sendUpdatePlaceholder(data.tabID, 'Open a new tab to chat with Q') + this.messenger.sendUpdatePlaceholder(data.tabID, CodeWhispererConstants.openNewTabPlaceholder) const session = this.sessionStorage.getSession() switch (session.conversationState) { - case ConversationState.PROMPT_JAVA_HOME: { + case ConversationState.PROMPT_SOURCE_JAVA_HOME: { const pathToJavaHome = extractPath(data.message) if (pathToJavaHome) { - await this.prepareLanguageUpgradeProject({ - pathToJavaHome, - tabID: data.tabID, - }) + transformByQState.setSourceJavaHome(pathToJavaHome) + // if source and target JDK versions are the same, just re-use the source JAVA_HOME and start the build + if (transformByQState.getTargetJDKVersion() === transformByQState.getSourceJDKVersion()) { + transformByQState.setTargetJavaHome(pathToJavaHome) + await this.prepareLanguageUpgradeProject(data.tabID) + } else { + this.promptJavaHome('target', data.tabID) + } + } else { + this.messenger.sendUnrecoverableErrorResponse('invalid-java-home', data.tabID) + } + break + } + + case ConversationState.PROMPT_TARGET_JAVA_HOME: { + const pathToJavaHome = extractPath(data.message) + if (pathToJavaHome) { + transformByQState.setTargetJavaHome(pathToJavaHome) + await this.prepareLanguageUpgradeProject(data.tabID) // build right after we get target JDK path } else { this.messenger.sendUnrecoverableErrorResponse('invalid-java-home', data.tabID) } @@ -747,7 +783,7 @@ export class GumbyController { }) } - this.messenger.sendStaticTextResponse('end-HIL-early', message.tabID) + this.messenger.sendMessage(CodeWhispererConstants.continueWithoutHilMessage, message.tabID, 'ai-prompt') } } diff --git a/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts b/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts index 1f80aaf26c6..30324bab06f 100644 --- a/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts +++ b/packages/core/src/amazonqGumby/chat/controller/messenger/messenger.ts @@ -8,6 +8,7 @@ * As much as possible, all strings used in the experience should originate here. */ +import vscode from 'vscode' import { AuthFollowUpType, AuthMessageDataMap } from '../../../../amazonq/auth/model' import { JDKVersion, TransformationCandidateProject, transformByQState } from '../../../../codewhisperer/models/model' import { FeatureAuthState } from '../../../../codewhisperer/util/authUtil' @@ -49,6 +50,7 @@ export type UnrecoverableErrorType = | 'job-start-failed' | 'unsupported-source-db' | 'unsupported-target-db' + | 'invalid-custom-versions-file' | 'error-parsing-sct-file' | 'invalid-zip-no-sct-file' | 'invalid-from-to-jdk' @@ -416,38 +418,12 @@ export class Messenger { this.dispatcher.sendChatMessage(jobSubmittedMessage) } - public sendUserPrompt(prompt: string, tabID: string) { + public sendMessage(prompt: string, tabID: string, type: 'prompt' | 'ai-prompt') { this.dispatcher.sendChatMessage( new ChatMessage( { message: prompt, - messageType: 'prompt', - }, - tabID - ) - ) - } - - public sendStaticTextResponse(messageType: StaticTextResponseType, tabID: string) { - let message = '...' - - switch (messageType) { - case 'java-home-not-set': - message = MessengerUtils.createJavaHomePrompt() - break - case 'end-HIL-early': - message = 'I will continue transforming your code without upgrading this dependency.' - break - case 'choose-transformation-objective': - message = CodeWhispererConstants.chooseTransformationObjective - break - } - - this.dispatcher.sendChatMessage( - new ChatMessage( - { - message, - messageType: 'ai-prompt', + messageType: type, }, tabID ) @@ -486,6 +462,9 @@ export class Messenger { case 'unsupported-target-db': message = CodeWhispererConstants.invalidMetadataFileUnsupportedTargetDB break + case 'invalid-custom-versions-file': + message = CodeWhispererConstants.invalidCustomVersionsFileMessage + break case 'error-parsing-sct-file': message = CodeWhispererConstants.invalidMetadataFileErrorParsing break @@ -668,7 +647,7 @@ ${codeSnippet} this.sendInProgressMessage(tabID, message) } - public sendInProgressMessage(tabID: string, message: string, messageName?: string) { + public sendInProgressMessage(tabID: string, message: string) { this.dispatcher.sendAsyncEventProgress( new AsyncEventProgressMessage(tabID, { inProgress: true, message: undefined }) ) @@ -772,7 +751,56 @@ ${codeSnippet} ) } - public async sendSelectSQLMetadataFileMessage(tabID: string) { + public async sendCustomDependencyVersionMessage(tabID: string) { + const message = CodeWhispererConstants.chooseYamlMessage + const buttons: ChatItemButton[] = [] + + buttons.push({ + keepCardAfterClick: true, + text: 'Select .yaml file', + id: ButtonActions.SELECT_CUSTOM_DEPENDENCY_VERSION_FILE, + disabled: false, + }) + + buttons.push({ + keepCardAfterClick: false, + text: 'Continue without this', + id: ButtonActions.CONTINUE_TRANSFORMATION_FORM, + disabled: false, + }) + + this.dispatcher.sendChatMessage( + new ChatMessage( + { + message, + messageType: 'ai-prompt', + buttons, + }, + tabID + ) + ) + const sampleYAML = `name: "custom-dependency-management" +description: "Custom dependency version management for Java migration from JDK 8/11/17 to JDK 17/21" + +dependencyManagement: + dependencies: + - identifier: "com.example:library1" + targetVersion: "2.1.0" + versionProperty: "library1.version" # Optional + originType: "FIRST_PARTY" # or "THIRD_PARTY" # Optional + - identifier: "com.example:library2" + targetVersion: "3.0.0" + originType: "THIRD_PARTY" + plugins: + - identifier: "com.example.plugin" + targetVersion: "1.2.0" + versionProperty: "plugin.version" # Optional` + + const doc = await vscode.workspace.openTextDocument({ content: sampleYAML, language: 'yaml' }) + await vscode.window.showTextDocument(doc) + } + + public sendSelectSQLMetadataFileMessage(tabID: string) { const message = CodeWhispererConstants.selectSQLMetadataFileHelpMessage const buttons: ChatItemButton[] = [] diff --git a/packages/core/src/amazonqGumby/chat/controller/messenger/messengerUtils.ts b/packages/core/src/amazonqGumby/chat/controller/messenger/messengerUtils.ts index a7275ac98a3..ad1aade7c7e 100644 --- a/packages/core/src/amazonqGumby/chat/controller/messenger/messengerUtils.ts +++ b/packages/core/src/amazonqGumby/chat/controller/messenger/messengerUtils.ts @@ -5,7 +5,7 @@ */ import * as os from 'os' -import { transformByQState, JDKVersion } from '../../../../codewhisperer/models/model' +import { JDKVersion } from '../../../../codewhisperer/models/model' import * as CodeWhispererConstants from '../../../../codewhisperer/models/constants' import DependencyVersions from '../../../models/dependencies' @@ -20,6 +20,8 @@ export enum ButtonActions { CONFIRM_SKIP_TESTS_FORM = 'gumbyTransformSkipTestsFormConfirm', CONFIRM_SELECTIVE_TRANSFORMATION_FORM = 'gumbyTransformOneOrMultipleDiffsFormConfirm', SELECT_SQL_CONVERSION_METADATA_FILE = 'gumbySQLConversionMetadataTransformFormConfirm', + SELECT_CUSTOM_DEPENDENCY_VERSION_FILE = 'gumbyCustomDependencyVersionTransformFormConfirm', + CONTINUE_TRANSFORMATION_FORM = 'gumbyTransformFormContinue', CONFIRM_DEPENDENCY_FORM = 'gumbyTransformDependencyFormConfirm', CANCEL_DEPENDENCY_FORM = 'gumbyTransformDependencyFormCancel', CONFIRM_JAVA_HOME_FORM = 'gumbyJavaHomeFormConfirm', @@ -35,14 +37,11 @@ export enum GumbyCommands { } export default class MessengerUtils { - static createJavaHomePrompt = (): string => { - let javaHomePrompt = `${ - CodeWhispererConstants.enterJavaHomeChatMessage - } ${transformByQState.getSourceJDKVersion()}. \n` + static createJavaHomePrompt = (jdkVersion: JDKVersion | undefined): string => { + let javaHomePrompt = `${CodeWhispererConstants.enterJavaHomeChatMessage} ${jdkVersion}. \n` if (os.platform() === 'win32') { javaHomePrompt += CodeWhispererConstants.windowsJavaHomeHelpChatMessage } else if (os.platform() === 'darwin') { - const jdkVersion = transformByQState.getSourceJDKVersion() if (jdkVersion === JDKVersion.JDK8) { javaHomePrompt += ` ${CodeWhispererConstants.macJavaVersionHomeHelpChatMessage(1.8)}` } else if (jdkVersion === JDKVersion.JDK11) { diff --git a/packages/core/src/amazonqGumby/chat/session/session.ts b/packages/core/src/amazonqGumby/chat/session/session.ts index b0ed125c7d8..f1a69eb60ff 100644 --- a/packages/core/src/amazonqGumby/chat/session/session.ts +++ b/packages/core/src/amazonqGumby/chat/session/session.ts @@ -7,7 +7,8 @@ import { TransformationCandidateProject } from '../../../codewhisperer/models/mo export enum ConversationState { IDLE, - PROMPT_JAVA_HOME, + PROMPT_SOURCE_JAVA_HOME, + PROMPT_TARGET_JAVA_HOME, COMPILING, JOB_SUBMITTED, WAITING_FOR_HIL_INPUT, diff --git a/packages/core/src/codewhisperer/commands/startTransformByQ.ts b/packages/core/src/codewhisperer/commands/startTransformByQ.ts index 37b85be3fad..eb31839686d 100644 --- a/packages/core/src/codewhisperer/commands/startTransformByQ.ts +++ b/packages/core/src/codewhisperer/commands/startTransformByQ.ts @@ -56,7 +56,6 @@ import { submitFeedback } from '../../feedback/vue/submitFeedback' import { placeholder } from '../../shared/vscode/commands2' import { AlternateDependencyVersionsNotFoundError, - JavaHomeNotSetError, JobStartError, ModuleUploadError, PollJobError, @@ -69,8 +68,7 @@ import { getJsonValuesFromManifestFile, highlightPomIssueInProject, parseVersionsListFromPomFile, - setMaven, - writeLogs, + writeAndShowBuildLogs, } from '../service/transformByQ/transformFileHandler' import { sleep } from '../../shared/utilities/timeoutUtils' import DependencyVersions from '../../amazonqGumby/models/dependencies' @@ -111,37 +109,6 @@ export async function processSQLConversionTransformFormInput(pathToProject: stri transformByQState.setTargetJDKVersion(JDKVersion.JDK17) } -async function validateJavaHome(): Promise { - const versionData = await getVersionData() - let javaVersionUsedByMaven = versionData[1] - if (javaVersionUsedByMaven !== undefined) { - javaVersionUsedByMaven = javaVersionUsedByMaven.slice(0, 3) - if (javaVersionUsedByMaven === '1.8') { - javaVersionUsedByMaven = JDKVersion.JDK8 - } else if (javaVersionUsedByMaven === '11.') { - javaVersionUsedByMaven = JDKVersion.JDK11 - } else if (javaVersionUsedByMaven === '17.') { - javaVersionUsedByMaven = JDKVersion.JDK17 - } else if (javaVersionUsedByMaven === '21.') { - javaVersionUsedByMaven = JDKVersion.JDK21 - } - } - if (javaVersionUsedByMaven !== transformByQState.getSourceJDKVersion()) { - // means either javaVersionUsedByMaven is undefined or it does not match the project JDK - return false - } - - return true -} - -export async function validateCanCompileProject() { - await setMaven() - const javaHomeFound = await validateJavaHome() - if (!javaHomeFound) { - throw new JavaHomeNotSetError() - } -} - export async function compileProject() { try { const dependenciesFolder: FolderInfo = getDependenciesFolderInfo() @@ -150,9 +117,7 @@ export async function compileProject() { await prepareProjectDependencies(dependenciesFolder, modulePath) } catch (err) { // open build-logs.txt file to show user error logs - const logFilePath = await writeLogs() - const doc = await vscode.workspace.openTextDocument(logFilePath) - await vscode.window.showTextDocument(doc) + await writeAndShowBuildLogs(true) throw err } } @@ -300,7 +265,7 @@ export async function initiateHumanInTheLoopPrompt(jobId: string) { const profile = AuthUtil.instance.regionProfileManager.activeRegionProfile const humanInTheLoopManager = HumanInTheLoopManager.instance // 1) We need to call GetTransformationPlan to get artifactId - const transformationSteps = await getTransformationSteps(jobId, false, profile) + const transformationSteps = await getTransformationSteps(jobId, profile) const { transformationStep, progressUpdate } = findDownloadArtifactStep(transformationSteps) if (!transformationStep || !progressUpdate) { @@ -566,7 +531,7 @@ export async function pollTransformationStatusUntilPlanReady(jobId: string, prof try { const tempToolkitFolder = await makeTemporaryToolkitFolder() const tempBuildLogsDir = path.join(tempToolkitFolder, 'q-transformation-build-logs') - await downloadAndExtractResultArchive(jobId, undefined, tempBuildLogsDir, 'Logs') + await downloadAndExtractResultArchive(jobId, tempBuildLogsDir) pathToLog = path.join(tempBuildLogsDir, 'buildCommandOutput.log') transformByQState.setPreBuildLogFilePath(pathToLog) } catch (e) { @@ -788,7 +753,8 @@ export async function postTransformationJob() { } if (transformByQState.getPayloadFilePath() !== '') { - fs.rmSync(transformByQState.getPayloadFilePath(), { recursive: true, force: true }) // delete ZIP if it exists + // delete original upload ZIP at very end of transformation + fs.rmSync(transformByQState.getPayloadFilePath(), { recursive: true, force: true }) } // attempt download for user diff --git a/packages/core/src/codewhisperer/models/constants.ts b/packages/core/src/codewhisperer/models/constants.ts index 9d17b166cb8..2fb3dd10069 100644 --- a/packages/core/src/codewhisperer/models/constants.ts +++ b/packages/core/src/codewhisperer/models/constants.ts @@ -599,6 +599,9 @@ export const invalidMetadataFileUnsupportedSourceDB = export const invalidMetadataFileUnsupportedTargetDB = 'I can only convert SQL for migrations to Aurora PostgreSQL or Amazon RDS for PostgreSQL target databases. The provided .sct file indicates another target database for this migration.' +export const invalidCustomVersionsFileMessage = + 'Your .YAML file is not formatted correctly. Make sure that the .YAML file you upload follows the format of the sample file provided.' + export const invalidMetadataFileErrorParsing = "It looks like the .sct file you provided isn't valid. Make sure that you've uploaded the .zip file you retrieved from your schema conversion in AWS DMS." @@ -655,6 +658,17 @@ export const jobCancelledChatMessage = export const jobCancelledNotification = 'You cancelled the transformation.' +export const continueWithoutHilMessage = 'I will continue transforming your code without upgrading this dependency.' + +export const continueWithoutYamlMessage = 'Ok, I will continue without this information.' + +export const chooseYamlMessage = + 'You can optionally upload a YAML file to specify which dependency versions to upgrade to.' + +export const enterJavaHomePlaceholder = 'Enter the path to your Java installation' + +export const openNewTabPlaceholder = 'Open a new tab to chat with Q' + export const diffMessage = (multipleDiffs: boolean) => { return multipleDiffs ? 'You can review the diffs to see my proposed changes and accept or reject them. You will be able to accept changes from one diff at a time. If you reject changes in one diff, you will not be able to view or accept changes in the other diffs.' @@ -756,7 +770,7 @@ export const cleanInstallErrorChatMessage = `Sorry, I couldn\'t run the Maven cl export const cleanInstallErrorNotification = `Amazon Q could not run the Maven clean install command to build your project. For more information, see the [Amazon Q documentation](${codeTransformTroubleshootMvnFailure}).` -export const enterJavaHomeChatMessage = 'Enter the path to JDK ' +export const enterJavaHomeChatMessage = 'Enter the path to JDK' export const projectPromptChatMessage = 'I can upgrade your Java project. To start the transformation, I need some information from you. Choose the project you want to upgrade and the target code version to upgrade to. Then, choose Confirm.' diff --git a/packages/core/src/codewhisperer/models/model.ts b/packages/core/src/codewhisperer/models/model.ts index 99689748320..28072249371 100644 --- a/packages/core/src/codewhisperer/models/model.ts +++ b/packages/core/src/codewhisperer/models/model.ts @@ -678,12 +678,14 @@ export enum BuildSystem { Unknown = 'Unknown', } +// TO-DO: include the custom YAML file path here somewhere? export class ZipManifest { sourcesRoot: string = 'sources/' dependenciesRoot: string = 'dependencies/' buildLogs: string = 'build-logs.txt' version: string = '1.0' hilCapabilities: string[] = ['HIL_1pDependency_VersionUpgrade'] + // TO-DO: add 'CLIENT_SIDE_BUILD' here when releasing transformCapabilities: string[] = ['EXPLAINABILITY_V1'] customBuildCommand: string = 'clean test' requestedConversions?: { @@ -771,6 +773,8 @@ export class TransformByQState { private metadataPathSQL: string = '' + private customVersionPath: string = '' + private linesOfCodeSubmitted: number | undefined = undefined private planFilePath: string = '' @@ -790,11 +794,13 @@ export class TransformByQState { private jobFailureErrorChatMessage: string | undefined = undefined - private errorLog: string = '' + private buildLog: string = '' private mavenName: string = '' - private javaHome: string | undefined = undefined + private sourceJavaHome: string | undefined = undefined + + private targetJavaHome: string | undefined = undefined private chatControllers: ChatControllerEventEmitters | undefined = undefined private chatMessenger: Messenger | undefined = undefined @@ -897,6 +903,10 @@ export class TransformByQState { return this.metadataPathSQL } + public getCustomDependencyVersionFilePath() { + return this.customVersionPath + } + public getStatus() { return this.transformByQState } @@ -937,16 +947,20 @@ export class TransformByQState { return this.jobFailureErrorChatMessage } - public getErrorLog() { - return this.errorLog + public getBuildLog() { + return this.buildLog } public getMavenName() { return this.mavenName } - public getJavaHome() { - return this.javaHome + public getSourceJavaHome() { + return this.sourceJavaHome + } + + public getTargetJavaHome() { + return this.targetJavaHome } public getChatControllers() { @@ -969,8 +983,12 @@ export class TransformByQState { return this.intervalId } - public appendToErrorLog(message: string) { - this.errorLog += `${message}\n\n` + public appendToBuildLog(message: string) { + this.buildLog += `${message}\n\n` + } + + public clearBuildLog() { + this.buildLog = '' } public setToNotStarted() { @@ -1061,6 +1079,10 @@ export class TransformByQState { this.metadataPathSQL = path } + public setCustomDependencyVersionFilePath(path: string) { + this.customVersionPath = path + } + public setPlanFilePath(filePath: string) { this.planFilePath = filePath } @@ -1101,8 +1123,12 @@ export class TransformByQState { this.mavenName = mavenName } - public setJavaHome(javaHome: string) { - this.javaHome = javaHome + public setSourceJavaHome(javaHome: string) { + this.sourceJavaHome = javaHome + } + + public setTargetJavaHome(javaHome: string) { + this.targetJavaHome = javaHome } public setChatControllers(controllers: ChatControllerEventEmitters) { @@ -1144,6 +1170,7 @@ export class TransformByQState { this.jobFailureMetadata = '' this.payloadFilePath = '' this.metadataPathSQL = '' + this.customVersionPath = '' this.sourceJDKVersion = undefined this.targetJDKVersion = undefined this.sourceDB = undefined @@ -1151,7 +1178,7 @@ export class TransformByQState { this.sourceServerName = '' this.schemaOptions.clear() this.schema = '' - this.errorLog = '' + this.buildLog = '' this.customBuildCommand = '' this.intervalId = undefined this.produceMultipleDiffs = false diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts index 97dacc1a664..476123f2d6d 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts @@ -41,10 +41,10 @@ import { calculateTotalLatency } from '../../../amazonqGumby/telemetry/codeTrans import { MetadataResult } from '../../../shared/telemetry/telemetryClient' import request from '../../../shared/request' import { JobStoppedError, ZipExceedsSizeLimitError } from '../../../amazonqGumby/errors' -import { writeLogs } from './transformFileHandler' +import { createLocalBuildUploadZip, extractOriginalProjectSources, writeAndShowBuildLogs } from './transformFileHandler' import { createCodeWhispererChatStreamingClient } from '../../../shared/clients/codewhispererChatClient' import { downloadExportResultArchive } from '../../../shared/utilities/download' -import { ExportIntent, TransformationDownloadArtifactType } from '@amzn/codewhisperer-streaming' +import { ExportContext, ExportIntent, TransformationDownloadArtifactType } from '@amzn/codewhisperer-streaming' import fs from '../../../shared/fs/fs' import { ChatSessionManager } from '../../../amazonqGumby/chat/storages/chatSession' import { encodeHTML } from '../../../shared/utilities/textUtilities' @@ -52,6 +52,9 @@ import { convertToTimeString } from '../../../shared/datetime' import { getAuthType } from '../../../auth/utils' import { UserWrittenCodeTracker } from '../../tracker/userWrittenCodeTracker' import { AuthUtil } from '../../util/authUtil' +import { DiffModel } from './transformationResultsViewProvider' +import { spawnSync } from 'child_process' // eslint-disable-line no-restricted-imports +import { isClientSideBuildEnabled } from '../../../dev/config' export function getSha256(buffer: Buffer) { const hasher = crypto.createHash('sha256') @@ -167,10 +170,10 @@ export async function resumeTransformationJob(jobId: string, userActionStatus: T transformationJobId: jobId, userActionStatus, // can be "COMPLETED" or "REJECTED" }) - if (response) { - // always store request ID, but it will only show up in a notification if an error occurs - return response.transformationStatus - } + getLogger().info( + `CodeTransformation: resumeTransformation API status code = ${response.$response.httpResponse.statusCode}` + ) + return response.transformationStatus } catch (e: any) { const errorMessage = `Resuming the job failed due to: ${(e as Error).message}` getLogger().error(`CodeTransformation: ResumeTransformation error = %O`, e) @@ -219,6 +222,8 @@ export async function uploadPayload( throw new Error(errorMessage) } + getLogger().info('CodeTransformation: created upload URL successfully') + try { await uploadArtifactToS3(payloadFileName, response, sha256, buffer) } catch (e: any) { @@ -251,7 +256,8 @@ export async function uploadPayload( */ const mavenExcludedExtensions = ['.repositories', '.sha1'] -const sourceExcludedExtensions = ['.DS_Store'] +// exclude .DS_Store (not relevant) and Maven executables (can cause permissions issues when building if user has not ran 'chmod') +const sourceExcludedExtensions = ['.DS_Store', 'mvnw', 'mvnw.cmd'] /** * Determines if the specified file path corresponds to a Maven metadata file @@ -360,7 +366,6 @@ export async function zipCode( sctFileName: metadataZip.getEntries().filter((entry) => entry.name.endsWith('.sct'))[0].name, }, } - // TO-DO: later consider making this add to path.join(zipManifest.dependenciesRoot, 'qct-sct-metadata', entry.entryName) so that it's more organized for (const entry of metadataZip.getEntries()) { zip.addFile(path.join(zipManifest.dependenciesRoot, entry.name), entry.getData()) } @@ -391,12 +396,21 @@ export async function zipCode( dependenciesCopied = true } + // TO-DO: decide where exactly to put the YAML file / what to name it + if (transformByQState.getCustomDependencyVersionFilePath() && zipManifest instanceof ZipManifest) { + zip.addLocalFile( + transformByQState.getCustomDependencyVersionFilePath(), + 'custom-upgrades', + 'dependency-versions.yaml' + ) + } + zip.addFile('manifest.json', Buffer.from(JSON.stringify(zipManifest)), 'utf-8') throwIfCancelled() // add text file with logs from mvn clean install and mvn copy-dependencies - logFilePath = await writeLogs() + logFilePath = await writeAndShowBuildLogs() // We don't add build-logs.txt file to the manifest if we are // uploading HIL artifacts if (!humanInTheLoopFlag) { @@ -633,16 +647,8 @@ export async function getTransformationPlan(jobId: string, profile: RegionProfil } } -export async function getTransformationSteps( - jobId: string, - handleThrottleFlag: boolean, - profile: RegionProfile | undefined -) { +export async function getTransformationSteps(jobId: string, profile: RegionProfile | undefined) { try { - // prevent ThrottlingException - if (handleThrottleFlag) { - await sleep(2000) - } const response = await codeWhisperer.codeWhispererClient.codeModernizerGetCodeTransformationPlan({ transformationJobId: jobId, profileArn: profile?.arn, @@ -683,6 +689,9 @@ export async function pollTransformationJob(jobId: string, validStates: string[] const errorMessage = response.transformationJob.reason if (errorMessage !== undefined) { + getLogger().error( + `CodeTransformation: GetTransformation returned transformation error reason = ${errorMessage}` + ) transformByQState.setJobFailureErrorChatMessage( `${CodeWhispererConstants.failedToCompleteJobGenericChatMessage} ${errorMessage}` ) @@ -693,6 +702,17 @@ export async function pollTransformationJob(jobId: string, validStates: string[] if (validStates.includes(status)) { break } + + // TO-DO: remove isClientSideBuildEnabled when releasing CSB + if ( + isClientSideBuildEnabled && + status === 'TRANSFORMING' && + transformByQState.getTransformationType() === TransformationType.LANGUAGE_UPGRADE + ) { + // client-side build is N/A for SQL conversions + await attemptLocalBuild() + } + /** * If we find a paused state, we need the user to take action. We will set the global * state for polling status and early exit. @@ -718,7 +738,111 @@ export async function pollTransformationJob(jobId: string, validStates: string[] return status } -export function getArtifactsFromProgressUpdate(progressUpdate?: TransformationProgressUpdate) { +async function attemptLocalBuild() { + const jobId = transformByQState.getJobId() + let artifactId + try { + artifactId = await getClientInstructionArtifactId(jobId) + getLogger().info(`CodeTransformation: found artifactId = ${artifactId}`) + } catch (e: any) { + // don't throw error so that we can try to get progress updates again in next polling cycle + getLogger().error(`CodeTransformation: failed to get client instruction artifact ID = %O`, e) + } + if (artifactId) { + const clientInstructionsPath = await downloadClientInstructions(jobId, artifactId) + getLogger().info( + `CodeTransformation: downloaded clientInstructions with diff.patch at: ${clientInstructionsPath}` + ) + await processClientInstructions(jobId, clientInstructionsPath, artifactId) + } +} + +async function getClientInstructionArtifactId(jobId: string) { + const steps = await getTransformationSteps(jobId, AuthUtil.instance.regionProfileManager.activeRegionProfile) + const progressUpdate = findDownloadArtifactProgressUpdate(steps) + + let artifactId = undefined + if (progressUpdate?.downloadArtifacts) { + artifactId = progressUpdate.downloadArtifacts[0].downloadArtifactId + } + return artifactId +} + +async function downloadClientInstructions(jobId: string, artifactId: string) { + const exportDestination = `downloadClientInstructions_${jobId}_${artifactId}` + const exportZipPath = path.join(os.tmpdir(), exportDestination) + + const exportContext: ExportContext = { + transformationExportContext: { + downloadArtifactType: TransformationDownloadArtifactType.CLIENT_INSTRUCTIONS, + downloadArtifactId: artifactId, + }, + } + + await downloadAndExtractResultArchive(jobId, exportZipPath, exportContext) + return path.join(exportZipPath, 'diff.patch') +} + +async function processClientInstructions(jobId: string, clientInstructionsPath: any, artifactId: string) { + const destinationPath = path.join(os.tmpdir(), `originalCopy_${jobId}_${artifactId}`) + await extractOriginalProjectSources(destinationPath) + getLogger().info(`CodeTransformation: copied project to ${destinationPath}`) + const diffModel = new DiffModel() + diffModel.parseDiff(clientInstructionsPath, path.join(destinationPath, 'sources'), undefined, 1, true) + // show user the diff.patch + const doc = await vscode.workspace.openTextDocument(clientInstructionsPath) + await vscode.window.showTextDocument(doc, { viewColumn: vscode.ViewColumn.One }) + await runClientSideBuild(transformByQState.getProjectCopyFilePath(), artifactId) +} + +export async function runClientSideBuild(projectCopyPath: string, clientInstructionArtifactId: string) { + const baseCommand = transformByQState.getMavenName() + const args = [] + if (transformByQState.getCustomBuildCommand() === CodeWhispererConstants.skipUnitTestsBuildCommand) { + args.push('test-compile') + } else { + args.push('test') + } + const environment = { ...process.env, JAVA_HOME: transformByQState.getTargetJavaHome() } + + const argString = args.join(' ') + const spawnResult = spawnSync(baseCommand, args, { + cwd: projectCopyPath, + shell: true, + encoding: 'utf-8', + env: environment, + }) + + const buildLogs = `Intermediate build result from running ${baseCommand} ${argString}:\n\n${spawnResult.stdout}` + transformByQState.clearBuildLog() + transformByQState.appendToBuildLog(buildLogs) + await writeAndShowBuildLogs() + + const uploadZipBaseDir = path.join( + os.tmpdir(), + `clientInstructionsResult_${transformByQState.getJobId()}_${clientInstructionArtifactId}` + ) + const uploadZipPath = await createLocalBuildUploadZip(uploadZipBaseDir, spawnResult.status, spawnResult.stdout) + + // upload build results + const uploadContext: UploadContext = { + transformationUploadContext: { + jobId: transformByQState.getJobId(), + uploadArtifactType: 'ClientBuildResult', + }, + } + getLogger().info(`CodeTransformation: uploading client build results at ${uploadZipPath} and resuming job now`) + try { + await uploadPayload(uploadZipPath, AuthUtil.instance.regionProfileManager.activeRegionProfile, uploadContext) + await resumeTransformationJob(transformByQState.getJobId(), 'COMPLETED') + } finally { + await fs.delete(projectCopyPath, { recursive: true }) + await fs.delete(uploadZipBaseDir, { recursive: true }) + getLogger().info(`CodeTransformation: Just deleted project copy and uploadZipBaseDir after client-side build`) + } +} + +export function getArtifactsFromProgressUpdate(progressUpdate: TransformationProgressUpdate) { const artifactType = progressUpdate?.downloadArtifacts?.[0]?.downloadArtifactType const artifactId = progressUpdate?.downloadArtifacts?.[0]?.downloadArtifactId return { @@ -727,6 +851,16 @@ export function getArtifactsFromProgressUpdate(progressUpdate?: TransformationPr } } +// used for client-side build +export function findDownloadArtifactProgressUpdate(transformationSteps: TransformationSteps) { + return transformationSteps + .flatMap((step) => step.progressUpdates ?? []) + .find( + (update) => update.status === 'AWAITING_CLIENT_ACTION' && update.downloadArtifacts?.[0]?.downloadArtifactId + ) +} + +// used for HIL export function findDownloadArtifactStep(transformationSteps: TransformationSteps) { for (let i = 0; i < transformationSteps.length; i++) { const progressUpdates = transformationSteps[i].progressUpdates @@ -750,21 +884,23 @@ export function findDownloadArtifactStep(transformationSteps: TransformationStep } } -export async function downloadResultArchive( - jobId: string, - downloadArtifactId: string | undefined, - pathToArchive: string, - downloadArtifactType: TransformationDownloadArtifactType -) { +export async function downloadResultArchive(jobId: string, pathToArchive: string, exportContext?: ExportContext) { const cwStreamingClient = await createCodeWhispererChatStreamingClient() try { + const args = exportContext + ? { + exportId: jobId, + exportIntent: ExportIntent.TRANSFORMATION, + exportContext: exportContext, + } + : { + exportId: jobId, + exportIntent: ExportIntent.TRANSFORMATION, + } await downloadExportResultArchive( cwStreamingClient, - { - exportId: jobId, - exportIntent: ExportIntent.TRANSFORMATION, - }, + args, pathToArchive, AuthUtil.instance.regionProfileManager.activeRegionProfile ) @@ -779,9 +915,8 @@ export async function downloadResultArchive( export async function downloadAndExtractResultArchive( jobId: string, - downloadArtifactId: string | undefined, pathToArchiveDir: string, - downloadArtifactType: TransformationDownloadArtifactType + exportContext?: ExportContext ) { const archivePathExists = await fs.existsDir(pathToArchiveDir) if (!archivePathExists) { @@ -793,9 +928,10 @@ export async function downloadAndExtractResultArchive( let downloadErrorMessage = undefined try { // Download and deserialize the zip - await downloadResultArchive(jobId, downloadArtifactId, pathToArchive, downloadArtifactType) + await downloadResultArchive(jobId, pathToArchive, exportContext) const zip = new AdmZip(pathToArchive) zip.extractAllTo(pathToArchiveDir) + getLogger().info(`CodeTransformation: downloaded result archive to: ${pathToArchiveDir}`) } catch (e) { downloadErrorMessage = (e as Error).message getLogger().error(`CodeTransformation: ExportResultArchive error = %O`, e) @@ -804,12 +940,7 @@ export async function downloadAndExtractResultArchive( } export async function downloadHilResultArchive(jobId: string, downloadArtifactId: string, pathToArchiveDir: string) { - await downloadAndExtractResultArchive( - jobId, - downloadArtifactId, - pathToArchiveDir, - TransformationDownloadArtifactType.CLIENT_INSTRUCTIONS - ) + await downloadAndExtractResultArchive(jobId, pathToArchiveDir) // manifest.json // pomFolder/pom.xml or manifest has pomFolderName path diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts index c2a0617c15f..fd74ca7b147 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts @@ -9,14 +9,14 @@ import * as os from 'os' import xml2js = require('xml2js') import * as CodeWhispererConstants from '../../models/constants' import { existsSync, readFileSync, writeFileSync } from 'fs' // eslint-disable-line no-restricted-imports -import { BuildSystem, DB, FolderInfo, transformByQState } from '../../models/model' +import { BuildSystem, DB, FolderInfo, TransformationType, transformByQState } from '../../models/model' import { IManifestFile } from '../../../amazonqFeatureDev/models' import fs from '../../../shared/fs/fs' import globals from '../../../shared/extensionGlobals' import { ChatSessionManager } from '../../../amazonqGumby/chat/storages/chatSession' import { AbsolutePathDetectedError } from '../../../amazonqGumby/errors' import { getLogger } from '../../../shared/logger/logger' -import { isWin } from '../../../shared/vscode/env' +import AdmZip from 'adm-zip' export function getDependenciesFolderInfo(): FolderInfo { const dependencyFolderName = `${CodeWhispererConstants.dependencyFolderName}${globals.clock.Date.now()}` @@ -27,12 +27,55 @@ export function getDependenciesFolderInfo(): FolderInfo { } } -export async function writeLogs() { +export async function writeAndShowBuildLogs(isLocalInstall: boolean = false) { const logFilePath = path.join(os.tmpdir(), 'build-logs.txt') - writeFileSync(logFilePath, transformByQState.getErrorLog()) + writeFileSync(logFilePath, transformByQState.getBuildLog()) + const doc = await vscode.workspace.openTextDocument(logFilePath) + if ( + !transformByQState.getBuildLog().includes('clean install succeeded') && + transformByQState.getTransformationType() !== TransformationType.SQL_CONVERSION + ) { + // only show the log if the build failed; show it in second column for intermediate builds only + const options = isLocalInstall ? undefined : { viewColumn: vscode.ViewColumn.Two } + await vscode.window.showTextDocument(doc, options) + } return logFilePath } +export async function createLocalBuildUploadZip(baseDir: string, exitCode: number | null, stdout: string) { + const manifestFilePath = path.join(baseDir, 'manifest.json') + const buildResultsManifest = { + capability: 'CLIENT_SIDE_BUILD', + exitCode: exitCode, + commandLogFileName: 'build-output.log', + } + const formattedManifest = JSON.stringify(buildResultsManifest) + await fs.writeFile(manifestFilePath, formattedManifest) + + const buildLogsFilePath = path.join(baseDir, 'build-output.log') + await fs.writeFile(buildLogsFilePath, stdout) + + const zip = new AdmZip() + zip.addLocalFile(buildLogsFilePath) + zip.addLocalFile(manifestFilePath) + + const zipPath = `${baseDir}.zip` + zip.writeZip(zipPath) + getLogger().info(`CodeTransformation: created local build upload zip at ${zipPath}`) + return zipPath +} + +// extract the 'sources' directory of the upload ZIP so that we can apply the diff.patch to a copy of the source code +export async function extractOriginalProjectSources(destinationPath: string) { + const zip = new AdmZip(transformByQState.getPayloadFilePath()) + const zipEntries = zip.getEntries() + for (const zipEntry of zipEntries) { + if (zipEntry.entryName.startsWith('sources')) { + zip.extractEntryTo(zipEntry, destinationPath, true, true) + } + } +} + export async function checkBuildSystem(projectPath: string) { const mavenBuildFilePath = path.join(projectPath, 'pom.xml') if (existsSync(mavenBuildFilePath)) { @@ -76,6 +119,17 @@ export async function parseBuildFile() { return undefined } +export async function validateCustomVersionsFile(fileContents: string) { + const requiredKeys = ['dependencyManagement:', 'identifier:', 'targetVersion:'] + for (const key of requiredKeys) { + if (!fileContents.includes(key)) { + getLogger().info(`CodeTransformation: .YAML file is missing required key: ${key}`) + return false + } + } + return true +} + export async function validateSQLMetadataFile(fileContents: string, message: any) { try { const sctData = await xml2js.parseStringPromise(fileContents) @@ -119,20 +173,10 @@ export async function validateSQLMetadataFile(fileContents: string, message: any return true } -export async function setMaven() { - let mavenWrapperExecutableName = isWin() ? 'mvnw.cmd' : 'mvnw' - const mavenWrapperExecutablePath = path.join(transformByQState.getProjectPath(), mavenWrapperExecutableName) - if (existsSync(mavenWrapperExecutablePath)) { - if (mavenWrapperExecutableName === 'mvnw') { - mavenWrapperExecutableName = './mvnw' // add the './' for non-Windows - } else if (mavenWrapperExecutableName === 'mvnw.cmd') { - mavenWrapperExecutableName = '.\\mvnw.cmd' // add the '.\' for Windows - } - transformByQState.setMavenName(mavenWrapperExecutableName) - } else { - transformByQState.setMavenName('mvn') - } - getLogger().info(`CodeTransformation: using Maven ${transformByQState.getMavenName()}`) +export function setMaven() { + // for now, just use regular Maven since the Maven executables can + // cause permissions issues when building if user has not ran 'chmod' + transformByQState.setMavenName('mvn') } export async function openBuildLogFile() { diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts index dacd78f6dc3..ebcbfec8970 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts @@ -11,31 +11,29 @@ import { spawnSync } from 'child_process' // eslint-disable-line no-restricted-i import { CodeTransformBuildCommand, telemetry } from '../../../shared/telemetry/telemetry' import { CodeTransformTelemetryState } from '../../../amazonqGumby/telemetry/codeTransformTelemetryState' import { ToolkitError } from '../../../shared/errors' -import { setMaven, writeLogs } from './transformFileHandler' +import { setMaven } from './transformFileHandler' import { throwIfCancelled } from './transformApiHandler' import { sleep } from '../../../shared/utilities/timeoutUtils' -// run 'install' with either 'mvnw.cmd', './mvnw', or 'mvn' (if wrapper exists, we use that, otherwise we use regular 'mvn') function installProjectDependencies(dependenciesFolder: FolderInfo, modulePath: string) { telemetry.codeTransform_localBuildProject.run(() => { telemetry.record({ codeTransformSessionId: CodeTransformTelemetryState.instance.getSessionId() }) - // baseCommand will be one of: '.\mvnw.cmd', './mvnw', 'mvn' + // will always be 'mvn' const baseCommand = transformByQState.getMavenName() - transformByQState.appendToErrorLog(`Running command ${baseCommand} clean install`) - - // Note: IntelliJ runs 'clean' separately from 'install'. Evaluate benefits (if any) of this. const args = [`-Dmaven.repo.local=${dependenciesFolder.path}`, 'clean', 'install', '-q'] + transformByQState.appendToBuildLog(`Running ${baseCommand} ${args.join(' ')}`) + if (transformByQState.getCustomBuildCommand() === CodeWhispererConstants.skipUnitTestsBuildCommand) { args.push('-DskipTests') } let environment = process.env - if (transformByQState.getJavaHome() !== undefined) { - environment = { ...process.env, JAVA_HOME: transformByQState.getJavaHome() } + if (transformByQState.getSourceJavaHome()) { + environment = { ...process.env, JAVA_HOME: transformByQState.getSourceJavaHome() } } const argString = args.join(' ') @@ -47,37 +45,27 @@ function installProjectDependencies(dependenciesFolder: FolderInfo, modulePath: maxBuffer: CodeWhispererConstants.maxBufferSize, }) - let mavenBuildCommand = transformByQState.getMavenName() - // slashes not allowed in telemetry - if (mavenBuildCommand === './mvnw') { - mavenBuildCommand = 'mvnw' - } else if (mavenBuildCommand === '.\\mvnw.cmd') { - mavenBuildCommand = 'mvnw.cmd' - } - + const mavenBuildCommand = transformByQState.getMavenName() telemetry.record({ codeTransformBuildCommand: mavenBuildCommand as CodeTransformBuildCommand }) if (spawnResult.status !== 0) { let errorLog = '' errorLog += spawnResult.error ? JSON.stringify(spawnResult.error) : '' errorLog += `${spawnResult.stderr}\n${spawnResult.stdout}` - transformByQState.appendToErrorLog(`${baseCommand} ${argString} failed: \n ${errorLog}`) + transformByQState.appendToBuildLog(`${baseCommand} ${argString} failed: \n ${errorLog}`) getLogger().error( - `CodeTransformation: Error in running Maven ${argString} command ${baseCommand} = ${errorLog}` + `CodeTransformation: Error in running Maven command ${baseCommand} ${argString} = ${errorLog}` ) throw new ToolkitError(`Maven ${argString} error`, { code: 'MavenExecutionError' }) } else { - transformByQState.appendToErrorLog(`${baseCommand} ${argString} succeeded`) + transformByQState.appendToBuildLog(`mvn clean install succeeded`) } }) } function copyProjectDependencies(dependenciesFolder: FolderInfo, modulePath: string) { - // baseCommand will be one of: '.\mvnw.cmd', './mvnw', 'mvn' const baseCommand = transformByQState.getMavenName() - transformByQState.appendToErrorLog(`Running command ${baseCommand} copy-dependencies`) - const args = [ 'dependency:copy-dependencies', `-DoutputDirectory=${dependenciesFolder.path}`, @@ -88,8 +76,8 @@ function copyProjectDependencies(dependenciesFolder: FolderInfo, modulePath: str ] let environment = process.env - if (transformByQState.getJavaHome() !== undefined) { - environment = { ...process.env, JAVA_HOME: transformByQState.getJavaHome() } + if (transformByQState.getSourceJavaHome()) { + environment = { ...process.env, JAVA_HOME: transformByQState.getSourceJavaHome() } } const spawnResult = spawnSync(baseCommand, args, { @@ -103,18 +91,15 @@ function copyProjectDependencies(dependenciesFolder: FolderInfo, modulePath: str let errorLog = '' errorLog += spawnResult.error ? JSON.stringify(spawnResult.error) : '' errorLog += `${spawnResult.stderr}\n${spawnResult.stdout}` - transformByQState.appendToErrorLog(`${baseCommand} copy-dependencies failed: \n ${errorLog}`) getLogger().info( - `CodeTransformation: Maven copy-dependencies command ${baseCommand} failed, but still continuing with transformation: ${errorLog}` + `CodeTransformation: Maven command ${baseCommand} ${args} failed, but still continuing with transformation: ${errorLog}` ) throw new Error('Maven copy-deps error') - } else { - transformByQState.appendToErrorLog(`${baseCommand} copy-dependencies succeeded`) } } export async function prepareProjectDependencies(dependenciesFolder: FolderInfo, rootPomPath: string) { - await setMaven() + setMaven() getLogger().info('CodeTransformation: running Maven copy-dependencies') // pause to give chat time to update await sleep(100) @@ -132,10 +117,6 @@ export async function prepareProjectDependencies(dependenciesFolder: FolderInfo, installProjectDependencies(dependenciesFolder, rootPomPath) } catch (err) { void vscode.window.showErrorMessage(CodeWhispererConstants.cleanInstallErrorNotification) - // open build-logs.txt file to show user error logs - const logFilePath = await writeLogs() - const doc = await vscode.workspace.openTextDocument(logFilePath) - await vscode.window.showTextDocument(doc) throw err } @@ -144,7 +125,7 @@ export async function prepareProjectDependencies(dependenciesFolder: FolderInfo, } export async function getVersionData() { - const baseCommand = transformByQState.getMavenName() // will be one of: 'mvnw.cmd', './mvnw', 'mvn' + const baseCommand = transformByQState.getMavenName() const projectPath = transformByQState.getProjectPath() const args = ['-v'] const spawnResult = spawnSync(baseCommand, args, { cwd: projectPath, shell: true, encoding: 'utf-8' }) @@ -174,12 +155,9 @@ export async function getVersionData() { return [localMavenVersion, localJavaVersion] } -// run maven 'versions:dependency-updates-aggregate-report' with either 'mvnw.cmd', './mvnw', or 'mvn' (if wrapper exists, we use that, otherwise we use regular 'mvn') export function runMavenDependencyUpdateCommands(dependenciesFolder: FolderInfo) { - // baseCommand will be one of: '.\mvnw.cmd', './mvnw', 'mvn' - const baseCommand = transformByQState.getMavenName() // will be one of: 'mvnw.cmd', './mvnw', 'mvn' + const baseCommand = transformByQState.getMavenName() - // Note: IntelliJ runs 'clean' separately from 'install'. Evaluate benefits (if any) of this. const args = [ 'versions:dependency-updates-aggregate-report', `-DoutputDirectory=${dependenciesFolder.path}`, @@ -188,9 +166,9 @@ export function runMavenDependencyUpdateCommands(dependenciesFolder: FolderInfo) ] let environment = process.env - // if JAVA_HOME not found or not matching project JDK, get user input for it and set here - if (transformByQState.getJavaHome() !== undefined) { - environment = { ...process.env, JAVA_HOME: transformByQState.getJavaHome() } + + if (transformByQState.getSourceJavaHome()) { + environment = { ...process.env, JAVA_HOME: transformByQState.getSourceJavaHome() } } const spawnResult = spawnSync(baseCommand, args, { diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformationHubViewProvider.ts b/packages/core/src/codewhisperer/service/transformByQ/transformationHubViewProvider.ts index 2d0585085a9..052ef53b56c 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformationHubViewProvider.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformationHubViewProvider.ts @@ -193,6 +193,8 @@ export class TransformationHubViewProvider implements vscode.WebviewViewProvider return '

' case 'COMPLETED': return '

' + case 'AWAITING_CLIENT_ACTION': + return '

' case 'FAILED': default: return '

𐔧

' @@ -326,9 +328,19 @@ export class TransformationHubViewProvider implements vscode.WebviewViewProvider jobPlanProgress['generatePlan'] === StepProgress.Succeeded && transformByQState.isRunning() ) { - const profile = AuthUtil.instance.regionProfileManager.activeRegionProfile - planSteps = await getTransformationSteps(transformByQState.getJobId(), false, profile) - transformByQState.setPlanSteps(planSteps) + try { + planSteps = await getTransformationSteps( + transformByQState.getJobId(), + AuthUtil.instance.regionProfileManager.activeRegionProfile + ) + transformByQState.setPlanSteps(planSteps) + } catch (e: any) { + // no-op; re-use current plan steps and try again in next polling cycle + getLogger().error( + `CodeTransformation: failed to get plan steps to show updates in transformation hub, continuing transformation; error = %O`, + e + ) + } } let progressHtml // for each step that has succeeded, increment activeStepId by 1 diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts b/packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts index d1c6f368259..411571f0693 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts @@ -168,7 +168,8 @@ export class DiffModel { pathToDiff: string, pathToWorkspace: string, diffDescription: PatchInfo | undefined, - totalDiffPatches: number + totalDiffPatches: number, + isIntermediateBuild: boolean = false ): PatchFileNode { this.patchFileNodes = [] const diffContents = fs.readFileSync(pathToDiff, 'utf8') @@ -180,8 +181,9 @@ export class DiffModel { const changedFiles = parsePatch(diffContents) getLogger().info('CodeTransformation: parsed patch file successfully') - // path to the directory containing copy of the changed files in the transformed project - const pathToTmpSrcDir = this.copyProject(pathToWorkspace, changedFiles) + // if doing intermediate client-side build, pathToWorkspace is the path to the unzipped project's 'sources' directory (re-using upload ZIP) + // otherwise, we are at the very end of the transformation and need to copy the changed files in the project to show the diff(s) + const pathToTmpSrcDir = isIntermediateBuild ? pathToWorkspace : this.copyProject(pathToWorkspace, changedFiles) transformByQState.setProjectCopyFilePath(pathToTmpSrcDir) applyPatches(changedFiles, { diff --git a/packages/core/src/dev/config.ts b/packages/core/src/dev/config.ts index d5fa49b2426..b4df78f64b0 100644 --- a/packages/core/src/dev/config.ts +++ b/packages/core/src/dev/config.ts @@ -10,3 +10,6 @@ export const betaUrl = { amazonq: '', toolkit: '', } + +// TO-DO: remove when releasing CSB +export const isClientSideBuildEnabled = false diff --git a/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts b/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts index 3ec69e70b04..4b478e1876e 100644 --- a/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts +++ b/packages/core/src/test/codewhisperer/commands/transformByQ.test.ts @@ -41,6 +41,9 @@ import { setMaven, parseBuildFile, validateSQLMetadataFile, + createLocalBuildUploadZip, + validateCustomVersionsFile, + extractOriginalProjectSources, } from '../../../codewhisperer/service/transformByQ/transformFileHandler' import { uploadArtifactToS3 } from '../../../codewhisperer/indexNode' import request from '../../../shared/request' @@ -49,6 +52,19 @@ import * as nodefs from 'fs' // eslint-disable-line no-restricted-imports describe('transformByQ', function () { let fetchStub: sinon.SinonStub let tempDir: string + const validCustomVersionsFile = `name: "custom-dependency-management" +description: "Custom dependency version management for Java migration from JDK 8/11/17 to JDK 17/21" +dependencyManagement: + dependencies: + - identifier: "com.example:library1" + targetVersion: "2.1.0" + versionProperty: "library1.version" + originType: "FIRST_PARTY" + plugins: + - identifier: "com.example.plugin" + targetVersion: "1.2.0" + versionProperty: "plugin.version"` + const validSctFile = ` @@ -284,8 +300,55 @@ describe('transformByQ', function () { const tempFilePath = path.join(tempDir, tempFileName) await toFile('', tempFilePath) transformByQState.setProjectPath(tempDir) - await setMaven() - assert.strictEqual(transformByQState.getMavenName(), '.\\mvnw.cmd') + setMaven() + // mavenName should always be 'mvn' + assert.strictEqual(transformByQState.getMavenName(), 'mvn') + }) + + it(`WHEN local build zip created THEN zip contains all expected files and no unexpected files`, async function () { + const zipPath = await createLocalBuildUploadZip(tempDir, 0, 'sample stdout after running local build') + const zip = new AdmZip(zipPath) + const manifestEntry = zip.getEntry('manifest.json') + if (!manifestEntry) { + fail('manifest.json not found in the zip') + } + const manifestBuffer = manifestEntry.getData() + const manifestText = manifestBuffer.toString('utf8') + const manifest = JSON.parse(manifestText) + assert.strictEqual(manifest.capability, 'CLIENT_SIDE_BUILD') + assert.strictEqual(manifest.exitCode, 0) + assert.strictEqual(manifest.commandLogFileName, 'build-output.log') + assert.strictEqual(zip.getEntries().length, 2) // expecting only manifest.json and build-output.log + }) + + it('WHEN extractOriginalProjectSources THEN only source files are extracted to destination', async function () { + const tempDir = (await TestFolder.create()).path + const destinationPath = path.join(tempDir, 'originalCopy_jobId_artifactId') + await fs.mkdir(destinationPath) + + const zip = new AdmZip() + const testFiles = [ + { path: 'sources/file1.java', content: 'test content 1' }, + { path: 'sources/dir/file2.java', content: 'test content 2' }, + { path: 'dependencies/file3.jar', content: 'should not extract' }, + { path: 'manifest.json', content: '{"version": "1.0"}' }, + ] + + for (const file of testFiles) { + zip.addFile(file.path, Buffer.from(file.content)) + } + + const zipPath = path.join(tempDir, 'test.zip') + zip.writeZip(zipPath) + + transformByQState.setPayloadFilePath(zipPath) + + await extractOriginalProjectSources(destinationPath) + + const extractedFiles = getFilesRecursively(destinationPath, false) + assert.strictEqual(extractedFiles.length, 2) + assert(extractedFiles.includes(path.join(destinationPath, 'sources', 'file1.java'))) + assert(extractedFiles.includes(path.join(destinationPath, 'sources', 'dir', 'file2.java'))) }) it(`WHEN zip created THEN manifest.json contains test-compile custom build command`, async function () { @@ -453,6 +516,17 @@ describe('transformByQ', function () { assert.strictEqual(expectedWarning, warningMessage) }) + it(`WHEN validateCustomVersionsFile on fully valid .yaml file THEN passes validation`, async function () { + const isValidFile = await validateCustomVersionsFile(validCustomVersionsFile) + assert.strictEqual(isValidFile, true) + }) + + it(`WHEN validateCustomVersionsFile on invalid .yaml file THEN fails validation`, async function () { + const invalidFile = validCustomVersionsFile.replace('dependencyManagement', 'invalidKey') + const isValidFile = await validateCustomVersionsFile(invalidFile) + assert.strictEqual(isValidFile, false) + }) + it(`WHEN validateMetadataFile on fully valid .sct file THEN passes validation`, async function () { const isValidMetadata = await validateSQLMetadataFile(validSctFile, { tabID: 'abc123' }) assert.strictEqual(isValidMetadata, true)