more refactoring

Author: David Hasani

packages/core/src/codewhisperer/commands/startTransformByQ.ts

@@ -744,7 +744,8 @@ export async function postTransformationJob() {
     }
 
     if (transformByQState.getPayloadFilePath() !== '') {
-        fs.rmSync(transformByQState.getPayloadFilePath(), { recursive: true, force: true }) // delete ZIP if it exists
+        // delete original upload ZIP at very end of transformation
+        fs.rmSync(transformByQState.getPayloadFilePath(), { recursive: true, force: true })
     }
 
     // attempt download for user

packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts

@@ -41,8 +41,8 @@ import { MetadataResult } from '../../../shared/telemetry/telemetryClient'
 import request from '../../../shared/request'
 import { JobStoppedError, ZipExceedsSizeLimitError } from '../../../amazonqGumby/errors'
 import {
-    copyDirectory,
     createLocalBuildUploadZip,
+    extractOriginalProjectSources,
     loadManifestFile,
     writeAndShowBuildLogs,
 } from './transformFileHandler'
@@ -253,7 +253,8 @@ export async function uploadPayload(payloadFileName: string, uploadContext?: Upl
  */
 const mavenExcludedExtensions = ['.repositories', '.sha1']
 
-const sourceExcludedExtensions = ['.DS_Store']
+// exclude .DS_Store (not relevant) and Maven executables (can cause permissions issues when building if user has not ran 'chmod')
+const sourceExcludedExtensions = ['.DS_Store', 'mvnw', 'mvnw.cmd']
 
 /**
  * Determines if the specified file path corresponds to a Maven metadata file
@@ -726,8 +727,14 @@ export async function pollTransformationJob(jobId: string, validStates: string[]
 
 async function attemptLocalBuild() {
     const jobId = transformByQState.getJobId()
-    const artifactId = await getClientInstructionArtifactId(jobId)
-    getLogger().info(`CodeTransformation: found artifactId = ${artifactId}`)
+    let artifactId
+    try {
+        artifactId = await getClientInstructionArtifactId(jobId)
+        getLogger().info(`CodeTransformation: found artifactId = ${artifactId}`)
+    } catch (e: any) {
+        // don't throw error so that we can try to get progress updates again in next polling cycle
+        getLogger().error(`CodeTransformation: failed to get client instruction artifact ID = %O`, e)
+    }
     if (artifactId) {
         const clientInstructionsPath = await downloadClientInstructions(jobId, artifactId)
         getLogger().info(
@@ -750,7 +757,7 @@ async function getClientInstructionArtifactId(jobId: string) {
 
 async function downloadClientInstructions(jobId: string, artifactId: string) {
     const exportDestination = `downloadClientInstructions_${jobId}_${artifactId}`
-    const exportZipPath = path.join(os.tmpdir(), `${exportDestination}.zip`)
+    const exportZipPath = path.join(os.tmpdir(), exportDestination)
 
     const exportContext: ExportContext = {
         transformationExportContext: {
@@ -766,34 +773,30 @@ async function downloadClientInstructions(jobId: string, artifactId: string) {
 }
 
 async function processClientInstructions(jobId: string, clientInstructionsPath: any, artifactId: string) {
-    const sourcePath = transformByQState.getProjectPath()
-    const destinationPath = path.join(os.tmpdir(), jobId, artifactId, 'originalCopy')
-    await copyDirectory(sourcePath, destinationPath)
+    const destinationPath = path.join(os.tmpdir(), `originalCopy_${jobId}_${artifactId}`)
+    await extractOriginalProjectSources(destinationPath)
     getLogger().info(`CodeTransformation: copied project to ${destinationPath}`)
     const diffModel = new DiffModel()
-    diffModel.parseDiff(clientInstructionsPath, destinationPath, undefined, 1, true)
+    diffModel.parseDiff(clientInstructionsPath, path.join(destinationPath, 'sources'), undefined, 1, true)
     // show user the diff.patch
     const doc = await vscode.workspace.openTextDocument(clientInstructionsPath)
     await vscode.window.showTextDocument(doc, { viewColumn: vscode.ViewColumn.One })
     await runClientSideBuild(transformByQState.getProjectCopyFilePath(), artifactId)
 }
 
-export async function runClientSideBuild(projectPath: string, clientInstructionArtifactId: string) {
-    // baseCommand will be one of: '.\mvnw.cmd', './mvnw', 'mvn'
+export async function runClientSideBuild(projectCopyPath: string, clientInstructionArtifactId: string) {
     const baseCommand = transformByQState.getMavenName()
-    const args = ['clean']
+    const args = []
     if (transformByQState.getCustomBuildCommand() === CodeWhispererConstants.skipUnitTestsBuildCommand) {
         args.push('test-compile')
     } else {
         args.push('test')
     }
-    // TO-DO / QUESTION: why not use the build command from the downloaded manifest?
-    transformByQState.appendToBuildLog(`Running ${baseCommand} ${args}`)
     const environment = { ...process.env, JAVA_HOME: transformByQState.getTargetJavaHome() }
 
     const argString = args.join(' ')
     const spawnResult = spawnSync(baseCommand, args, {
-        cwd: projectPath,
+        cwd: projectCopyPath,
         shell: true,
         encoding: 'utf-8',
         env: environment,
@@ -804,11 +807,11 @@ export async function runClientSideBuild(projectPath: string, clientInstructionA
     transformByQState.appendToBuildLog(buildLogs)
     await writeAndShowBuildLogs()
 
-    const baseDir = path.join(
+    const uploadZipBaseDir = path.join(
         os.tmpdir(),
         `clientInstructionsResult_${transformByQState.getJobId()}_${clientInstructionArtifactId}`
     )
-    const zipPath = await createLocalBuildUploadZip(baseDir, spawnResult.status, spawnResult.stdout)
+    const uploadZipPath = await createLocalBuildUploadZip(uploadZipBaseDir, spawnResult.status, spawnResult.stdout)
 
     // upload build results
     const uploadContext: UploadContext = {
@@ -817,14 +820,16 @@ export async function runClientSideBuild(projectPath: string, clientInstructionA
             uploadArtifactType: 'ClientBuildResult',
         },
     }
-    getLogger().info(`CodeTransformation: uploading client build results at ${zipPath} and resuming job now`)
-    await uploadPayload(zipPath, uploadContext)
+    getLogger().info(`CodeTransformation: uploading client build results at ${uploadZipPath} and resuming job now`)
+    await uploadPayload(uploadZipPath, uploadContext)
     await resumeTransformationJob(transformByQState.getJobId(), 'COMPLETED')
     try {
-        await fs.delete(transformByQState.getProjectCopyFilePath(), { recursive: true })
+        await fs.delete(projectCopyPath, { recursive: true })
+        await fs.delete(uploadZipBaseDir, { recursive: true })
+        // TODO: do we need to delete the downloaded client instructions and uploadZipPath? they can help in debugging
     } catch {
         getLogger().error(
-            `CodeTransformation: failed to delete project copy at ${transformByQState.getProjectCopyFilePath()} after client-side build`
+            `CodeTransformation: failed to delete project copy and uploadZipBaseDir after client-side build`
         )
     }
 }

packages/core/src/codewhisperer/service/transformByQ/transformFileHandler.ts

@@ -16,7 +16,6 @@ import globals from '../../../shared/extensionGlobals'
 import { ChatSessionManager } from '../../../amazonqGumby/chat/storages/chatSession'
 import { AbsolutePathDetectedError } from '../../../amazonqGumby/errors'
 import { getLogger } from '../../../shared/logger/logger'
-import { isWin } from '../../../shared/vscode/env'
 import AdmZip from 'adm-zip'
 
 export function getDependenciesFolderInfo(): FolderInfo {
@@ -28,7 +27,6 @@ export function getDependenciesFolderInfo(): FolderInfo {
     }
 }
 
-// TO-DO: what happens when intermediate build logs are massive from downloading dependencies? exlude those lines?
 export async function writeAndShowBuildLogs() {
     const logFilePath = path.join(os.tmpdir(), 'build-logs.txt')
     writeFileSync(logFilePath, transformByQState.getBuildLog())
@@ -48,42 +46,18 @@ export async function loadManifestFile(directory: string) {
     return manifest
 }
 
-export async function copyDirectory(sourcePath: string, destinationPath: string) {
-    await fs.mkdir(destinationPath)
-    const files = await fs.readdir(sourcePath)
-
-    for (const file of files) {
-        const sourceFilePath = path.join(sourcePath, file[0])
-        const destinationFilePath = path.join(destinationPath, file[0])
-        if (file[1] === vscode.FileType.Directory) {
-            // if the item is a directory, recursively copy it
-            const destinationFilePath = path.join(destinationPath, path.relative(sourcePath, sourceFilePath))
-            await copyDirectory(sourceFilePath, destinationFilePath)
-        } else {
-            // if the item is a file, copy its contents
-            try {
-                await fs.copy(sourceFilePath, destinationFilePath)
-            } catch (err: any) {
-                getLogger().error(
-                    `CodeTransformation: error copying file ${sourceFilePath} to ${destinationFilePath}: ${err}`
-                )
-            }
-        }
-    }
-}
-
 export async function createLocalBuildUploadZip(baseDir: string, exitCode: number | null, stdout: string) {
     const manifestFilePath = path.join(baseDir, 'manifest.json')
     const buildResultsManifest = {
         capability: 'CLIENT_SIDE_BUILD',
         exitCode: exitCode,
-        commandLogFileName: 'clientBuildLogs.log',
+        commandLogFileName: 'build-output.log',
     }
     const formattedManifest = JSON.stringify(buildResultsManifest)
-    await fs.writeFile(manifestFilePath, formattedManifest, 'utf8')
+    await fs.writeFile(manifestFilePath, formattedManifest)
 
-    const buildLogsFilePath = path.join(baseDir, 'clientBuildLogs.log')
-    await fs.writeFile(buildLogsFilePath, stdout, 'utf8')
+    const buildLogsFilePath = path.join(baseDir, 'build-output.log')
+    await fs.writeFile(buildLogsFilePath, stdout)
 
     const zip = new AdmZip()
     zip.addLocalFile(buildLogsFilePath)
@@ -95,6 +69,17 @@ export async function createLocalBuildUploadZip(baseDir: string, exitCode: numbe
     return zipPath
 }
 
+// extract the 'sources' directory of the upload ZIP so that we can apply the diff.patch to a copy of the source code
+export async function extractOriginalProjectSources(destinationPath: string) {
+    const zip = new AdmZip(transformByQState.getPayloadFilePath())
+    const zipEntries = zip.getEntries()
+    for (const zipEntry of zipEntries) {
+        if (zipEntry.entryName.startsWith('sources')) {
+            zip.extractEntryTo(zipEntry, destinationPath, true, true)
+        }
+    }
+}
+
 export async function checkBuildSystem(projectPath: string) {
     const mavenBuildFilePath = path.join(projectPath, 'pom.xml')
     if (existsSync(mavenBuildFilePath)) {
@@ -193,20 +178,10 @@ export async function validateSQLMetadataFile(fileContents: string, message: any
     return true
 }
 
-export async function setMaven() {
-    let mavenWrapperExecutableName = isWin() ? 'mvnw.cmd' : 'mvnw'
-    const mavenWrapperExecutablePath = path.join(transformByQState.getProjectPath(), mavenWrapperExecutableName)
-    if (existsSync(mavenWrapperExecutablePath)) {
-        if (mavenWrapperExecutableName === 'mvnw') {
-            mavenWrapperExecutableName = './mvnw' // add the './' for non-Windows
-        } else if (mavenWrapperExecutableName === 'mvnw.cmd') {
-            mavenWrapperExecutableName = '.\\mvnw.cmd' // add the '.\' for Windows
-        }
-        transformByQState.setMavenName(mavenWrapperExecutableName)
-    } else {
-        transformByQState.setMavenName('mvn')
-    }
-    getLogger().info(`CodeTransformation: using Maven ${transformByQState.getMavenName()}`)
+export function setMaven() {
+    // for now, just use regular Maven since the Maven executables can
+    // cause permissions issues when building if user has not ran 'chmod'
+    transformByQState.setMavenName('mvn')
 }
 
 export async function openBuildLogFile() {

packages/core/src/codewhisperer/service/transformByQ/transformMavenHandler.ts

@@ -15,15 +15,13 @@ import { setMaven, writeAndShowBuildLogs } from './transformFileHandler'
 import { throwIfCancelled } from './transformApiHandler'
 import { sleep } from '../../../shared/utilities/timeoutUtils'
 
-// run 'install' with either 'mvnw.cmd', './mvnw', or 'mvn' (if wrapper exists, we use that, otherwise we use regular 'mvn')
 function installProjectDependencies(dependenciesFolder: FolderInfo, modulePath: string) {
     telemetry.codeTransform_localBuildProject.run(() => {
         telemetry.record({ codeTransformSessionId: CodeTransformTelemetryState.instance.getSessionId() })
 
-        // baseCommand will be one of: '.\mvnw.cmd', './mvnw', 'mvn'
+        // will always be 'mvn'
         const baseCommand = transformByQState.getMavenName()
 
-        // Note: IntelliJ runs 'clean' separately from 'install'. Evaluate benefits (if any) of this.
         const args = [`-Dmaven.repo.local=${dependenciesFolder.path}`, 'clean', 'install', '-q']
 
         transformByQState.appendToBuildLog(`Running ${baseCommand} ${args.join(' ')}`)
@@ -47,14 +45,7 @@ function installProjectDependencies(dependenciesFolder: FolderInfo, modulePath:
             maxBuffer: CodeWhispererConstants.maxBufferSize,
         })
 
-        let mavenBuildCommand = transformByQState.getMavenName()
-        // slashes not allowed in telemetry
-        if (mavenBuildCommand === './mvnw') {
-            mavenBuildCommand = 'mvnw'
-        } else if (mavenBuildCommand === '.\\mvnw.cmd') {
-            mavenBuildCommand = 'mvnw.cmd'
-        }
-
+        const mavenBuildCommand = transformByQState.getMavenName()
         telemetry.record({ codeTransformBuildCommand: mavenBuildCommand as CodeTransformBuildCommand })
 
         if (spawnResult.status !== 0) {
@@ -73,7 +64,6 @@ function installProjectDependencies(dependenciesFolder: FolderInfo, modulePath:
 }
 
 function copyProjectDependencies(dependenciesFolder: FolderInfo, modulePath: string) {
-    // baseCommand will be one of: '.\mvnw.cmd', './mvnw', 'mvn'
     const baseCommand = transformByQState.getMavenName()
 
     const args = [
@@ -142,7 +132,7 @@ export async function prepareProjectDependencies(dependenciesFolder: FolderInfo,
 }
 
 export async function getVersionData() {
-    const baseCommand = transformByQState.getMavenName() // will be one of: 'mvnw.cmd', './mvnw', 'mvn'
+    const baseCommand = transformByQState.getMavenName()
     const projectPath = transformByQState.getProjectPath()
     const args = ['-v']
     const spawnResult = spawnSync(baseCommand, args, { cwd: projectPath, shell: true, encoding: 'utf-8' })
@@ -172,12 +162,9 @@ export async function getVersionData() {
     return [localMavenVersion, localJavaVersion]
 }
 
-// run maven 'versions:dependency-updates-aggregate-report' with either 'mvnw.cmd', './mvnw', or 'mvn' (if wrapper exists, we use that, otherwise we use regular 'mvn')
 export function runMavenDependencyUpdateCommands(dependenciesFolder: FolderInfo) {
-    // baseCommand will be one of: '.\mvnw.cmd', './mvnw', 'mvn'
-    const baseCommand = transformByQState.getMavenName() // will be one of: 'mvnw.cmd', './mvnw', 'mvn'
+    const baseCommand = transformByQState.getMavenName()
 
-    // Note: IntelliJ runs 'clean' separately from 'install'. Evaluate benefits (if any) of this.
     const args = [
         'versions:dependency-updates-aggregate-report',
         `-DoutputDirectory=${dependenciesFolder.path}`,

packages/core/src/codewhisperer/service/transformByQ/transformationHubViewProvider.ts

@@ -327,8 +327,16 @@ export class TransformationHubViewProvider implements vscode.WebviewViewProvider
             jobPlanProgress['generatePlan'] === StepProgress.Succeeded &&
             transformByQState.isRunning()
         ) {
-            planSteps = await getTransformationSteps(transformByQState.getJobId())
-            transformByQState.setPlanSteps(planSteps)
+            try {
+                planSteps = await getTransformationSteps(transformByQState.getJobId())
+                transformByQState.setPlanSteps(planSteps)
+            } catch (e: any) {
+                // no-op; re-use current plan steps and try again in next polling cycle
+                getLogger().error(
+                    `CodeTransformation: failed to get plan steps to show updates in transformation hub, continuing transformation; error = %O`,
+                    e
+                )
+            }
         }
         let progressHtml
         // for each step that has succeeded, increment activeStepId by 1
@@ -349,8 +357,6 @@ export class TransformationHubViewProvider implements vscode.WebviewViewProvider
                 CodeWhispererConstants.uploadingCodeStepMessage,
                 activeStepId === 0
             )
-            // TO-DO: remove this step entirely since we do entirely client-side builds
-            // TO-DO: do we still show the "Building in Java 17/21 environment" progress update?
             const buildMarkup =
                 activeStepId >= 1 && transformByQState.getTransformationType() !== TransformationType.SQL_CONVERSION // for SQL conversions, don't show buildCode step
                     ? simpleStep(

packages/core/src/codewhisperer/service/transformByQ/transformationResultsViewProvider.ts

@@ -180,9 +180,8 @@ export class DiffModel {
 
         const changedFiles = parsePatch(diffContents)
         getLogger().info('CodeTransformation: parsed patch file successfully')
-        // path to the directory containing copy of the changed files in the transformed project
-        // if doing intermediate client-side build, pathToWorkspace is the already-copied project directory
-        // otherwise, we are at the very end of the transformation and need to copy the project to show the changes
+        // if doing intermediate client-side build, pathToWorkspace is the path to the unzipped project's 'sources' directory (re-using upload ZIP)
+        // otherwise, we are at the very end of the transformation and need to copy the changed files in the project to show the diff(s)
         const pathToTmpSrcDir = isIntermediateBuild ? pathToWorkspace : this.copyProject(pathToWorkspace, changedFiles)
         transformByQState.setProjectCopyFilePath(pathToTmpSrcDir)