feat(amazonq): Adding UTG build, execute logic for internal users

package.json

@@ -41,7 +41,7 @@
         "skippedTestReport": "ts-node ./scripts/skippedTestReport.ts ./packages/amazonq/test/e2e/"
     },
     "devDependencies": {
-        "@aws-toolkits/telemetry": "^1.0.296",
+        "@aws-toolkits/telemetry": "^1.0.299",
         "@playwright/browser-chromium": "^1.43.1",
         "@stylistic/eslint-plugin": "^2.11.0",
         "@types/he": "^1.2.3",

packages/core/src/amazonq/webview/ui/main.ts

@@ -609,7 +609,7 @@ export const createMynahUI = (
                 mynahUI.addChatItem(tabID, {
                     type: ChatItemType.ANSWER_STREAM,
                 })
-            } else if (tabType === 'gumby') {
+            } else if (tabType === 'gumby' || tabType === 'testgen') {
                 connector.requestAnswer(tabID, {
                     chatMessage: prompt.prompt ?? '',
                 })

packages/core/src/amazonqTest/chat/session/session.ts

@@ -35,6 +35,7 @@ export class Session {
     public testGenerationJob: TestGenerationJob | undefined
 
     // Start Test generation
+    public isSupportedLanguage: boolean = false
     public conversationState: ConversationState = ConversationState.IDLE
     public shortAnswer: ShortAnswer | undefined
     public sourceFilePath: string = ''

packages/core/src/amazonqTest/models/constants.ts

@@ -49,7 +49,14 @@ export const cancelBuildProgressButton: ChatItemButton = {
 export const buildProgressField: ProgressField = {
     status: 'default',
     value: -1,
-    text: 'Executing...',
+    text: 'Compiling project...',
+    actions: [cancelBuildProgressButton],
+}
+
+export const fixingTestProgressField: ProgressField = {
+    status: 'default',
+    value: -1,
+    text: 'Fixing test failures...',
     actions: [cancelBuildProgressButton],
 }
 
@@ -70,8 +77,8 @@ ${planSummary ? `\n\n${planSummary}` : ''}
 `
 
 const checkIcons = {
-    wait: '☐',
-    current: '☐',
+    wait: '⏳',
+    current: '⏳',
     done: '<span style="color: green;">&#10004;</span>',
     error: '&#10060;',
 }
@@ -85,9 +92,7 @@ const stepStatuses: StepStatus[] = []
 
 export const testGenBuildProgressMessage = (currentStep: TestGenerationBuildStep, status?: string) => {
     const session = ChatSessionManager.Instance.getSession()
-    const statusText = BuildStatus[session.buildStatus].toLowerCase()
-    const icon = session.buildStatus === BuildStatus.SUCCESS ? checkIcons['done'] : checkIcons['error']
-    let message = `Sure. This may take a few minutes and I'll share updates on my progress here.
+    let message = `Sure. This may take a few minutes and I'll update the progress here.\n
 **Progress summary**\n\n`
 
     if (currentStep === TestGenerationBuildStep.START_STEP) {
@@ -96,41 +101,53 @@ export const testGenBuildProgressMessage = (currentStep: TestGenerationBuildStep
 
     updateStepStatuses(currentStep, status)
 
-    if (currentStep >= TestGenerationBuildStep.RUN_BUILD) {
-        message += `${getIconForStep(TestGenerationBuildStep.RUN_BUILD)} Started build execution\n`
+    if (currentStep === TestGenerationBuildStep.RUN_BUILD) {
+        message += `${getIconForStep(TestGenerationBuildStep.RUN_BUILD)} Project compiling\n`
+    } else if (currentStep >= TestGenerationBuildStep.RUN_BUILD) {
+        message += `${getIconForStep(TestGenerationBuildStep.RUN_BUILD)} Project compiled\n`
     }
 
-    if (currentStep >= TestGenerationBuildStep.RUN_EXECUTION_TESTS) {
-        message += `${getIconForStep(TestGenerationBuildStep.RUN_EXECUTION_TESTS)} Executing tests\n`
+    if (currentStep === TestGenerationBuildStep.RUN_EXECUTION_TESTS) {
+        message += `${getIconForStep(TestGenerationBuildStep.RUN_EXECUTION_TESTS)} Running tests\n`
+    } else if (currentStep >= TestGenerationBuildStep.RUN_EXECUTION_TESTS) {
+        message += `${getIconForStep(TestGenerationBuildStep.RUN_EXECUTION_TESTS)} Ran tests\n`
     }
 
-    if (currentStep >= TestGenerationBuildStep.FIXING_TEST_CASES && session.buildStatus === BuildStatus.FAILURE) {
-        message += `${getIconForStep(TestGenerationBuildStep.FIXING_TEST_CASES)} Fixing errors in tests\n\n`
+    if (currentStep === TestGenerationBuildStep.FIXING_TEST_CASES && session.buildStatus === BuildStatus.FAILURE) {
+        message += `${getIconForStep(TestGenerationBuildStep.FIXING_TEST_CASES)} Fixing test failures\n\n`
+    } else if (
+        currentStep >= TestGenerationBuildStep.FIXING_TEST_CASES &&
+        session.buildStatus === BuildStatus.FAILURE
+    ) {
+        message += `${checkIcons['done']} Fixed test failures\n\n`
     }
 
-    if (currentStep > TestGenerationBuildStep.PROCESS_TEST_RESULTS) {
-        message += `**Test case summary**
-${session.shortAnswer?.testCoverage ? `- Unit test coverage ${session.shortAnswer?.testCoverage}%` : ``}
-${icon} Build ${statusText}
-${icon} Assertion ${statusText}`
-        // TODO: Update Assertion %
+    if (currentStep > TestGenerationBuildStep.PROCESS_TEST_RESULTS && session.buildStatus === BuildStatus.FAILURE) {
+        message += `**Results**\n
+Amazon Q executed the tests and identified at least one failure. Below are the suggested fixes.\n\n`
     }
 
     return message.trim()
 }
-// TODO: Work on UX to show the build error in the progress message
+
 const updateStepStatuses = (currentStep: TestGenerationBuildStep, status?: string) => {
+    const session = ChatSessionManager.Instance.getSession()
     for (let step = TestGenerationBuildStep.INSTALL_DEPENDENCIES; step <= currentStep; step++) {
         const stepStatus: StepStatus = {
             step: step,
             status: 'wait',
         }
 
-        if (step === currentStep) {
-            stepStatus.status = status === 'failed' ? 'error' : 'current'
-        } else if (step < currentStep) {
-            stepStatus.status = 'done'
-        }
+        stepStatus.status =
+            step === currentStep
+                ? status === 'error' || status === 'done'
+                    ? status
+                    : 'current'
+                : step < currentStep
+                  ? session.buildStatus === BuildStatus.FAILURE
+                      ? 'error'
+                      : 'done'
+                  : stepStatus.status
 
         const existingIndex = stepStatuses.findIndex((s) => s.step === step)
         if (existingIndex !== -1) {

packages/core/src/codewhisperer/commands/startTestGeneration.ts

@@ -44,6 +44,7 @@ export async function startTestGenerationProcess(
             logger.verbose(`Tab ID mismatch: ${tabID} !== ${session.tabID}`)
             return
         }
+        testGenState.setToNotStarted()
         /**
          * Step 1: Zip the project
          */
@@ -212,7 +213,7 @@ function runLocalBuild(
         if (spawnResult.stderr) {
             spawnResult.stderr.on('data', async (data) => {
                 const output = data.toString().trim()
-                getLogger().warn(`BUILD ERROR: ${output}`)
+                getLogger().warn(`${output}`)
                 buildLogs += output
             })
         }

packages/core/src/codewhisperer/models/model.ts

@@ -1177,7 +1177,7 @@ export interface ShortAnswerReference {
 
 export interface ShortAnswer {
     testFilePath: string
-    buildCommands: string[]
+    buildCommand: string
     planSummary: string
     sourceFilePath?: string
     testFramework?: string

packages/core/src/codewhisperer/service/testGenHandler.ts

@@ -41,6 +41,7 @@ import fs from '../../shared/fs/fs'
 export function throwIfCancelled() {
     // TODO: fileName will be '' if user gives propt without opening
     if (testGenState.isCancelling()) {
+        testGenState.setToNotStarted()
         throw new TestGenStoppedError()
     }
 }
@@ -151,11 +152,13 @@ export async function pollTestJobStatus(
         logger.debug('pollTestJobStatus testGenerationJob %O', resp.testGenerationJob)
         ChatSessionManager.Instance.getSession().testGenerationJob = resp.testGenerationJob
         const progressRate = resp.testGenerationJob?.progressRate ?? 0
-        testGenState.getChatControllers()?.sendUpdatePromptProgress.fire({
-            tabID: ChatSessionManager.Instance.getSession().tabID,
-            status: 'InProgress',
-            progressRate,
-        })
+        if (initialExecution) {
+            testGenState.getChatControllers()?.sendUpdatePromptProgress.fire({
+                tabID: ChatSessionManager.Instance.getSession().tabID,
+                status: 'InProgress',
+                progressRate,
+            })
+        }
         const shortAnswerString = resp.testGenerationJob?.shortAnswer
         if (shortAnswerString) {
             const parsedShortAnswer = JSON.parse(shortAnswerString)
@@ -253,13 +256,11 @@ export async function exportResultsArchive(
                 filePath: testFilePath,
                 projectName,
             })
-
-            // If User accepts the diff
-            testGenState.getChatControllers()?.sendUpdatePromptProgress.fire({
-                tabID: ChatSessionManager.Instance.getSession().tabID,
-                status: 'Completed',
-            })
         }
+        testGenState.getChatControllers()?.sendUpdatePromptProgress.fire({
+            tabID: ChatSessionManager.Instance.getSession().tabID,
+            status: 'Completed',
+        })
     } catch (e) {
         session.numberOfTestsGenerated = 0
         downloadErrorMessage = (e as Error).message

packages/core/src/codewhisperer/util/telemetryHelper.ts

@@ -13,6 +13,7 @@ import {
     CodewhispererPreviousSuggestionState,
     CodewhispererUserDecision,
     CodewhispererUserTriggerDecision,
+    Result,
     telemetry,
 } from '../../shared/telemetry/telemetry'
 import { CodewhispererCompletionType, CodewhispererSuggestionState } from '../../shared/telemetry/telemetry'
@@ -27,7 +28,7 @@ import { CodeWhispererSupplementalContext } from '../models/model'
 import { FeatureConfigProvider } from '../../shared/featureConfig'
 import { CodeScanRemediationsEventType } from '../client/codewhispereruserclient'
 import { CodeAnalysisScope as CodeAnalysisScopeClientSide } from '../models/constants'
-import { Session } from '../../amazonqTest/chat/session/session'
+import { BuildStatus, Session } from '../../amazonqTest/chat/session/session'
 
 export class TelemetryHelper {
     // Some variables for client component latency
@@ -90,7 +91,7 @@ export class TelemetryHelper {
         telemetry.amazonq_utgGenerateTests.emit({
             cwsprChatProgrammingLanguage: session.fileLanguage ?? 'plaintext',
             hasUserPromptSupplied: session.hasUserPromptSupplied,
-            isSupportedLanguage: isSupportedLanguage,
+            isSupportedLanguage: session.isSupportedLanguage,
             isFileInWorkspace: isFileInWorkspace,
             result: result,
             artifactsUploadDuration: artifactsUploadDuration,
@@ -113,6 +114,61 @@ export class TelemetryHelper {
         })
     }
 
+    /**
+     * amazonq_unitTestGeneration event is emitted in 6 cases for Unit Test Generation
+     * 1) If user close chat window
+     * 2) If build succeeds in build and execute
+     * 3) If user reject generated code
+     * 4) If user click on skip and finish button
+     * 5) If user completes 3 iterations in build and execute
+     * 6) If there is any error/exception
+     */
+    public sendUnitTestGenerationEvent(
+        session: Session,
+        reasonDesc?: string,
+        artifactsUploadDuration?: number,
+        buildZipFileBytes?: number,
+        acceptedCharactersCount?: number,
+        acceptedCount?: number,
+        acceptedLinesCount?: number,
+        generatedCharactersCount?: number,
+        generatedCount?: number,
+        generatedLinesCount?: number,
+        reason?: string,
+        result?: BuildStatus
+    ) {
+        telemetry.amazonq_unitTestGeneration.emit({
+            cwsprChatProgrammingLanguage: session.fileLanguage ?? 'plaintext',
+            hasUserPromptSupplied: session.hasUserPromptSupplied,
+            isSupportedLanguage: session.isSupportedLanguage,
+            result:
+                result === BuildStatus.SUCCESS
+                    ? 'Succeeded'
+                    : result === BuildStatus.FAILURE
+                      ? 'Failed'
+                      : ('Cancelled' as Result),
+
+            artifactsUploadDuration: artifactsUploadDuration,
+            buildZipFileBytes: buildZipFileBytes,
+            credentialStartUrl: AuthUtil.instance.startUrl,
+            acceptedCharactersCount: acceptedCharactersCount,
+            acceptedCount: acceptedCount,
+            acceptedLinesCount: acceptedLinesCount,
+            generatedCharactersCount: generatedCharactersCount,
+            generatedCount: generatedCount,
+            generatedLinesCount: generatedLinesCount,
+            isCodeBlockSelected: session.isCodeBlockSelected,
+            jobGroup: session.testGenerationJobGroupName,
+            jobId: session.listOfTestGenerationJobId.at(-1) ?? undefined,
+            perfClientLatency: 0, // TODO for V2 version: Calculate the client side latency between iterations
+            requestId: session.startTestGenerationRequestId,
+            reasonDesc: reasonDesc,
+            reason: reason,
+            update: (session.updatedBuildCommands?.length ?? 0) > 0, // If user modifies command return true else false
+            count: session.listOfTestGenerationJobId.length - 1, // Number of build and execute iterations if count = -1 then user did not even start the build and cycle loop and failed/terminated at the vanilla UTG.
+        })
+    }
+
     public recordServiceInvocationTelemetry(
         requestId: string,
         sessionId: string,