feat(q-dev): /dev can be configured to run code, tests #6453

## Problem
Customers cannot configure whether their generated code should be
run and tested for quality assurance before being provided to them.

## Solution
Enable customers to configure whether to run code and
tests with /dev per project. On the first run of /dev for a project, if
a devfile.yaml exists in the root of the project, the customer is
prompted to know if they want to use the commands in the file to test
the code we generate. The feature additionally allows customers to
generate a devfile if one doesn't exist. Their choice is persisted and
configurable in the Amazon Q settings.

Co-authored-by: wilson <[email protected]>
Co-authored-by: Justin M. Keyes <[email protected]>
Co-authored-by: Karan Ahluwalia <[email protected]>
Co-authored-by: Hamed Soleimani <[email protected]>
Co-authored-by: Josh Pinkney <[email protected]>

Author: 6 people

packages/amazonq/.changes/next-release/Feature-10418729-4b46-4126-b1af-623a08f0223c.json

@@ -0,0 +1,4 @@
+{
+	"type": "Feature",
+	"description": "Add setting to allow Q /dev to run code and test commands"
+}

packages/amazonq/package.json

@@ -127,6 +127,11 @@
                     "markdownDescription": "%AWS.configuration.description.amazonq%",
                     "default": true
                 },
+                "amazonQ.allowFeatureDevelopmentToRunCodeAndTests": {
+                    "markdownDescription": "%AWS.configuration.description.featureDevelopment.allowRunningCodeAndTests%",
+                    "type": "object",
+                    "default": {}
+                },
                 "amazonQ.importRecommendationForInlineCodeSuggestions": {
                     "type": "boolean",
                     "description": "%AWS.configuration.description.amazonq.importRecommendation%",

packages/amazonq/test/unit/amazonqFeatureDev/session/session.test.ts

@@ -38,8 +38,9 @@ describe('session', () => {
     describe('preloader', () => {
         it('emits start chat telemetry', async () => {
             const session = await createSession({ messenger, conversationID, scheme: featureDevScheme })
+            session.latestMessage = 'implement twosum in typescript'
 
-            await session.preloader('implement twosum in typescript')
+            await session.preloader()
 
             assertTelemetry('amazonq_startConversationInvoke', {
                 amazonqConversationId: conversationID,

packages/amazonq/test/unit/amazonqFeatureDev/util/files.test.ts

@@ -10,55 +10,113 @@ import {
     ContentLengthError,
     maxRepoSizeBytes,
 } from 'aws-core-vscode/amazonqFeatureDev'
-import { assertTelemetry, createTestWorkspace } from 'aws-core-vscode/test'
+import { assertTelemetry, getWorkspaceFolder, TestFolder } from 'aws-core-vscode/test'
 import { fs, AmazonqCreateUpload } from 'aws-core-vscode/shared'
-import { Span } from 'aws-core-vscode/telemetry'
+import { MetricName, Span } from 'aws-core-vscode/telemetry'
 import sinon from 'sinon'
+import { CodeWhispererSettings } from 'aws-core-vscode/codewhisperer'
+
+import AdmZip from 'adm-zip'
+
+const testDevfilePrepareRepo = async (devfileEnabled: boolean) => {
+    const files: Record<string, string> = {
+        'file.md': 'test content',
+        // only include when execution is enabled
+        'devfile.yaml': 'test',
+        // .git folder is always dropped (because of vscode global exclude rules)
+        '.git/ref': '####',
+        // .gitignore should always be included
+        '.gitignore': 'node_models/*',
+        // non code files only when dev execution is enabled
+        'abc.jar': 'jar-content',
+        'data/logo.ico': 'binary-content',
+    }
+    const folder = await TestFolder.create()
+
+    for (const [fileName, content] of Object.entries(files)) {
+        await folder.write(fileName, content)
+    }
+
+    const expectedFiles = !devfileEnabled
+        ? ['./file.md', './.gitignore']
+        : ['./devfile.yaml', './file.md', './.gitignore', './abc.jar', 'data/logo.ico']
+
+    const workspace = getWorkspaceFolder(folder.path)
+    sinon
+        .stub(CodeWhispererSettings.instance, 'getAutoBuildSetting')
+        .returns(devfileEnabled ? { [workspace.uri.fsPath]: true } : {})
+
+    await testPrepareRepoData(workspace, expectedFiles)
+}
+
+const testPrepareRepoData = async (
+    workspace: vscode.WorkspaceFolder,
+    expectedFiles: string[],
+    expectedTelemetryMetrics?: Array<{ metricName: MetricName; value: any }>
+) => {
+    expectedFiles.sort((a, b) => a.localeCompare(b))
+    const telemetry = new TelemetryHelper()
+    const result = await prepareRepoData([workspace.uri.fsPath], [workspace], telemetry, {
+        record: () => {},
+    } as unknown as Span<AmazonqCreateUpload>)
+
+    assert.strictEqual(Buffer.isBuffer(result.zipFileBuffer), true)
+    // checksum is not the same across different test executions because some unique random folder names are generated
+    assert.strictEqual(result.zipFileChecksum.length, 44)
+
+    if (expectedTelemetryMetrics) {
+        for (const metric of expectedTelemetryMetrics) {
+            assertTelemetry(metric.metricName, metric.value)
+        }
+    }
+
+    // Unzip the buffer and compare the entry names
+    const zip = new AdmZip(result.zipFileBuffer)
+    const actualZipEntries = zip.getEntries().map((entry) => entry.entryName)
+    actualZipEntries.sort((a, b) => a.localeCompare(b))
+    assert.deepStrictEqual(actualZipEntries, expectedFiles)
+}
 
 describe('file utils', () => {
     describe('prepareRepoData', function () {
-        it('returns files in the workspace as a zip', async function () {
-            // these variables are a manual selection of settings for the test in order to test the collectFiles function
-            const fileAmount = 2
-            const fileNamePrefix = 'file'
-            const fileNameSuffix = '.md'
-            const fileContent = 'test content'
+        afterEach(() => {
+            sinon.restore()
+        })
 
-            const workspace = await createTestWorkspace(fileAmount, { fileNamePrefix, fileContent, fileNameSuffix })
+        it('returns files in the workspace as a zip', async function () {
+            const folder = await TestFolder.create()
+            await folder.write('file1.md', 'test content')
+            await folder.write('file2.md', 'test content')
+            const workspace = getWorkspaceFolder(folder.path)
 
-            const telemetry = new TelemetryHelper()
-            const result = await prepareRepoData([workspace.uri.fsPath], [workspace], telemetry, {
-                record: () => {},
-            } as unknown as Span<AmazonqCreateUpload>)
-            assert.strictEqual(Buffer.isBuffer(result.zipFileBuffer), true)
-            // checksum is not the same across different test executions because some unique random folder names are generated
-            assert.strictEqual(result.zipFileChecksum.length, 44)
-            assert.strictEqual(telemetry.repositorySize, 24)
+            await testPrepareRepoData(workspace, ['./file1.md', './file2.md'])
         })
 
         it('prepareRepoData ignores denied file extensions', async function () {
-            // these variables are a manual selection of settings for the test in order to test the collectFiles function
-            const fileAmount = 1
-            const fileNamePrefix = 'file'
-            const fileNameSuffix = '.mp4'
-            const fileContent = 'test content'
+            const folder = await TestFolder.create()
+            await folder.write('file.mp4', 'test content')
+            const workspace = getWorkspaceFolder(folder.path)
 
-            const workspace = await createTestWorkspace(fileAmount, { fileNamePrefix, fileContent, fileNameSuffix })
-            const telemetry = new TelemetryHelper()
-            const result = await prepareRepoData([workspace.uri.fsPath], [workspace], telemetry, {
-                record: () => {},
-            } as unknown as Span<AmazonqCreateUpload>)
-
-            assert.strictEqual(Buffer.isBuffer(result.zipFileBuffer), true)
-            // checksum is not the same across different test executions because some unique random folder names are generated
-            assert.strictEqual(result.zipFileChecksum.length, 44)
-            assert.strictEqual(telemetry.repositorySize, 0)
-            assertTelemetry('amazonq_bundleExtensionIgnored', { filenameExt: 'mp4', count: 1 })
+            await testPrepareRepoData(
+                workspace,
+                [],
+                [{ metricName: 'amazonq_bundleExtensionIgnored', value: { filenameExt: 'mp4', count: 1 } }]
+            )
+        })
+
+        it('should ignore devfile.yaml when setting is disabled', async function () {
+            await testDevfilePrepareRepo(false)
+        })
+
+        it('should include devfile.yaml when setting is enabled', async function () {
+            await testDevfilePrepareRepo(true)
         })
 
         // Test the logic that allows the customer to modify root source folder
         it('prepareRepoData throws a ContentLengthError code when repo is too big', async function () {
-            const workspace = await createTestWorkspace(1, {})
+            const folder = await TestFolder.create()
+            await folder.write('file.md', 'test content')
+            const workspace = getWorkspaceFolder(folder.path)
             const telemetry = new TelemetryHelper()
 
             sinon.stub(fs, 'stat').resolves({ size: 2 * maxRepoSizeBytes } as vscode.FileStat)

packages/core/package.nls.json

@@ -20,6 +20,7 @@
     "AWS.configuration.description.suppressPrompts": "Prompts which ask for confirmation. Checking an item suppresses the prompt.",
     "AWS.configuration.enableCodeLenses": "Enable SAM hints in source code and template.yaml files",
     "AWS.configuration.description.resources.enabledResources": "AWS resources to display in the 'Resources' portion of the explorer.",
+      "AWS.configuration.description.featureDevelopment.allowRunningCodeAndTests": "Allow /dev to run code and test commands",
     "AWS.configuration.description.experiments": "Try experimental features and give feedback. Note that experimental features may be removed at any time.\n * `jsonResourceModification` - Enables basic create, update, and delete support for cloud resources via the JSON Resources explorer component.",
     "AWS.stepFunctions.asl.format.enable.desc": "Enables the default formatter used with Amazon States Language files",
     "AWS.stepFunctions.asl.maxItemsComputed.desc": "The maximum number of outline symbols and folding regions computed (limited for performance reasons).",
@@ -362,12 +363,18 @@
     "AWS.amazonq.featureDev.pillText.selectOption": "Choose an option to proceed",
     "AWS.amazonq.featureDev.pillText.unableGenerateChanges": "Unable to generate any file changes",
     "AWS.amazonq.featureDev.pillText.provideFeedback": "Provide feedback & regenerate",
-    "AWS.amazonq.featureDev.answer.generateSuggestion": "Would you like to generate a suggestion for this? You'll review a file diff before inserting into your project.",
+    "AWS.amazonq.featureDev.pillText.generateDevFile": "Generate devfile to build code",
+    "AWS.amazonq.featureDev.pillText.acceptForProject": "Yes, use my devfile for this project",
+    "AWS.amazonq.featureDev.pillText.declineForProject": "No, thanks",
+    "AWS.amazonq.featureDev.answer.generateSuggestion": "Would you like to generate a suggestion for this? You’ll review a file diff before inserting into your project.",
     "AWS.amazonq.featureDev.answer.qGeneratedCode": "The Amazon Q Developer Agent for software development has generated code for you to review",
     "AWS.amazonq.featureDev.answer.howCodeCanBeImproved": "How can I improve the code for your use case?",
     "AWS.amazonq.featureDev.answer.updateCode": "Okay, I updated your code files. Would you like to work on another task?",
     "AWS.amazonq.featureDev.answer.sessionClosed": "Okay, I've ended this chat session. You can open a new tab to chat or start another workflow.",
     "AWS.amazonq.featureDev.answer.newTaskChanges": "What new task would you like to work on?",
+    "AWS.amazonq.featureDev.answer.devFileSuggestion": "For future tasks in this project, I can create a devfile to build and test code as I generate it. This can improve the quality of generated code. To allow me to create a devfile, choose **Generate devfile to build code**.",
+    "AWS.amazonq.featureDev.answer.settingUpdated": "I've updated your settings so I can run code and test commands based on your devfile for this project. You can update this setting under **Amazon Q: Allow Q /dev to run code and test commands**.",
+    "AWS.amazonq.featureDev.answer.devFileInRepository": "I noticed that your repository has a `devfile.yaml`. Would you like me to use the devfile to build and test your project as I generate code? \n\nFor more information on using devfiles to improve code generation, see the  <a href=\"https://docs.aws.amazon.com/amazonq/latest/qdeveloper-ug/software-dev.html\" target=\"_blank\">Amazon Q Developer documentation</a>.",
     "AWS.amazonq.featureDev.placeholder.chatInputDisabled": "Chat input is disabled",
     "AWS.amazonq.featureDev.placeholder.additionalImprovements": "Describe your task or issue in detail",
     "AWS.amazonq.featureDev.placeholder.feedback": "Provide feedback or comments",

packages/core/src/amazonq/commons/types.ts

@@ -24,6 +24,9 @@ export enum FollowUpTypes {
     NewTask = 'NewTask',
     CloseSession = 'CloseSession',
     SendFeedback = 'SendFeedback',
+    AcceptAutoBuild = 'AcceptAutoBuild',
+    DenyAutoBuild = 'DenyAutoBuild',
+    GenerateDevFile = 'GenerateDevFile',
     // Doc
     CreateDocumentation = 'CreateDocumentation',
     ChooseFolder = 'ChooseFolder',

packages/core/src/amazonqFeatureDev/constants.ts

@@ -14,6 +14,9 @@ export const featureDevChat = 'featureDevChat'
 
 export const featureName = 'Amazon Q Developer Agent for software development'
 
+export const generateDevFilePrompt =
+    "generate a devfile in my repository. Note that you should only use devfile version 2.0.0 and the only supported commands are install, build and test (are all optional). so you may have to bundle some commands together using '&&'. also you can use ”public.ecr.aws/aws-mde/universal-image:latest” as universal image if you aren’t sure which image to use. here is an example for a node repository (but don't assume it's always a node project. look at the existing repository structure before generating the devfile): schemaVersion: 2.0.0 components: - name: dev container: image: public.ecr.aws/aws-mde/universal-image:latest commands: - id: install exec: component: dev commandLine: ”npm install” - id: build exec: component: dev commandLine: ”npm run build” - id: test exec: component: dev commandLine: ”npm run test”"
+
 // Max allowed size for file collection
 export const maxRepoSizeBytes = 200 * 1024 * 1024